tools/lib/bpf/linker.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2 /*
   3  * BPF static linker
   4  *
   5  * Copyright (c) 2021 Facebook
   6  */
   7 #include <stdbool.h>
   8 #include <stddef.h>
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <unistd.h>
  13 #include <errno.h>
  14 #include <linux/err.h>
  15 #include <linux/btf.h>
  16 #include <elf.h>
  17 #include <libelf.h>
  18 #include <gelf.h>
  19 #include <fcntl.h>
  20 #include "libbpf.h"
  21 #include "btf.h"
  22 #include "libbpf_internal.h"
  23 #include "strset.h"
  24
  25 struct src_sec {
  26         const char *sec_name;
  27         /* positional (not necessarily ELF) index in an array of sections */
  28         int id;
  29         /* positional (not necessarily ELF) index of a matching section in a final object file */
  30         int dst_id;
  31         /* section data offset in a matching output section */
  32         int dst_off;
  33         /* whether section is omitted from the final ELF file */
  34         bool skipped;
  35         /* whether section is an ephemeral section, not mapped to an ELF section */
  36         bool ephemeral;
  37
  38         /* ELF info */
  39         size_t sec_idx;
  40         Elf_Scn *scn;
  41         Elf64_Shdr *shdr;
  42         Elf_Data *data;
  43
  44         /* corresponding BTF DATASEC type ID */
  45         int sec_type_id;
  46 };
  47
  48 struct src_obj {
  49         const char *filename;
  50         int fd;
  51         Elf *elf;
  52         /* Section header strings section index */
  53         size_t shstrs_sec_idx;
  54         /* SYMTAB section index */
  55         size_t symtab_sec_idx;
  56
  57         struct btf *btf;
  58         struct btf_ext *btf_ext;
  59
  60         /* List of sections (including ephemeral). Slot zero is unused. */
  61         struct src_sec *secs;
  62         int sec_cnt;
  63
  64         /* mapping of symbol indices from src to dst ELF */
  65         int *sym_map;
  66         /* mapping from the src BTF type IDs to dst ones */
  67         int *btf_type_map;
  68 };
  69
  70 /* single .BTF.ext data section */
  71 struct btf_ext_sec_data {
  72         size_t rec_cnt;
  73         __u32 rec_sz;
  74         void *recs;
  75 };
  76
  77 struct dst_sec {
  78         char *sec_name;
  79         /* positional (not necessarily ELF) index in an array of sections */
  80         int id;
  81
  82         /* ELF info */
  83         size_t sec_idx;
  84         Elf_Scn *scn;
  85         Elf64_Shdr *shdr;
  86         Elf_Data *data;
  87
  88         /* final output section size */
  89         int sec_sz;
  90         /* final output contents of the section */
  91         void *raw_data;
  92
  93         /* corresponding STT_SECTION symbol index in SYMTAB */
  94         int sec_sym_idx;
  95
  96         /* section's DATASEC variable info, emitted on BTF finalization */
  97         bool has_btf;
  98         int sec_var_cnt;
  99         struct btf_var_secinfo *sec_vars;
 100
 101         /* section's .BTF.ext data */
 102         struct btf_ext_sec_data func_info;
 103         struct btf_ext_sec_data line_info;
 104         struct btf_ext_sec_data core_relo_info;
 105 };
 106
 107 struct bpf_linker {
 108         char *filename;
 109         int fd;
 110         Elf *elf;
 111         Elf64_Ehdr *elf_hdr;
 112
 113         /* Output sections metadata */
 114         struct dst_sec *secs;
 115         int sec_cnt;
 116
 117         struct strset *strtab_strs; /* STRTAB unique strings */
 118         size_t strtab_sec_idx; /* STRTAB section index */
 119         size_t symtab_sec_idx; /* SYMTAB section index */
 120
 121         struct btf *btf;
 122         struct btf_ext *btf_ext;
 123 };
 124
 125 #define pr_warn_elf(fmt, ...)                                                                   \
 126         libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 127
 128 static int init_output_elf(struct bpf_linker *linker, const char *file);
 129
 130 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj);
 131 static int linker_sanity_check_elf(struct src_obj *obj);
 132 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 133 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 134 static int linker_sanity_check_btf(struct src_obj *obj);
 135 static int linker_sanity_check_btf_ext(struct src_obj *obj);
 136 static int linker_fixup_btf(struct src_obj *obj);
 137 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 138 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 139 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 140 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 141 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 142
 143 static int finalize_btf(struct bpf_linker *linker);
 144 static int finalize_btf_ext(struct bpf_linker *linker);
 145
 146 void bpf_linker__free(struct bpf_linker *linker)
 147 {
 148         int i;
 149
 150         if (!linker)
 151                 return;
 152
 153         free(linker->filename);
 154
 155         if (linker->elf)
 156                 elf_end(linker->elf);
 157
 158         if (linker->fd >= 0)
 159                 close(linker->fd);
 160
 161         strset__free(linker->strtab_strs);
 162
 163         btf__free(linker->btf);
 164         btf_ext__free(linker->btf_ext);
 165
 166         for (i = 1; i < linker->sec_cnt; i++) {
 167                 struct dst_sec *sec = &linker->secs[i];
 168
 169                 free(sec->sec_name);
 170                 free(sec->raw_data);
 171                 free(sec->sec_vars);
 172
 173                 free(sec->func_info.recs);
 174                 free(sec->line_info.recs);
 175                 free(sec->core_relo_info.recs);
 176         }
 177         free(linker->secs);
 178
 179         free(linker);
 180 }
 181
 182 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 183 {
 184         struct bpf_linker *linker;
 185         int err;
 186
 187         if (!OPTS_VALID(opts, bpf_linker_opts))
 188                 return NULL;
 189
 190         if (elf_version(EV_CURRENT) == EV_NONE) {
 191                 pr_warn_elf("libelf initialization failed");
 192                 return NULL;
 193         }
 194
 195         linker = calloc(1, sizeof(*linker));
 196         if (!linker)
 197                 return NULL;
 198
 199         linker->fd = -1;
 200
 201         err = init_output_elf(linker, filename);
 202         if (err)
 203                 goto err_out;
 204
 205         return linker;
 206
 207 err_out:
 208         bpf_linker__free(linker);
 209         return NULL;
 210 }
 211
 212 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 213 {
 214         struct dst_sec *secs = linker->secs, *sec;
 215         size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 216
 217         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 218         if (!secs)
 219                 return NULL;
 220
 221         /* zero out newly allocated memory */
 222         memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 223
 224         linker->secs = secs;
 225         linker->sec_cnt = new_cnt;
 226
 227         sec = &linker->secs[new_cnt - 1];
 228         sec->id = new_cnt - 1;
 229         sec->sec_name = strdup(sec_name);
 230         if (!sec->sec_name)
 231                 return NULL;
 232
 233         return sec;
 234 }
 235
 236 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 237 {
 238         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 239         Elf64_Sym *syms, *sym;
 240         size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 241
 242         syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 243         if (!syms)
 244                 return NULL;
 245
 246         sym = &syms[sym_cnt];
 247         memset(sym, 0, sizeof(*sym));
 248
 249         symtab->raw_data = syms;
 250         symtab->sec_sz += sizeof(*sym);
 251         symtab->shdr->sh_size += sizeof(*sym);
 252         symtab->data->d_size += sizeof(*sym);
 253
 254         if (sym_idx)
 255                 *sym_idx = sym_cnt;
 256
 257         return sym;
 258 }
 259
 260 static int init_output_elf(struct bpf_linker *linker, const char *file)
 261 {
 262         int err, str_off;
 263         Elf64_Sym *init_sym;
 264         struct dst_sec *sec;
 265
 266         linker->filename = strdup(file);
 267         if (!linker->filename)
 268                 return -ENOMEM;
 269
 270         linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
 271         if (linker->fd < 0) {
 272                 err = -errno;
 273                 pr_warn("failed to create '%s': %d\n", file, err);
 274                 return err;
 275         }
 276
 277         linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 278         if (!linker->elf) {
 279                 pr_warn_elf("failed to create ELF object");
 280                 return -EINVAL;
 281         }
 282
 283         /* ELF header */
 284         linker->elf_hdr = elf64_newehdr(linker->elf);
 285         if (!linker->elf_hdr) {
 286                 pr_warn_elf("failed to create ELF header");
 287                 return -EINVAL;
 288         }
 289
 290         linker->elf_hdr->e_machine = EM_BPF;
 291         linker->elf_hdr->e_type = ET_REL;
 292 #if __BYTE_ORDER == __LITTLE_ENDIAN
 293         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
 294 #elif __BYTE_ORDER == __BIG_ENDIAN
 295         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
 296 #else
 297 #error "Unknown __BYTE_ORDER"
 298 #endif
 299
 300         /* STRTAB */
 301         /* initialize strset with an empty string to conform to ELF */
 302         linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 303         if (libbpf_get_error(linker->strtab_strs))
 304                 return libbpf_get_error(linker->strtab_strs);
 305
 306         sec = add_dst_sec(linker, ".strtab");
 307         if (!sec)
 308                 return -ENOMEM;
 309
 310         sec->scn = elf_newscn(linker->elf);
 311         if (!sec->scn) {
 312                 pr_warn_elf("failed to create STRTAB section");
 313                 return -EINVAL;
 314         }
 315
 316         sec->shdr = elf64_getshdr(sec->scn);
 317         if (!sec->shdr)
 318                 return -EINVAL;
 319
 320         sec->data = elf_newdata(sec->scn);
 321         if (!sec->data) {
 322                 pr_warn_elf("failed to create STRTAB data");
 323                 return -EINVAL;
 324         }
 325
 326         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 327         if (str_off < 0)
 328                 return str_off;
 329
 330         sec->sec_idx = elf_ndxscn(sec->scn);
 331         linker->elf_hdr->e_shstrndx = sec->sec_idx;
 332         linker->strtab_sec_idx = sec->sec_idx;
 333
 334         sec->shdr->sh_name = str_off;
 335         sec->shdr->sh_type = SHT_STRTAB;
 336         sec->shdr->sh_flags = SHF_STRINGS;
 337         sec->shdr->sh_offset = 0;
 338         sec->shdr->sh_link = 0;
 339         sec->shdr->sh_info = 0;
 340         sec->shdr->sh_addralign = 1;
 341         sec->shdr->sh_size = sec->sec_sz = 0;
 342         sec->shdr->sh_entsize = 0;
 343
 344         /* SYMTAB */
 345         sec = add_dst_sec(linker, ".symtab");
 346         if (!sec)
 347                 return -ENOMEM;
 348
 349         sec->scn = elf_newscn(linker->elf);
 350         if (!sec->scn) {
 351                 pr_warn_elf("failed to create SYMTAB section");
 352                 return -EINVAL;
 353         }
 354
 355         sec->shdr = elf64_getshdr(sec->scn);
 356         if (!sec->shdr)
 357                 return -EINVAL;
 358
 359         sec->data = elf_newdata(sec->scn);
 360         if (!sec->data) {
 361                 pr_warn_elf("failed to create SYMTAB data");
 362                 return -EINVAL;
 363         }
 364
 365         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 366         if (str_off < 0)
 367                 return str_off;
 368
 369         sec->sec_idx = elf_ndxscn(sec->scn);
 370         linker->symtab_sec_idx = sec->sec_idx;
 371
 372         sec->shdr->sh_name = str_off;
 373         sec->shdr->sh_type = SHT_SYMTAB;
 374         sec->shdr->sh_flags = 0;
 375         sec->shdr->sh_offset = 0;
 376         sec->shdr->sh_link = linker->strtab_sec_idx;
 377         /* sh_info should be one greater than the index of the last local
 378          * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 379          */
 380         sec->shdr->sh_info = 0;
 381         sec->shdr->sh_addralign = 8;
 382         sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 383
 384         /* .BTF */
 385         linker->btf = btf__new_empty();
 386         err = libbpf_get_error(linker->btf);
 387         if (err)
 388                 return err;
 389
 390         /* add the special all-zero symbol */
 391         init_sym = add_new_sym(linker, NULL);
 392         if (!init_sym)
 393                 return -EINVAL;
 394
 395         init_sym->st_name = 0;
 396         init_sym->st_info = 0;
 397         init_sym->st_other = 0;
 398         init_sym->st_shndx = SHN_UNDEF;
 399         init_sym->st_value = 0;
 400         init_sym->st_size = 0;
 401
 402         return 0;
 403 }
 404
 405 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename)
 406 {
 407         struct src_obj obj = {};
 408         int err = 0;
 409
 410         if (!linker->elf)
 411                 return -EINVAL;
 412
 413         err = err ?: linker_load_obj_file(linker, filename, &obj);
 414         err = err ?: linker_append_sec_data(linker, &obj);
 415         err = err ?: linker_append_elf_syms(linker, &obj);
 416         err = err ?: linker_append_elf_relos(linker, &obj);
 417         err = err ?: linker_append_btf(linker, &obj);
 418         err = err ?: linker_append_btf_ext(linker, &obj);
 419
 420         /* free up src_obj resources */
 421         free(obj.btf_type_map);
 422         btf__free(obj.btf);
 423         btf_ext__free(obj.btf_ext);
 424         free(obj.secs);
 425         free(obj.sym_map);
 426         if (obj.elf)
 427                 elf_end(obj.elf);
 428         if (obj.fd >= 0)
 429                 close(obj.fd);
 430
 431         return err;
 432 }
 433
 434 static bool is_dwarf_sec_name(const char *name)
 435 {
 436         /* approximation, but the actual list is too long */
 437         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 438 }
 439
 440 static bool is_ignored_sec(struct src_sec *sec)
 441 {
 442         Elf64_Shdr *shdr = sec->shdr;
 443         const char *name = sec->sec_name;
 444
 445         /* no special handling of .strtab */
 446         if (shdr->sh_type == SHT_STRTAB)
 447                 return true;
 448
 449         /* ignore .llvm_addrsig section as well */
 450         if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 451                 return true;
 452
 453         /* no subprograms will lead to an empty .text section, ignore it */
 454         if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 455             strcmp(sec->sec_name, ".text") == 0)
 456                 return true;
 457
 458         /* DWARF sections */
 459         if (is_dwarf_sec_name(sec->sec_name))
 460                 return true;
 461
 462         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 463                 name += sizeof(".rel") - 1;
 464                 /* DWARF section relocations */
 465                 if (is_dwarf_sec_name(name))
 466                         return true;
 467
 468                 /* .BTF and .BTF.ext don't need relocations */
 469                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
 470                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
 471                         return true;
 472         }
 473
 474         return false;
 475 }
 476
 477 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 478 {
 479         struct src_sec *secs = obj->secs, *sec;
 480         size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 481
 482         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 483         if (!secs)
 484                 return NULL;
 485
 486         /* zero out newly allocated memory */
 487         memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 488
 489         obj->secs = secs;
 490         obj->sec_cnt = new_cnt;
 491
 492         sec = &obj->secs[new_cnt - 1];
 493         sec->id = new_cnt - 1;
 494         sec->sec_name = sec_name;
 495
 496         return sec;
 497 }
 498
 499 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj)
 500 {
 501 #if __BYTE_ORDER == __LITTLE_ENDIAN
 502         const int host_endianness = ELFDATA2LSB;
 503 #elif __BYTE_ORDER == __BIG_ENDIAN
 504         const int host_endianness = ELFDATA2MSB;
 505 #else
 506 #error "Unknown __BYTE_ORDER"
 507 #endif
 508         int err = 0;
 509         Elf_Scn *scn;
 510         Elf_Data *data;
 511         Elf64_Ehdr *ehdr;
 512         Elf64_Shdr *shdr;
 513         struct src_sec *sec;
 514
 515         pr_debug("linker: adding object file '%s'...\n", filename);
 516
 517         obj->filename = filename;
 518
 519         obj->fd = open(filename, O_RDONLY);
 520         if (obj->fd < 0) {
 521                 err = -errno;
 522                 pr_warn("failed to open file '%s': %d\n", filename, err);
 523                 return err;
 524         }
 525         obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 526         if (!obj->elf) {
 527                 err = -errno;
 528                 pr_warn_elf("failed to parse ELF file '%s'", filename);
 529                 return err;
 530         }
 531
 532         /* Sanity check ELF file high-level properties */
 533         ehdr = elf64_getehdr(obj->elf);
 534         if (!ehdr) {
 535                 err = -errno;
 536                 pr_warn_elf("failed to get ELF header for %s", filename);
 537                 return err;
 538         }
 539         if (ehdr->e_ident[EI_DATA] != host_endianness) {
 540                 err = -EOPNOTSUPP;
 541                 pr_warn_elf("unsupported byte order of ELF file %s", filename);
 542                 return err;
 543         }
 544         if (ehdr->e_type != ET_REL
 545             || ehdr->e_machine != EM_BPF
 546             || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 547                 err = -EOPNOTSUPP;
 548                 pr_warn_elf("unsupported kind of ELF file %s", filename);
 549                 return err;
 550         }
 551
 552         if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 553                 err = -errno;
 554                 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
 555                 return err;
 556         }
 557
 558         scn = NULL;
 559         while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 560                 size_t sec_idx = elf_ndxscn(scn);
 561                 const char *sec_name;
 562
 563                 shdr = elf64_getshdr(scn);
 564                 if (!shdr) {
 565                         err = -errno;
 566                         pr_warn_elf("failed to get section #%zu header for %s",
 567                                     sec_idx, filename);
 568                         return err;
 569                 }
 570
 571                 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 572                 if (!sec_name) {
 573                         err = -errno;
 574                         pr_warn_elf("failed to get section #%zu name for %s",
 575                                     sec_idx, filename);
 576                         return err;
 577                 }
 578
 579                 data = elf_getdata(scn, 0);
 580                 if (!data) {
 581                         err = -errno;
 582                         pr_warn_elf("failed to get section #%zu (%s) data from %s",
 583                                     sec_idx, sec_name, filename);
 584                         return err;
 585                 }
 586
 587                 sec = add_src_sec(obj, sec_name);
 588                 if (!sec)
 589                         return -ENOMEM;
 590
 591                 sec->scn = scn;
 592                 sec->shdr = shdr;
 593                 sec->data = data;
 594                 sec->sec_idx = elf_ndxscn(scn);
 595
 596                 if (is_ignored_sec(sec)) {
 597                         sec->skipped = true;
 598                         continue;
 599                 }
 600
 601                 switch (shdr->sh_type) {
 602                 case SHT_SYMTAB:
 603                         if (obj->symtab_sec_idx) {
 604                                 err = -EOPNOTSUPP;
 605                                 pr_warn("multiple SYMTAB sections found, not supported\n");
 606                                 return err;
 607                         }
 608                         obj->symtab_sec_idx = sec_idx;
 609                         break;
 610                 case SHT_STRTAB:
 611                         /* we'll construct our own string table */
 612                         break;
 613                 case SHT_PROGBITS:
 614                         if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 615                                 obj->btf = btf__new(data->d_buf, shdr->sh_size);
 616                                 err = libbpf_get_error(obj->btf);
 617                                 if (err) {
 618                                         pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
 619                                         return err;
 620                                 }
 621                                 sec->skipped = true;
 622                                 continue;
 623                         }
 624                         if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 625                                 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 626                                 err = libbpf_get_error(obj->btf_ext);
 627                                 if (err) {
 628                                         pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
 629                                         return err;
 630                                 }
 631                                 sec->skipped = true;
 632                                 continue;
 633                         }
 634
 635                         /* data & code */
 636                         break;
 637                 case SHT_NOBITS:
 638                         /* BSS */
 639                         break;
 640                 case SHT_REL:
 641                         /* relocations */
 642                         break;
 643                 default:
 644                         pr_warn("unrecognized section #%zu (%s) in %s\n",
 645                                 sec_idx, sec_name, filename);
 646                         err = -EINVAL;
 647                         return err;
 648                 }
 649         }
 650
 651         err = err ?: linker_sanity_check_elf(obj);
 652         err = err ?: linker_sanity_check_btf(obj);
 653         err = err ?: linker_sanity_check_btf_ext(obj);
 654         err = err ?: linker_fixup_btf(obj);
 655
 656         return err;
 657 }
 658
 659 static bool is_pow_of_2(size_t x)
 660 {
 661         return x && (x & (x - 1)) == 0;
 662 }
 663
 664 static int linker_sanity_check_elf(struct src_obj *obj)
 665 {
 666         struct src_sec *sec;
 667         int i, err;
 668
 669         if (!obj->symtab_sec_idx) {
 670                 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 671                 return -EINVAL;
 672         }
 673         if (!obj->shstrs_sec_idx) {
 674                 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 675                 return -EINVAL;
 676         }
 677
 678         for (i = 1; i < obj->sec_cnt; i++) {
 679                 sec = &obj->secs[i];
 680
 681                 if (sec->sec_name[0] == '\0') {
 682                         pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 683                         return -EINVAL;
 684                 }
 685
 686                 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
 687                         return -EINVAL;
 688                 if (sec->shdr->sh_addralign != sec->data->d_align)
 689                         return -EINVAL;
 690
 691                 if (sec->shdr->sh_size != sec->data->d_size)
 692                         return -EINVAL;
 693
 694                 switch (sec->shdr->sh_type) {
 695                 case SHT_SYMTAB:
 696                         err = linker_sanity_check_elf_symtab(obj, sec);
 697                         if (err)
 698                                 return err;
 699                         break;
 700                 case SHT_STRTAB:
 701                         break;
 702                 case SHT_PROGBITS:
 703                         if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 704                                 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
 705                                         return -EINVAL;
 706                         }
 707                         break;
 708                 case SHT_NOBITS:
 709                         break;
 710                 case SHT_REL:
 711                         err = linker_sanity_check_elf_relos(obj, sec);
 712                         if (err)
 713                                 return err;
 714                         break;
 715                 case SHT_LLVM_ADDRSIG:
 716                         break;
 717                 default:
 718                         pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 719                                 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 720                         return -EINVAL;
 721                 }
 722         }
 723
 724         return 0;
 725 }
 726
 727 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 728 {
 729         struct src_sec *link_sec;
 730         Elf64_Sym *sym;
 731         int i, n;
 732
 733         if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 734                 return -EINVAL;
 735         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 736                 return -EINVAL;
 737
 738         if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 739                 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 740                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 741                 return -EINVAL;
 742         }
 743         link_sec = &obj->secs[sec->shdr->sh_link];
 744         if (link_sec->shdr->sh_type != SHT_STRTAB) {
 745                 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 746                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 747                 return -EINVAL;
 748         }
 749
 750         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 751         sym = sec->data->d_buf;
 752         for (i = 0; i < n; i++, sym++) {
 753                 int sym_type = ELF64_ST_TYPE(sym->st_info);
 754                 int sym_bind = ELF64_ST_BIND(sym->st_info);
 755                 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
 756
 757                 if (i == 0) {
 758                         if (sym->st_name != 0 || sym->st_info != 0
 759                             || sym->st_other != 0 || sym->st_shndx != 0
 760                             || sym->st_value != 0 || sym->st_size != 0) {
 761                                 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
 762                                 return -EINVAL;
 763                         }
 764                         continue;
 765                 }
 766                 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
 767                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
 768                                 i, sec->sec_idx, sym_bind);
 769                         return -EINVAL;
 770                 }
 771                 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
 772                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
 773                                 i, sec->sec_idx, sym_vis);
 774                         return -EINVAL;
 775                 }
 776                 if (sym->st_shndx == 0) {
 777                         if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
 778                             || sym->st_value != 0 || sym->st_size != 0) {
 779                                 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
 780                                         i, obj->filename);
 781
 782                                 return -EINVAL;
 783                         }
 784                         continue;
 785                 }
 786                 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
 787                         pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 788                                 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 789                         return -EINVAL;
 790                 }
 791                 if (sym_type == STT_SECTION) {
 792                         if (sym->st_value != 0)
 793                                 return -EINVAL;
 794                         continue;
 795                 }
 796         }
 797
 798         return 0;
 799 }
 800
 801 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 802 {
 803         struct src_sec *link_sec, *sym_sec;
 804         Elf64_Rel *relo;
 805         int i, n;
 806
 807         if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 808                 return -EINVAL;
 809         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 810                 return -EINVAL;
 811
 812         /* SHT_REL's sh_link should point to SYMTAB */
 813         if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 814                 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
 815                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 816                 return -EINVAL;
 817         }
 818
 819         /* SHT_REL's sh_info points to relocated section */
 820         if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
 821                 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
 822                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 823                 return -EINVAL;
 824         }
 825         link_sec = &obj->secs[sec->shdr->sh_info];
 826
 827         /* .rel<secname> -> <secname> pattern is followed */
 828         if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
 829             || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
 830                 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
 831                         sec->sec_idx, obj->filename);
 832                 return -EINVAL;
 833         }
 834
 835         /* don't further validate relocations for ignored sections */
 836         if (link_sec->skipped)
 837                 return 0;
 838
 839         /* relocatable section is data or instructions */
 840         if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
 841                 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
 842                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 843                 return -EINVAL;
 844         }
 845
 846         /* check sanity of each relocation */
 847         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 848         relo = sec->data->d_buf;
 849         sym_sec = &obj->secs[obj->symtab_sec_idx];
 850         for (i = 0; i < n; i++, relo++) {
 851                 size_t sym_idx = ELF64_R_SYM(relo->r_info);
 852                 size_t sym_type = ELF64_R_TYPE(relo->r_info);
 853
 854                 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32) {
 855                         pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
 856                                 i, sec->sec_idx, sym_type, obj->filename);
 857                         return -EINVAL;
 858                 }
 859
 860                 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
 861                         pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
 862                                 i, sec->sec_idx, sym_idx, obj->filename);
 863                         return -EINVAL;
 864                 }
 865
 866                 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
 867                         if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
 868                                 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
 869                                         i, sec->sec_idx, sym_idx, obj->filename);
 870                                 return -EINVAL;
 871                         }
 872                 }
 873         }
 874
 875         return 0;
 876 }
 877
 878 static int check_btf_type_id(__u32 *type_id, void *ctx)
 879 {
 880         struct btf *btf = ctx;
 881
 882         if (*type_id > btf__get_nr_types(btf))
 883                 return -EINVAL;
 884
 885         return 0;
 886 }
 887
 888 static int check_btf_str_off(__u32 *str_off, void *ctx)
 889 {
 890         struct btf *btf = ctx;
 891         const char *s;
 892
 893         s = btf__str_by_offset(btf, *str_off);
 894
 895         if (!s)
 896                 return -EINVAL;
 897
 898         return 0;
 899 }
 900
 901 static int linker_sanity_check_btf(struct src_obj *obj)
 902 {
 903         struct btf_type *t;
 904         int i, n, err = 0;
 905
 906         if (!obj->btf)
 907                 return 0;
 908
 909         n = btf__get_nr_types(obj->btf);
 910         for (i = 1; i <= n; i++) {
 911                 t = btf_type_by_id(obj->btf, i);
 912
 913                 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
 914                 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
 915                 if (err)
 916                         return err;
 917         }
 918
 919         return 0;
 920 }
 921
 922 static int linker_sanity_check_btf_ext(struct src_obj *obj)
 923 {
 924         int err = 0;
 925
 926         if (!obj->btf_ext)
 927                 return 0;
 928
 929         /* can't use .BTF.ext without .BTF */
 930         if (!obj->btf)
 931                 return -EINVAL;
 932
 933         err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
 934         err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
 935         if (err)
 936                 return err;
 937
 938         return 0;
 939 }
 940
 941 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
 942 {
 943         Elf_Scn *scn;
 944         Elf_Data *data;
 945         Elf64_Shdr *shdr;
 946         int name_off;
 947
 948         dst_sec->sec_sz = 0;
 949         dst_sec->sec_idx = 0;
 950
 951         /* ephemeral sections are just thin section shells lacking most parts */
 952         if (src_sec->ephemeral)
 953                 return 0;
 954
 955         scn = elf_newscn(linker->elf);
 956         if (!scn)
 957                 return -ENOMEM;
 958         data = elf_newdata(scn);
 959         if (!data)
 960                 return -ENOMEM;
 961         shdr = elf64_getshdr(scn);
 962         if (!shdr)
 963                 return -ENOMEM;
 964
 965         dst_sec->scn = scn;
 966         dst_sec->shdr = shdr;
 967         dst_sec->data = data;
 968         dst_sec->sec_idx = elf_ndxscn(scn);
 969
 970         name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
 971         if (name_off < 0)
 972                 return name_off;
 973
 974         shdr->sh_name = name_off;
 975         shdr->sh_type = src_sec->shdr->sh_type;
 976         shdr->sh_flags = src_sec->shdr->sh_flags;
 977         shdr->sh_size = 0;
 978         /* sh_link and sh_info have different meaning for different types of
 979          * sections, so we leave it up to the caller code to fill them in, if
 980          * necessary
 981          */
 982         shdr->sh_link = 0;
 983         shdr->sh_info = 0;
 984         shdr->sh_addralign = src_sec->shdr->sh_addralign;
 985         shdr->sh_entsize = src_sec->shdr->sh_entsize;
 986
 987         data->d_type = src_sec->data->d_type;
 988         data->d_size = 0;
 989         data->d_buf = NULL;
 990         data->d_align = src_sec->data->d_align;
 991         data->d_off = 0;
 992
 993         return 0;
 994 }
 995
 996 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
 997 {
 998         struct dst_sec *sec;
 999         int i;
1000
1001         for (i = 1; i < linker->sec_cnt; i++) {
1002                 sec = &linker->secs[i];
1003
1004                 if (strcmp(sec->sec_name, sec_name) == 0)
1005                         return sec;
1006         }
1007
1008         return NULL;
1009 }
1010
1011 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1012 {
1013         if (dst->shdr->sh_type != src->shdr->sh_type) {
1014                 pr_warn("sec %s types mismatch\n", dst->sec_name);
1015                 return false;
1016         }
1017         if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1018                 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1019                 return false;
1020         }
1021         if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1022                 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1023                 return false;
1024         }
1025
1026         return true;
1027 }
1028
1029 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1030 {
1031         if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1032                 return false;
1033         if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1034                 return false;
1035         return true;
1036 }
1037
1038 static int extend_sec(struct dst_sec *dst, struct src_sec *src)
1039 {
1040         void *tmp;
1041         size_t dst_align = dst->shdr->sh_addralign;
1042         size_t src_align = src->shdr->sh_addralign;
1043         size_t dst_align_sz, dst_final_sz;
1044
1045         if (dst_align == 0)
1046                 dst_align = 1;
1047         if (dst_align < src_align)
1048                 dst_align = src_align;
1049
1050         dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1051
1052         /* no need to re-align final size */
1053         dst_final_sz = dst_align_sz + src->shdr->sh_size;
1054
1055         if (src->shdr->sh_type != SHT_NOBITS) {
1056                 tmp = realloc(dst->raw_data, dst_final_sz);
1057                 if (!tmp)
1058                         return -ENOMEM;
1059                 dst->raw_data = tmp;
1060
1061                 /* pad dst section, if it's alignment forced size increase */
1062                 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1063                 /* now copy src data at a properly aligned offset */
1064                 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1065         }
1066
1067         dst->sec_sz = dst_final_sz;
1068         dst->shdr->sh_size = dst_final_sz;
1069         dst->data->d_size = dst_final_sz;
1070
1071         dst->shdr->sh_addralign = dst_align;
1072         dst->data->d_align = dst_align;
1073
1074         src->dst_off = dst_align_sz;
1075
1076         return 0;
1077 }
1078
1079 static bool is_data_sec(struct src_sec *sec)
1080 {
1081         if (!sec || sec->skipped)
1082                 return false;
1083         /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1084         if (sec->ephemeral)
1085                 return true;
1086         return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1087 }
1088
1089 static bool is_relo_sec(struct src_sec *sec)
1090 {
1091         if (!sec || sec->skipped || sec->ephemeral)
1092                 return false;
1093         return sec->shdr->sh_type == SHT_REL;
1094 }
1095
1096 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1097 {
1098         int i, err;
1099
1100         for (i = 1; i < obj->sec_cnt; i++) {
1101                 struct src_sec *src_sec;
1102                 struct dst_sec *dst_sec;
1103
1104                 src_sec = &obj->secs[i];
1105                 if (!is_data_sec(src_sec))
1106                         continue;
1107
1108                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1109                 if (!dst_sec) {
1110                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1111                         if (!dst_sec)
1112                                 return -ENOMEM;
1113                         err = init_sec(linker, dst_sec, src_sec);
1114                         if (err) {
1115                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1116                                 return err;
1117                         }
1118                 } else {
1119                         if (!secs_match(dst_sec, src_sec)) {
1120                                 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1121                                 return -1;
1122                         }
1123
1124                         /* "license" and "version" sections are deduped */
1125                         if (strcmp(src_sec->sec_name, "license") == 0
1126                             || strcmp(src_sec->sec_name, "version") == 0) {
1127                                 if (!sec_content_is_same(dst_sec, src_sec)) {
1128                                         pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1129                                         return -EINVAL;
1130                                 }
1131                                 src_sec->skipped = true;
1132                                 src_sec->dst_id = dst_sec->id;
1133                                 continue;
1134                         }
1135                 }
1136
1137                 /* record mapped section index */
1138                 src_sec->dst_id = dst_sec->id;
1139
1140                 if (src_sec->ephemeral)
1141                         continue;
1142
1143                 err = extend_sec(dst_sec, src_sec);
1144                 if (err)
1145                         return err;
1146         }
1147
1148         return 0;
1149 }
1150
1151 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1152 {
1153         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1154         Elf64_Sym *sym = symtab->data->d_buf, *dst_sym;
1155         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
1156         int str_sec_idx = symtab->shdr->sh_link;
1157
1158         obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1159         if (!obj->sym_map)
1160                 return -ENOMEM;
1161
1162         for (i = 0; i < n; i++, sym++) {
1163                 struct src_sec *src_sec = NULL;
1164                 struct dst_sec *dst_sec = NULL;
1165                 const char *sym_name;
1166                 size_t dst_sym_idx;
1167                 int name_off;
1168
1169                 /* We already validated all-zero symbol #0 and we already
1170                  * appended it preventively to the final SYMTAB, so skip it.
1171                  */
1172                 if (i == 0)
1173                         continue;
1174
1175                 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1176                 if (!sym_name) {
1177                         pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1178                         return -EINVAL;
1179                 }
1180
1181                 if (sym->st_shndx && sym->st_shndx < SHN_LORESERVE) {
1182                         src_sec = &obj->secs[sym->st_shndx];
1183                         if (src_sec->skipped)
1184                                 continue;
1185                         dst_sec = &linker->secs[src_sec->dst_id];
1186
1187                         /* allow only one STT_SECTION symbol per section */
1188                         if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sec->sec_sym_idx) {
1189                                 obj->sym_map[i] = dst_sec->sec_sym_idx;
1190                                 continue;
1191                         }
1192                 }
1193
1194                 name_off = strset__add_str(linker->strtab_strs, sym_name);
1195                 if (name_off < 0)
1196                         return name_off;
1197
1198                 dst_sym = add_new_sym(linker, &dst_sym_idx);
1199                 if (!dst_sym)
1200                         return -ENOMEM;
1201
1202                 dst_sym->st_name = name_off;
1203                 dst_sym->st_info = sym->st_info;
1204                 dst_sym->st_other = sym->st_other;
1205                 dst_sym->st_shndx = src_sec ? dst_sec->sec_idx : sym->st_shndx;
1206                 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1207                 dst_sym->st_size = sym->st_size;
1208
1209                 obj->sym_map[i] = dst_sym_idx;
1210
1211                 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sym) {
1212                         dst_sec->sec_sym_idx = dst_sym_idx;
1213                         dst_sym->st_value = 0;
1214                 }
1215
1216         }
1217
1218         return 0;
1219 }
1220
1221 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
1222 {
1223         struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
1224         struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx];
1225         int i, err;
1226
1227         for (i = 1; i < obj->sec_cnt; i++) {
1228                 struct src_sec *src_sec, *src_linked_sec;
1229                 struct dst_sec *dst_sec, *dst_linked_sec;
1230                 Elf64_Rel *src_rel, *dst_rel;
1231                 int j, n;
1232
1233                 src_sec = &obj->secs[i];
1234                 if (!is_relo_sec(src_sec))
1235                         continue;
1236
1237                 /* shdr->sh_info points to relocatable section */
1238                 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
1239                 if (src_linked_sec->skipped)
1240                         continue;
1241
1242                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1243                 if (!dst_sec) {
1244                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1245                         if (!dst_sec)
1246                                 return -ENOMEM;
1247                         err = init_sec(linker, dst_sec, src_sec);
1248                         if (err) {
1249                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1250                                 return err;
1251                         }
1252                 } else if (!secs_match(dst_sec, src_sec)) {
1253                         pr_warn("sections %s are not compatible\n", src_sec->sec_name);
1254                         return -1;
1255                 }
1256
1257                 /* shdr->sh_link points to SYMTAB */
1258                 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
1259
1260                 /* shdr->sh_info points to relocated section */
1261                 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
1262                 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
1263
1264                 src_sec->dst_id = dst_sec->id;
1265                 err = extend_sec(dst_sec, src_sec);
1266                 if (err)
1267                         return err;
1268
1269                 src_rel = src_sec->data->d_buf;
1270                 dst_rel = dst_sec->raw_data + src_sec->dst_off;
1271                 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
1272                 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
1273                         size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info);
1274                         size_t sym_type = ELF64_R_TYPE(src_rel->r_info);
1275                         Elf64_Sym *src_sym, *dst_sym;
1276                         size_t dst_sym_idx;
1277
1278                         src_sym_idx = ELF64_R_SYM(src_rel->r_info);
1279                         src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
1280
1281                         dst_sym_idx = obj->sym_map[src_sym_idx];
1282                         dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx;
1283                         dst_rel->r_offset += src_linked_sec->dst_off;
1284                         sym_type = ELF64_R_TYPE(src_rel->r_info);
1285                         dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
1286
1287                         if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
1288                                 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
1289                                 struct bpf_insn *insn;
1290
1291                                 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
1292                                         /* calls to the very first static function inside
1293                                          * .text section at offset 0 will
1294                                          * reference section symbol, not the
1295                                          * function symbol. Fix that up,
1296                                          * otherwise it won't be possible to
1297                                          * relocate calls to two different
1298                                          * static functions with the same name
1299                                          * (rom two different object files)
1300                                          */
1301                                         insn = dst_linked_sec->raw_data + dst_rel->r_offset;
1302                                         if (insn->code == (BPF_JMP | BPF_CALL))
1303                                                 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
1304                                         else
1305                                                 insn->imm += sec->dst_off;
1306                                 } else {
1307                                         pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
1308                                         return -EINVAL;
1309                                 }
1310                         }
1311
1312                 }
1313         }
1314
1315         return 0;
1316 }
1317
1318 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1319 {
1320         struct src_sec *sec;
1321         int i;
1322
1323         for (i = 1; i < obj->sec_cnt; i++) {
1324                 sec = &obj->secs[i];
1325
1326                 if (strcmp(sec->sec_name, sec_name) == 0)
1327                         return sec;
1328         }
1329
1330         return NULL;
1331 }
1332
1333 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
1334                                    int sym_type, const char *sym_name)
1335 {
1336         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1337         Elf64_Sym *sym = symtab->data->d_buf;
1338         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
1339         int str_sec_idx = symtab->shdr->sh_link;
1340         const char *name;
1341
1342         for (i = 0; i < n; i++, sym++) {
1343                 if (sym->st_shndx != sec_idx)
1344                         continue;
1345                 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
1346                         continue;
1347
1348                 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1349                 if (!name)
1350                         return NULL;
1351
1352                 if (strcmp(sym_name, name) != 0)
1353                         continue;
1354
1355                 return sym;
1356         }
1357
1358         return NULL;
1359 }
1360
1361 static int linker_fixup_btf(struct src_obj *obj)
1362 {
1363         const char *sec_name;
1364         struct src_sec *sec;
1365         int i, j, n, m;
1366
1367         if (!obj->btf)
1368                 return 0;
1369
1370         n = btf__get_nr_types(obj->btf);
1371         for (i = 1; i <= n; i++) {
1372                 struct btf_var_secinfo *vi;
1373                 struct btf_type *t;
1374
1375                 t = btf_type_by_id(obj->btf, i);
1376                 if (btf_kind(t) != BTF_KIND_DATASEC)
1377                         continue;
1378
1379                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1380                 sec = find_src_sec_by_name(obj, sec_name);
1381                 if (sec) {
1382                         /* record actual section size, unless ephemeral */
1383                         if (sec->shdr)
1384                                 t->size = sec->shdr->sh_size;
1385                 } else {
1386                         /* BTF can have some sections that are not represented
1387                          * in ELF, e.g., .kconfig and .ksyms, which are used
1388                          * for special extern variables.  Here we'll
1389                          * pre-create "section shells" for them to be able to
1390                          * keep track of extra per-section metadata later
1391                          * (e.g., BTF variables).
1392                          */
1393                         sec = add_src_sec(obj, sec_name);
1394                         if (!sec)
1395                                 return -ENOMEM;
1396
1397                         sec->ephemeral = true;
1398                         sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
1399                 }
1400
1401                 /* remember ELF section and its BTF type ID match */
1402                 sec->sec_type_id = i;
1403
1404                 /* fix up variable offsets */
1405                 vi = btf_var_secinfos(t);
1406                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1407                         const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
1408                         const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
1409                         int var_linkage = btf_var(vt)->linkage;
1410                         Elf64_Sym *sym;
1411
1412                         /* no need to patch up static or extern vars */
1413                         if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
1414                                 continue;
1415
1416                         sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
1417                         if (!sym) {
1418                                 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
1419                                 return -ENOENT;
1420                         }
1421
1422                         vi->offset = sym->st_value;
1423                 }
1424         }
1425
1426         return 0;
1427 }
1428
1429 static int remap_type_id(__u32 *type_id, void *ctx)
1430 {
1431         int *id_map = ctx;
1432         int new_id = id_map[*type_id];
1433
1434         /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
1435         if (new_id == 0 && *type_id != 0) {
1436                 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
1437                 return -EINVAL;
1438         }
1439
1440         *type_id = id_map[*type_id];
1441
1442         return 0;
1443 }
1444
1445 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
1446 {
1447         const struct btf_type *t;
1448         int i, j, n, start_id, id;
1449
1450         if (!obj->btf)
1451                 return 0;
1452
1453         start_id = btf__get_nr_types(linker->btf) + 1;
1454         n = btf__get_nr_types(obj->btf);
1455
1456         obj->btf_type_map = calloc(n + 1, sizeof(int));
1457         if (!obj->btf_type_map)
1458                 return -ENOMEM;
1459
1460         for (i = 1; i <= n; i++) {
1461                 t = btf__type_by_id(obj->btf, i);
1462
1463                 /* DATASECs are handled specially below */
1464                 if (btf_kind(t) == BTF_KIND_DATASEC)
1465                         continue;
1466
1467                 id = btf__add_type(linker->btf, obj->btf, t);
1468                 if (id < 0) {
1469                         pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
1470                         return id;
1471                 }
1472
1473                 obj->btf_type_map[i] = id;
1474         }
1475
1476         /* remap all the types except DATASECs */
1477         n = btf__get_nr_types(linker->btf);
1478         for (i = start_id; i <= n; i++) {
1479                 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
1480
1481                 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
1482                         return -EINVAL;
1483         }
1484
1485         /* append DATASEC info */
1486         for (i = 1; i < obj->sec_cnt; i++) {
1487                 struct src_sec *src_sec;
1488                 struct dst_sec *dst_sec;
1489                 const struct btf_var_secinfo *src_var;
1490                 struct btf_var_secinfo *dst_var;
1491
1492                 src_sec = &obj->secs[i];
1493                 if (!src_sec->sec_type_id || src_sec->skipped)
1494                         continue;
1495                 dst_sec = &linker->secs[src_sec->dst_id];
1496
1497                 /* Mark section as having BTF regardless of the presence of
1498                  * variables. In some cases compiler might generate empty BTF
1499                  * with no variables information. E.g., when promoting local
1500                  * array/structure variable initial values and BPF object
1501                  * file otherwise has no read-only static variables in
1502                  * .rodata. We need to preserve such empty BTF and just set
1503                  * correct section size.
1504                  */
1505                 dst_sec->has_btf = true;
1506
1507                 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
1508                 src_var = btf_var_secinfos(t);
1509                 n = btf_vlen(t);
1510                 for (j = 0; j < n; j++, src_var++) {
1511                         void *sec_vars = dst_sec->sec_vars;
1512
1513                         sec_vars = libbpf_reallocarray(sec_vars,
1514                                                        dst_sec->sec_var_cnt + 1,
1515                                                        sizeof(*dst_sec->sec_vars));
1516                         if (!sec_vars)
1517                                 return -ENOMEM;
1518
1519                         dst_sec->sec_vars = sec_vars;
1520                         dst_sec->sec_var_cnt++;
1521
1522                         dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
1523                         dst_var->type = obj->btf_type_map[src_var->type];
1524                         dst_var->size = src_var->size;
1525                         dst_var->offset = src_sec->dst_off + src_var->offset;
1526                 }
1527         }
1528
1529         return 0;
1530 }
1531
1532 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
1533 {
1534         void *tmp;
1535
1536         tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
1537         if (!tmp)
1538                 return NULL;
1539         ext_data->recs = tmp;
1540
1541         tmp += ext_data->rec_cnt * ext_data->rec_sz;
1542         memcpy(tmp, src_rec, ext_data->rec_sz);
1543
1544         ext_data->rec_cnt++;
1545
1546         return tmp;
1547 }
1548
1549 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
1550 {
1551         const struct btf_ext_info_sec *ext_sec;
1552         const char *sec_name, *s;
1553         struct src_sec *src_sec;
1554         struct dst_sec *dst_sec;
1555         int rec_sz, str_off, i;
1556
1557         if (!obj->btf_ext)
1558                 return 0;
1559
1560         rec_sz = obj->btf_ext->func_info.rec_size;
1561         for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
1562                 struct bpf_func_info_min *src_rec, *dst_rec;
1563
1564                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1565                 src_sec = find_src_sec_by_name(obj, sec_name);
1566                 if (!src_sec) {
1567                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1568                         return -EINVAL;
1569                 }
1570                 dst_sec = &linker->secs[src_sec->dst_id];
1571
1572                 if (dst_sec->func_info.rec_sz == 0)
1573                         dst_sec->func_info.rec_sz = rec_sz;
1574                 if (dst_sec->func_info.rec_sz != rec_sz) {
1575                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1576                         return -EINVAL;
1577                 }
1578
1579                 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
1580                         dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
1581                         if (!dst_rec)
1582                                 return -ENOMEM;
1583
1584                         dst_rec->insn_off += src_sec->dst_off;
1585                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
1586                 }
1587         }
1588
1589         rec_sz = obj->btf_ext->line_info.rec_size;
1590         for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
1591                 struct bpf_line_info_min *src_rec, *dst_rec;
1592
1593                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1594                 src_sec = find_src_sec_by_name(obj, sec_name);
1595                 if (!src_sec) {
1596                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1597                         return -EINVAL;
1598                 }
1599                 dst_sec = &linker->secs[src_sec->dst_id];
1600
1601                 if (dst_sec->line_info.rec_sz == 0)
1602                         dst_sec->line_info.rec_sz = rec_sz;
1603                 if (dst_sec->line_info.rec_sz != rec_sz) {
1604                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1605                         return -EINVAL;
1606                 }
1607
1608                 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
1609                         dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
1610                         if (!dst_rec)
1611                                 return -ENOMEM;
1612
1613                         dst_rec->insn_off += src_sec->dst_off;
1614
1615                         s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
1616                         str_off = btf__add_str(linker->btf, s);
1617                         if (str_off < 0)
1618                                 return -ENOMEM;
1619                         dst_rec->file_name_off = str_off;
1620
1621                         s = btf__str_by_offset(obj->btf, src_rec->line_off);
1622                         str_off = btf__add_str(linker->btf, s);
1623                         if (str_off < 0)
1624                                 return -ENOMEM;
1625                         dst_rec->line_off = str_off;
1626
1627                         /* dst_rec->line_col is fine */
1628                 }
1629         }
1630
1631         rec_sz = obj->btf_ext->core_relo_info.rec_size;
1632         for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
1633                 struct bpf_core_relo *src_rec, *dst_rec;
1634
1635                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1636                 src_sec = find_src_sec_by_name(obj, sec_name);
1637                 if (!src_sec) {
1638                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1639                         return -EINVAL;
1640                 }
1641                 dst_sec = &linker->secs[src_sec->dst_id];
1642
1643                 if (dst_sec->core_relo_info.rec_sz == 0)
1644                         dst_sec->core_relo_info.rec_sz = rec_sz;
1645                 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
1646                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1647                         return -EINVAL;
1648                 }
1649
1650                 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
1651                         dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
1652                         if (!dst_rec)
1653                                 return -ENOMEM;
1654
1655                         dst_rec->insn_off += src_sec->dst_off;
1656                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
1657
1658                         s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
1659                         str_off = btf__add_str(linker->btf, s);
1660                         if (str_off < 0)
1661                                 return -ENOMEM;
1662                         dst_rec->access_str_off = str_off;
1663
1664                         /* dst_rec->kind is fine */
1665                 }
1666         }
1667
1668         return 0;
1669 }
1670
1671 int bpf_linker__finalize(struct bpf_linker *linker)
1672 {
1673         struct dst_sec *sec;
1674         size_t strs_sz;
1675         const void *strs;
1676         int err, i;
1677
1678         if (!linker->elf)
1679                 return -EINVAL;
1680
1681         err = finalize_btf(linker);
1682         if (err)
1683                 return err;
1684
1685         /* Finalize strings */
1686         strs_sz = strset__data_size(linker->strtab_strs);
1687         strs = strset__data(linker->strtab_strs);
1688
1689         sec = &linker->secs[linker->strtab_sec_idx];
1690         sec->data->d_align = 1;
1691         sec->data->d_off = 0LL;
1692         sec->data->d_buf = (void *)strs;
1693         sec->data->d_type = ELF_T_BYTE;
1694         sec->data->d_size = strs_sz;
1695         sec->shdr->sh_size = strs_sz;
1696
1697         for (i = 1; i < linker->sec_cnt; i++) {
1698                 sec = &linker->secs[i];
1699
1700                 /* STRTAB is handled specially above */
1701                 if (sec->sec_idx == linker->strtab_sec_idx)
1702                         continue;
1703
1704                 /* special ephemeral sections (.ksyms, .kconfig, etc) */
1705                 if (!sec->scn)
1706                         continue;
1707
1708                 sec->data->d_buf = sec->raw_data;
1709         }
1710
1711         /* Finalize ELF layout */
1712         if (elf_update(linker->elf, ELF_C_NULL) < 0) {
1713                 err = -errno;
1714                 pr_warn_elf("failed to finalize ELF layout");
1715                 return err;
1716         }
1717
1718         /* Write out final ELF contents */
1719         if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
1720                 err = -errno;
1721                 pr_warn_elf("failed to write ELF contents");
1722                 return err;
1723         }
1724
1725         elf_end(linker->elf);
1726         close(linker->fd);
1727
1728         linker->elf = NULL;
1729         linker->fd = -1;
1730
1731         return 0;
1732 }
1733
1734 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
1735                              size_t align, const void *raw_data, size_t raw_sz)
1736 {
1737         Elf_Scn *scn;
1738         Elf_Data *data;
1739         Elf64_Shdr *shdr;
1740         int name_off;
1741
1742         name_off = strset__add_str(linker->strtab_strs, sec_name);
1743         if (name_off < 0)
1744                 return name_off;
1745
1746         scn = elf_newscn(linker->elf);
1747         if (!scn)
1748                 return -ENOMEM;
1749         data = elf_newdata(scn);
1750         if (!data)
1751                 return -ENOMEM;
1752         shdr = elf64_getshdr(scn);
1753         if (!shdr)
1754                 return -EINVAL;
1755
1756         shdr->sh_name = name_off;
1757         shdr->sh_type = SHT_PROGBITS;
1758         shdr->sh_flags = 0;
1759         shdr->sh_size = raw_sz;
1760         shdr->sh_link = 0;
1761         shdr->sh_info = 0;
1762         shdr->sh_addralign = align;
1763         shdr->sh_entsize = 0;
1764
1765         data->d_type = ELF_T_BYTE;
1766         data->d_size = raw_sz;
1767         data->d_buf = (void *)raw_data;
1768         data->d_align = align;
1769         data->d_off = 0;
1770
1771         return 0;
1772 }
1773
1774 static int finalize_btf(struct bpf_linker *linker)
1775 {
1776         struct btf *btf = linker->btf;
1777         const void *raw_data;
1778         int i, j, id, err;
1779         __u32 raw_sz;
1780
1781         /* bail out if no BTF data was produced */
1782         if (btf__get_nr_types(linker->btf) == 0)
1783                 return 0;
1784
1785         for (i = 1; i < linker->sec_cnt; i++) {
1786                 struct dst_sec *sec = &linker->secs[i];
1787
1788                 if (!sec->has_btf)
1789                         continue;
1790
1791                 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
1792                 if (id < 0) {
1793                         pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
1794                                 sec->sec_name, id);
1795                         return id;
1796                 }
1797
1798                 for (j = 0; j < sec->sec_var_cnt; j++) {
1799                         struct btf_var_secinfo *vi = &sec->sec_vars[j];
1800
1801                         if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
1802                                 return -EINVAL;
1803                 }
1804         }
1805
1806         err = finalize_btf_ext(linker);
1807         if (err) {
1808                 pr_warn(".BTF.ext generation failed: %d\n", err);
1809                 return err;
1810         }
1811
1812         err = btf__dedup(linker->btf, linker->btf_ext, NULL);
1813         if (err) {
1814                 pr_warn("BTF dedup failed: %d\n", err);
1815                 return err;
1816         }
1817
1818         /* Emit .BTF section */
1819         raw_data = btf__get_raw_data(linker->btf, &raw_sz);
1820         if (!raw_data)
1821                 return -ENOMEM;
1822
1823         err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
1824         if (err) {
1825                 pr_warn("failed to write out .BTF ELF section: %d\n", err);
1826                 return err;
1827         }
1828
1829         /* Emit .BTF.ext section */
1830         if (linker->btf_ext) {
1831                 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
1832                 if (!raw_data)
1833                         return -ENOMEM;
1834
1835                 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
1836                 if (err) {
1837                         pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
1838                         return err;
1839                 }
1840         }
1841
1842         return 0;
1843 }
1844
1845 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
1846                              const char *sec_name, struct btf_ext_sec_data *sec_data)
1847 {
1848         struct btf_ext_info_sec *sec_info;
1849         void *cur = output;
1850         int str_off;
1851         size_t sz;
1852
1853         if (!sec_data->rec_cnt)
1854                 return 0;
1855
1856         str_off = btf__add_str(linker->btf, sec_name);
1857         if (str_off < 0)
1858                 return -ENOMEM;
1859
1860         sec_info = cur;
1861         sec_info->sec_name_off = str_off;
1862         sec_info->num_info = sec_data->rec_cnt;
1863         cur += sizeof(struct btf_ext_info_sec);
1864
1865         sz = sec_data->rec_cnt * sec_data->rec_sz;
1866         memcpy(cur, sec_data->recs, sz);
1867         cur += sz;
1868
1869         return cur - output;
1870 }
1871
1872 static int finalize_btf_ext(struct bpf_linker *linker)
1873 {
1874         size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
1875         size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
1876         struct btf_ext_header *hdr;
1877         void *data, *cur;
1878         int i, err, sz;
1879
1880         /* validate that all sections have the same .BTF.ext record sizes
1881          * and calculate total data size for each type of data (func info,
1882          * line info, core relos)
1883          */
1884         for (i = 1; i < linker->sec_cnt; i++) {
1885                 struct dst_sec *sec = &linker->secs[i];
1886
1887                 if (sec->func_info.rec_cnt) {
1888                         if (func_rec_sz == 0)
1889                                 func_rec_sz = sec->func_info.rec_sz;
1890                         if (func_rec_sz != sec->func_info.rec_sz) {
1891                                 pr_warn("mismatch in func_info record size %zu != %u\n",
1892                                         func_rec_sz, sec->func_info.rec_sz);
1893                                 return -EINVAL;
1894                         }
1895
1896                         funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
1897                 }
1898                 if (sec->line_info.rec_cnt) {
1899                         if (line_rec_sz == 0)
1900                                 line_rec_sz = sec->line_info.rec_sz;
1901                         if (line_rec_sz != sec->line_info.rec_sz) {
1902                                 pr_warn("mismatch in line_info record size %zu != %u\n",
1903                                         line_rec_sz, sec->line_info.rec_sz);
1904                                 return -EINVAL;
1905                         }
1906
1907                         lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
1908                 }
1909                 if (sec->core_relo_info.rec_cnt) {
1910                         if (core_relo_rec_sz == 0)
1911                                 core_relo_rec_sz = sec->core_relo_info.rec_sz;
1912                         if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
1913                                 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
1914                                         core_relo_rec_sz, sec->core_relo_info.rec_sz);
1915                                 return -EINVAL;
1916                         }
1917
1918                         core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
1919                 }
1920         }
1921
1922         if (!funcs_sz && !lines_sz && !core_relos_sz)
1923                 return 0;
1924
1925         total_sz += sizeof(struct btf_ext_header);
1926         if (funcs_sz) {
1927                 funcs_sz += sizeof(__u32); /* record size prefix */
1928                 total_sz += funcs_sz;
1929         }
1930         if (lines_sz) {
1931                 lines_sz += sizeof(__u32); /* record size prefix */
1932                 total_sz += lines_sz;
1933         }
1934         if (core_relos_sz) {
1935                 core_relos_sz += sizeof(__u32); /* record size prefix */
1936                 total_sz += core_relos_sz;
1937         }
1938
1939         cur = data = calloc(1, total_sz);
1940         if (!data)
1941                 return -ENOMEM;
1942
1943         hdr = cur;
1944         hdr->magic = BTF_MAGIC;
1945         hdr->version = BTF_VERSION;
1946         hdr->flags = 0;
1947         hdr->hdr_len = sizeof(struct btf_ext_header);
1948         cur += sizeof(struct btf_ext_header);
1949
1950         /* All offsets are in bytes relative to the end of this header */
1951         hdr->func_info_off = 0;
1952         hdr->func_info_len = funcs_sz;
1953         hdr->line_info_off = funcs_sz;
1954         hdr->line_info_len = lines_sz;
1955         hdr->core_relo_off = funcs_sz + lines_sz;
1956         hdr->core_relo_len = core_relos_sz;
1957
1958         if (funcs_sz) {
1959                 *(__u32 *)cur = func_rec_sz;
1960                 cur += sizeof(__u32);
1961
1962                 for (i = 1; i < linker->sec_cnt; i++) {
1963                         struct dst_sec *sec = &linker->secs[i];
1964
1965                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
1966                         if (sz < 0) {
1967                                 err = sz;
1968                                 goto out;
1969                         }
1970
1971                         cur += sz;
1972                 }
1973         }
1974
1975         if (lines_sz) {
1976                 *(__u32 *)cur = line_rec_sz;
1977                 cur += sizeof(__u32);
1978
1979                 for (i = 1; i < linker->sec_cnt; i++) {
1980                         struct dst_sec *sec = &linker->secs[i];
1981
1982                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
1983                         if (sz < 0) {
1984                                 err = sz;
1985                                 goto out;
1986                         }
1987
1988                         cur += sz;
1989                 }
1990         }
1991
1992         if (core_relos_sz) {
1993                 *(__u32 *)cur = core_relo_rec_sz;
1994                 cur += sizeof(__u32);
1995
1996                 for (i = 1; i < linker->sec_cnt; i++) {
1997                         struct dst_sec *sec = &linker->secs[i];
1998
1999                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2000                         if (sz < 0) {
2001                                 err = sz;
2002                                 goto out;
2003                         }
2004
2005                         cur += sz;
2006                 }
2007         }
2008
2009         linker->btf_ext = btf_ext__new(data, total_sz);
2010         err = libbpf_get_error(linker->btf_ext);
2011         if (err) {
2012                 linker->btf_ext = NULL;
2013                 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2014                 goto out;
2015         }
2016
2017 out:
2018         free(data);
2019         return err;
2020 }