scripts/sorttable.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * sorttable.h
   4  *
   5  * Added ORC unwind tables sort support and other updates:
   6  * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
   7  * Shile Zhang <shile.zhang@linux.alibaba.com>
   8  *
   9  * Copyright 2011 - 2012 Cavium, Inc.
  10  *
  11  * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by:
  12  * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
  13  *
  14  * Some of this code was taken out of recordmcount.h written by:
  15  *
  16  * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved.
  17  * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
  18  */
  19
  20 #undef extable_ent_size
  21 #undef compare_extable
  22 #undef get_mcount_loc
  23 #undef sort_mcount_loc
  24 #undef elf_mcount_loc
  25 #undef do_sort
  26 #undef Elf_Addr
  27 #undef Elf_Ehdr
  28 #undef Elf_Shdr
  29 #undef Elf_Rel
  30 #undef Elf_Rela
  31 #undef Elf_Sym
  32 #undef ELF_R_SYM
  33 #undef Elf_r_sym
  34 #undef ELF_R_INFO
  35 #undef Elf_r_info
  36 #undef ELF_ST_BIND
  37 #undef ELF_ST_TYPE
  38 #undef fn_ELF_R_SYM
  39 #undef fn_ELF_R_INFO
  40 #undef uint_t
  41 #undef _r
  42 #undef _w
  43
  44 #ifdef SORTTABLE_64
  45 # define extable_ent_size       16
  46 # define compare_extable        compare_extable_64
  47 # define get_mcount_loc         get_mcount_loc_64
  48 # define sort_mcount_loc        sort_mcount_loc_64
  49 # define elf_mcount_loc         elf_mcount_loc_64
  50 # define do_sort                do_sort_64
  51 # define Elf_Addr               Elf64_Addr
  52 # define Elf_Ehdr               Elf64_Ehdr
  53 # define Elf_Shdr               Elf64_Shdr
  54 # define Elf_Rel                Elf64_Rel
  55 # define Elf_Rela               Elf64_Rela
  56 # define Elf_Sym                Elf64_Sym
  57 # define ELF_R_SYM              ELF64_R_SYM
  58 # define Elf_r_sym              Elf64_r_sym
  59 # define ELF_R_INFO             ELF64_R_INFO
  60 # define Elf_r_info             Elf64_r_info
  61 # define ELF_ST_BIND            ELF64_ST_BIND
  62 # define ELF_ST_TYPE            ELF64_ST_TYPE
  63 # define fn_ELF_R_SYM           fn_ELF64_R_SYM
  64 # define fn_ELF_R_INFO          fn_ELF64_R_INFO
  65 # define uint_t                 uint64_t
  66 # define _r                     r8
  67 # define _w                     w8
  68 #else
  69 # define extable_ent_size       8
  70 # define compare_extable        compare_extable_32
  71 # define get_mcount_loc         get_mcount_loc_32
  72 # define sort_mcount_loc        sort_mcount_loc_32
  73 # define elf_mcount_loc         elf_mcount_loc_32
  74 # define do_sort                do_sort_32
  75 # define Elf_Addr               Elf32_Addr
  76 # define Elf_Ehdr               Elf32_Ehdr
  77 # define Elf_Shdr               Elf32_Shdr
  78 # define Elf_Rel                Elf32_Rel
  79 # define Elf_Rela               Elf32_Rela
  80 # define Elf_Sym                Elf32_Sym
  81 # define ELF_R_SYM              ELF32_R_SYM
  82 # define Elf_r_sym              Elf32_r_sym
  83 # define ELF_R_INFO             ELF32_R_INFO
  84 # define Elf_r_info             Elf32_r_info
  85 # define ELF_ST_BIND            ELF32_ST_BIND
  86 # define ELF_ST_TYPE            ELF32_ST_TYPE
  87 # define fn_ELF_R_SYM           fn_ELF32_R_SYM
  88 # define fn_ELF_R_INFO          fn_ELF32_R_INFO
  89 # define uint_t                 uint32_t
  90 # define _r                     r
  91 # define _w                     w
  92 #endif
  93
  94 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
  95 /* ORC unwinder only support X86_64 */
  96 #include <asm/orc_types.h>
  97
  98 #define ERRSTR_MAXSZ    256
  99
 100 char g_err[ERRSTR_MAXSZ];
 101 int *g_orc_ip_table;
 102 struct orc_entry *g_orc_table;
 103
 104 pthread_t orc_sort_thread;
 105
 106 static inline unsigned long orc_ip(const int *ip)
 107 {
 108         return (unsigned long)ip + *ip;
 109 }
 110
 111 static int orc_sort_cmp(const void *_a, const void *_b)
 112 {
 113         struct orc_entry *orc_a;
 114         const int *a = g_orc_ip_table + *(int *)_a;
 115         const int *b = g_orc_ip_table + *(int *)_b;
 116         unsigned long a_val = orc_ip(a);
 117         unsigned long b_val = orc_ip(b);
 118
 119         if (a_val > b_val)
 120                 return 1;
 121         if (a_val < b_val)
 122                 return -1;
 123
 124         /*
 125          * The "weak" section terminator entries need to always be on the left
 126          * to ensure the lookup code skips them in favor of real entries.
 127          * These terminator entries exist to handle any gaps created by
 128          * whitelisted .o files which didn't get objtool generation.
 129          */
 130         orc_a = g_orc_table + (a - g_orc_ip_table);
 131         return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
 132 }
 133
 134 static void *sort_orctable(void *arg)
 135 {
 136         int i;
 137         int *idxs = NULL;
 138         int *tmp_orc_ip_table = NULL;
 139         struct orc_entry *tmp_orc_table = NULL;
 140         unsigned int *orc_ip_size = (unsigned int *)arg;
 141         unsigned int num_entries = *orc_ip_size / sizeof(int);
 142         unsigned int orc_size = num_entries * sizeof(struct orc_entry);
 143
 144         idxs = (int *)malloc(*orc_ip_size);
 145         if (!idxs) {
 146                 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
 147                          strerror(errno));
 148                 pthread_exit(g_err);
 149         }
 150
 151         tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
 152         if (!tmp_orc_ip_table) {
 153                 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
 154                          strerror(errno));
 155                 pthread_exit(g_err);
 156         }
 157
 158         tmp_orc_table = (struct orc_entry *)malloc(orc_size);
 159         if (!tmp_orc_table) {
 160                 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
 161                          strerror(errno));
 162                 pthread_exit(g_err);
 163         }
 164
 165         /* initialize indices array, convert ip_table to absolute address */
 166         for (i = 0; i < num_entries; i++) {
 167                 idxs[i] = i;
 168                 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
 169         }
 170         memcpy(tmp_orc_table, g_orc_table, orc_size);
 171
 172         qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
 173
 174         for (i = 0; i < num_entries; i++) {
 175                 if (idxs[i] == i)
 176                         continue;
 177
 178                 /* convert back to relative address */
 179                 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
 180                 g_orc_table[i] = tmp_orc_table[idxs[i]];
 181         }
 182
 183         free(idxs);
 184         free(tmp_orc_ip_table);
 185         free(tmp_orc_table);
 186         pthread_exit(NULL);
 187 }
 188 #endif
 189
 190 static int compare_extable(const void *a, const void *b)
 191 {
 192         Elf_Addr av = _r(a);
 193         Elf_Addr bv = _r(b);
 194
 195         if (av < bv)
 196                 return -1;
 197         if (av > bv)
 198                 return 1;
 199         return 0;
 200 }
 201 #ifdef MCOUNT_SORT_ENABLED
 202 struct elf_mcount_loc {
 203         Elf_Ehdr *ehdr;
 204         Elf_Shdr *init_data_sec;
 205         uint_t start_mcount_loc;
 206         uint_t stop_mcount_loc;
 207 };
 208
 209 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */
 210 static void *sort_mcount_loc(void *arg)
 211 {
 212         struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg;
 213         uint_t offset = emloc->start_mcount_loc - _r(&(emloc->init_data_sec)->sh_addr)
 214                                         + _r(&(emloc->init_data_sec)->sh_offset);
 215         uint_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc;
 216         unsigned char *start_loc = (void *)emloc->ehdr + offset;
 217
 218         qsort(start_loc, count/sizeof(uint_t), sizeof(uint_t), compare_extable);
 219         return NULL;
 220 }
 221
 222 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */
 223 static void get_mcount_loc(uint_t *_start, uint_t *_stop)
 224 {
 225         FILE *file_start, *file_stop;
 226         char start_buff[20];
 227         char stop_buff[20];
 228         int len = 0;
 229
 230         file_start = popen(" grep start_mcount System.map | awk '{print $1}' ", "r");
 231         if (!file_start) {
 232                 fprintf(stderr, "get start_mcount_loc error!");
 233                 return;
 234         }
 235
 236         file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' ", "r");
 237         if (!file_stop) {
 238                 fprintf(stderr, "get stop_mcount_loc error!");
 239                 pclose(file_start);
 240                 return;
 241         }
 242
 243         while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) {
 244                 len = strlen(start_buff);
 245                 start_buff[len - 1] = '\0';
 246         }
 247         *_start = strtoul(start_buff, NULL, 16);
 248
 249         while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) {
 250                 len = strlen(stop_buff);
 251                 stop_buff[len - 1] = '\0';
 252         }
 253         *_stop = strtoul(stop_buff, NULL, 16);
 254
 255         pclose(file_start);
 256         pclose(file_stop);
 257 }
 258 #endif
 259 static int do_sort(Elf_Ehdr *ehdr,
 260                    char const *const fname,
 261                    table_sort_t custom_sort)
 262 {
 263         int rc = -1;
 264         Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff));
 265         Elf_Shdr *strtab_sec = NULL;
 266         Elf_Shdr *symtab_sec = NULL;
 267         Elf_Shdr *extab_sec = NULL;
 268         Elf_Sym *sym;
 269         const Elf_Sym *symtab;
 270         Elf32_Word *symtab_shndx = NULL;
 271         Elf_Sym *sort_needed_sym = NULL;
 272         Elf_Shdr *sort_needed_sec;
 273         Elf_Rel *relocs = NULL;
 274         int relocs_size = 0;
 275         uint32_t *sort_needed_loc;
 276         const char *secstrings;
 277         const char *strtab;
 278         char *extab_image;
 279         int extab_index = 0;
 280         int i;
 281         int idx;
 282         unsigned int shnum;
 283         unsigned int shstrndx;
 284 #ifdef MCOUNT_SORT_ENABLED
 285         struct elf_mcount_loc mstruct;
 286         uint_t _start_mcount_loc = 0;
 287         uint_t _stop_mcount_loc = 0;
 288         pthread_t mcount_sort_thread;
 289 #endif
 290 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
 291         unsigned int orc_ip_size = 0;
 292         unsigned int orc_size = 0;
 293         unsigned int orc_num_entries = 0;
 294 #endif
 295
 296         shstrndx = r2(&ehdr->e_shstrndx);
 297         if (shstrndx == SHN_XINDEX)
 298                 shstrndx = r(&shdr[0].sh_link);
 299         secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset);
 300
 301         shnum = r2(&ehdr->e_shnum);
 302         if (shnum == SHN_UNDEF)
 303                 shnum = _r(&shdr[0].sh_size);
 304
 305         for (i = 0, s = shdr; s < shdr + shnum; i++, s++) {
 306                 idx = r(&s->sh_name);
 307                 if (!strcmp(secstrings + idx, "__ex_table")) {
 308                         extab_sec = s;
 309                         extab_index = i;
 310                 }
 311                 if (!strcmp(secstrings + idx, ".symtab"))
 312                         symtab_sec = s;
 313                 if (!strcmp(secstrings + idx, ".strtab"))
 314                         strtab_sec = s;
 315
 316                 if ((r(&s->sh_type) == SHT_REL ||
 317                      r(&s->sh_type) == SHT_RELA) &&
 318                     r(&s->sh_info) == extab_index) {
 319                         relocs = (void *)ehdr + _r(&s->sh_offset);
 320                         relocs_size = _r(&s->sh_size);
 321                 }
 322                 if (r(&s->sh_type) == SHT_SYMTAB_SHNDX)
 323                         symtab_shndx = (Elf32_Word *)((const char *)ehdr +
 324                                                       _r(&s->sh_offset));
 325
 326 #ifdef MCOUNT_SORT_ENABLED
 327                 /* locate the .init.data section in vmlinux */
 328                 if (!strcmp(secstrings + idx, ".init.data")) {
 329                         get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc);
 330                         mstruct.ehdr = ehdr;
 331                         mstruct.init_data_sec = s;
 332                         mstruct.start_mcount_loc = _start_mcount_loc;
 333                         mstruct.stop_mcount_loc = _stop_mcount_loc;
 334                 }
 335 #endif
 336
 337 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
 338                 /* locate the ORC unwind tables */
 339                 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
 340                         orc_ip_size = s->sh_size;
 341                         g_orc_ip_table = (int *)((void *)ehdr +
 342                                                    s->sh_offset);
 343                 }
 344                 if (!strcmp(secstrings + idx, ".orc_unwind")) {
 345                         orc_size = s->sh_size;
 346                         g_orc_table = (struct orc_entry *)((void *)ehdr +
 347                                                              s->sh_offset);
 348                 }
 349 #endif
 350         } /* for loop */
 351
 352 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
 353         if (!g_orc_ip_table || !g_orc_table) {
 354                 fprintf(stderr,
 355                         "incomplete ORC unwind tables in file: %s\n", fname);
 356                 goto out;
 357         }
 358
 359         orc_num_entries = orc_ip_size / sizeof(int);
 360         if (orc_ip_size % sizeof(int) != 0 ||
 361             orc_size % sizeof(struct orc_entry) != 0 ||
 362             orc_num_entries != orc_size / sizeof(struct orc_entry)) {
 363                 fprintf(stderr,
 364                         "inconsistent ORC unwind table entries in file: %s\n",
 365                         fname);
 366                 goto out;
 367         }
 368
 369         /* create thread to sort ORC unwind tables concurrently */
 370         if (pthread_create(&orc_sort_thread, NULL,
 371                            sort_orctable, &orc_ip_size)) {
 372                 fprintf(stderr,
 373                         "pthread_create orc_sort_thread failed '%s': %s\n",
 374                         strerror(errno), fname);
 375                 goto out;
 376         }
 377 #endif
 378
 379 #ifdef MCOUNT_SORT_ENABLED
 380         if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) {
 381                 fprintf(stderr,
 382                         "incomplete mcount's sort in file: %s\n",
 383                         fname);
 384                 goto out;
 385         }
 386
 387         /* create thread to sort mcount_loc concurrently */
 388         if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) {
 389                 fprintf(stderr,
 390                         "pthread_create mcount_sort_thread failed '%s': %s\n",
 391                         strerror(errno), fname);
 392                 goto out;
 393         }
 394 #endif
 395         if (!extab_sec) {
 396                 fprintf(stderr, "no __ex_table in file: %s\n", fname);
 397                 goto out;
 398         }
 399
 400         if (!symtab_sec) {
 401                 fprintf(stderr, "no .symtab in file: %s\n", fname);
 402                 goto out;
 403         }
 404
 405         if (!strtab_sec) {
 406                 fprintf(stderr, "no .strtab in file: %s\n", fname);
 407                 goto out;
 408         }
 409
 410         extab_image = (void *)ehdr + _r(&extab_sec->sh_offset);
 411         strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset);
 412         symtab = (const Elf_Sym *)((const char *)ehdr +
 413                                                   _r(&symtab_sec->sh_offset));
 414
 415         if (custom_sort) {
 416                 custom_sort(extab_image, _r(&extab_sec->sh_size));
 417         } else {
 418                 int num_entries = _r(&extab_sec->sh_size) / extable_ent_size;
 419                 qsort(extab_image, num_entries,
 420                       extable_ent_size, compare_extable);
 421         }
 422
 423         /* If there were relocations, we no longer need them. */
 424         if (relocs)
 425                 memset(relocs, 0, relocs_size);
 426
 427         /* find the flag main_extable_sort_needed */
 428         for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset);
 429              sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym);
 430              sym++) {
 431                 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT)
 432                         continue;
 433                 if (!strcmp(strtab + r(&sym->st_name),
 434                             "main_extable_sort_needed")) {
 435                         sort_needed_sym = sym;
 436                         break;
 437                 }
 438         }
 439
 440         if (!sort_needed_sym) {
 441                 fprintf(stderr,
 442                         "no main_extable_sort_needed symbol in file: %s\n",
 443                         fname);
 444                 goto out;
 445         }
 446
 447         sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx),
 448                                              sort_needed_sym - symtab,
 449                                              symtab_shndx)];
 450         sort_needed_loc = (void *)ehdr +
 451                 _r(&sort_needed_sec->sh_offset) +
 452                 _r(&sort_needed_sym->st_value) -
 453                 _r(&sort_needed_sec->sh_addr);
 454
 455         /* extable has been sorted, clear the flag */
 456         w(0, sort_needed_loc);
 457         rc = 0;
 458
 459 out:
 460 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
 461         if (orc_sort_thread) {
 462                 void *retval = NULL;
 463                 /* wait for ORC tables sort done */
 464                 rc = pthread_join(orc_sort_thread, &retval);
 465                 if (rc) {
 466                         fprintf(stderr,
 467                                 "pthread_join failed '%s': %s\n",
 468                                 strerror(errno), fname);
 469                 } else if (retval) {
 470                         rc = -1;
 471                         fprintf(stderr,
 472                                 "failed to sort ORC tables '%s': %s\n",
 473                                 (char *)retval, fname);
 474                 }
 475         }
 476 #endif
 477
 478 #ifdef MCOUNT_SORT_ENABLED
 479         if (mcount_sort_thread) {
 480                 void *retval = NULL;
 481                 /* wait for mcount sort done */
 482                 rc = pthread_join(mcount_sort_thread, &retval);
 483                 if (rc) {
 484                         fprintf(stderr,
 485                                 "pthread_join failed '%s': %s\n",
 486                                 strerror(errno), fname);
 487                 } else if (retval) {
 488                         rc = -1;
 489                         fprintf(stderr,
 490                                 "failed to sort mcount '%s': %s\n",
 491                                 (char *)retval, fname);
 492                 }
 493         }
 494 #endif
 495         return rc;
 496 }