tools/perf/util/machine.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <dirent.h>
   3 #include <errno.h>
   4 #include <inttypes.h>
   5 #include <regex.h>
   6 #include <stdlib.h>
   7 #include "callchain.h"
   8 #include "debug.h"
   9 #include "dso.h"
  10 #include "env.h"
  11 #include "event.h"
  12 #include "evsel.h"
  13 #include "hist.h"
  14 #include "machine.h"
  15 #include "map.h"
  16 #include "map_symbol.h"
  17 #include "branch.h"
  18 #include "mem-events.h"
  19 #include "srcline.h"
  20 #include "symbol.h"
  21 #include "sort.h"
  22 #include "strlist.h"
  23 #include "target.h"
  24 #include "thread.h"
  25 #include "util.h"
  26 #include "vdso.h"
  27 #include <stdbool.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <unistd.h>
  31 #include "unwind.h"
  32 #include "linux/hash.h"
  33 #include "asm/bug.h"
  34 #include "bpf-event.h"
  35
  36 #include <linux/ctype.h>
  37 #include <symbol/kallsyms.h>
  38 #include <linux/mman.h>
  39 #include <linux/string.h>
  40 #include <linux/zalloc.h>
  41
  42 static void __machine__remove_thread(struct machine *machine, struct thread *th, bool lock);
  43
  44 static void dsos__init(struct dsos *dsos)
  45 {
  46         INIT_LIST_HEAD(&dsos->head);
  47         dsos->root = RB_ROOT;
  48         init_rwsem(&dsos->lock);
  49 }
  50
  51 static void machine__threads_init(struct machine *machine)
  52 {
  53         int i;
  54
  55         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
  56                 struct threads *threads = &machine->threads[i];
  57                 threads->entries = RB_ROOT_CACHED;
  58                 init_rwsem(&threads->lock);
  59                 threads->nr = 0;
  60                 INIT_LIST_HEAD(&threads->dead);
  61                 threads->last_match = NULL;
  62         }
  63 }
  64
  65 static int machine__set_mmap_name(struct machine *machine)
  66 {
  67         if (machine__is_host(machine))
  68                 machine->mmap_name = strdup("[kernel.kallsyms]");
  69         else if (machine__is_default_guest(machine))
  70                 machine->mmap_name = strdup("[guest.kernel.kallsyms]");
  71         else if (asprintf(&machine->mmap_name, "[guest.kernel.kallsyms.%d]",
  72                           machine->pid) < 0)
  73                 machine->mmap_name = NULL;
  74
  75         return machine->mmap_name ? 0 : -ENOMEM;
  76 }
  77
  78 int machine__init(struct machine *machine, const char *root_dir, pid_t pid)
  79 {
  80         int err = -ENOMEM;
  81
  82         memset(machine, 0, sizeof(*machine));
  83         map_groups__init(&machine->kmaps, machine);
  84         RB_CLEAR_NODE(&machine->rb_node);
  85         dsos__init(&machine->dsos);
  86
  87         machine__threads_init(machine);
  88
  89         machine->vdso_info = NULL;
  90         machine->env = NULL;
  91
  92         machine->pid = pid;
  93
  94         machine->id_hdr_size = 0;
  95         machine->kptr_restrict_warned = false;
  96         machine->comm_exec = false;
  97         machine->kernel_start = 0;
  98         machine->vmlinux_map = NULL;
  99
 100         machine->root_dir = strdup(root_dir);
 101         if (machine->root_dir == NULL)
 102                 return -ENOMEM;
 103
 104         if (machine__set_mmap_name(machine))
 105                 goto out;
 106
 107         if (pid != HOST_KERNEL_ID) {
 108                 struct thread *thread = machine__findnew_thread(machine, -1,
 109                                                                 pid);
 110                 char comm[64];
 111
 112                 if (thread == NULL)
 113                         goto out;
 114
 115                 snprintf(comm, sizeof(comm), "[guest/%d]", pid);
 116                 thread__set_comm(thread, comm, 0);
 117                 thread__put(thread);
 118         }
 119
 120         machine->current_tid = NULL;
 121         err = 0;
 122
 123 out:
 124         if (err) {
 125                 zfree(&machine->root_dir);
 126                 zfree(&machine->mmap_name);
 127         }
 128         return 0;
 129 }
 130
 131 struct machine *machine__new_host(void)
 132 {
 133         struct machine *machine = malloc(sizeof(*machine));
 134
 135         if (machine != NULL) {
 136                 machine__init(machine, "", HOST_KERNEL_ID);
 137
 138                 if (machine__create_kernel_maps(machine) < 0)
 139                         goto out_delete;
 140         }
 141
 142         return machine;
 143 out_delete:
 144         free(machine);
 145         return NULL;
 146 }
 147
 148 struct machine *machine__new_kallsyms(void)
 149 {
 150         struct machine *machine = machine__new_host();
 151         /*
 152          * FIXME:
 153          * 1) We should switch to machine__load_kallsyms(), i.e. not explicitly
 154          *    ask for not using the kcore parsing code, once this one is fixed
 155          *    to create a map per module.
 156          */
 157         if (machine && machine__load_kallsyms(machine, "/proc/kallsyms") <= 0) {
 158                 machine__delete(machine);
 159                 machine = NULL;
 160         }
 161
 162         return machine;
 163 }
 164
 165 static void dsos__purge(struct dsos *dsos)
 166 {
 167         struct dso *pos, *n;
 168
 169         down_write(&dsos->lock);
 170
 171         list_for_each_entry_safe(pos, n, &dsos->head, node) {
 172                 RB_CLEAR_NODE(&pos->rb_node);
 173                 pos->root = NULL;
 174                 list_del_init(&pos->node);
 175                 dso__put(pos);
 176         }
 177
 178         up_write(&dsos->lock);
 179 }
 180
 181 static void dsos__exit(struct dsos *dsos)
 182 {
 183         dsos__purge(dsos);
 184         exit_rwsem(&dsos->lock);
 185 }
 186
 187 void machine__delete_threads(struct machine *machine)
 188 {
 189         struct rb_node *nd;
 190         int i;
 191
 192         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
 193                 struct threads *threads = &machine->threads[i];
 194                 down_write(&threads->lock);
 195                 nd = rb_first_cached(&threads->entries);
 196                 while (nd) {
 197                         struct thread *t = rb_entry(nd, struct thread, rb_node);
 198
 199                         nd = rb_next(nd);
 200                         __machine__remove_thread(machine, t, false);
 201                 }
 202                 up_write(&threads->lock);
 203         }
 204 }
 205
 206 void machine__exit(struct machine *machine)
 207 {
 208         int i;
 209
 210         if (machine == NULL)
 211                 return;
 212
 213         machine__destroy_kernel_maps(machine);
 214         map_groups__exit(&machine->kmaps);
 215         dsos__exit(&machine->dsos);
 216         machine__exit_vdso(machine);
 217         zfree(&machine->root_dir);
 218         zfree(&machine->mmap_name);
 219         zfree(&machine->current_tid);
 220
 221         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
 222                 struct threads *threads = &machine->threads[i];
 223                 struct thread *thread, *n;
 224                 /*
 225                  * Forget about the dead, at this point whatever threads were
 226                  * left in the dead lists better have a reference count taken
 227                  * by who is using them, and then, when they drop those references
 228                  * and it finally hits zero, thread__put() will check and see that
 229                  * its not in the dead threads list and will not try to remove it
 230                  * from there, just calling thread__delete() straight away.
 231                  */
 232                 list_for_each_entry_safe(thread, n, &threads->dead, node)
 233                         list_del_init(&thread->node);
 234
 235                 exit_rwsem(&threads->lock);
 236         }
 237 }
 238
 239 void machine__delete(struct machine *machine)
 240 {
 241         if (machine) {
 242                 machine__exit(machine);
 243                 free(machine);
 244         }
 245 }
 246
 247 void machines__init(struct machines *machines)
 248 {
 249         machine__init(&machines->host, "", HOST_KERNEL_ID);
 250         machines->guests = RB_ROOT_CACHED;
 251 }
 252
 253 void machines__exit(struct machines *machines)
 254 {
 255         machine__exit(&machines->host);
 256         /* XXX exit guest */
 257 }
 258
 259 struct machine *machines__add(struct machines *machines, pid_t pid,
 260                               const char *root_dir)
 261 {
 262         struct rb_node **p = &machines->guests.rb_root.rb_node;
 263         struct rb_node *parent = NULL;
 264         struct machine *pos, *machine = malloc(sizeof(*machine));
 265         bool leftmost = true;
 266
 267         if (machine == NULL)
 268                 return NULL;
 269
 270         if (machine__init(machine, root_dir, pid) != 0) {
 271                 free(machine);
 272                 return NULL;
 273         }
 274
 275         while (*p != NULL) {
 276                 parent = *p;
 277                 pos = rb_entry(parent, struct machine, rb_node);
 278                 if (pid < pos->pid)
 279                         p = &(*p)->rb_left;
 280                 else {
 281                         p = &(*p)->rb_right;
 282                         leftmost = false;
 283                 }
 284         }
 285
 286         rb_link_node(&machine->rb_node, parent, p);
 287         rb_insert_color_cached(&machine->rb_node, &machines->guests, leftmost);
 288
 289         return machine;
 290 }
 291
 292 void machines__set_comm_exec(struct machines *machines, bool comm_exec)
 293 {
 294         struct rb_node *nd;
 295
 296         machines->host.comm_exec = comm_exec;
 297
 298         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 299                 struct machine *machine = rb_entry(nd, struct machine, rb_node);
 300
 301                 machine->comm_exec = comm_exec;
 302         }
 303 }
 304
 305 struct machine *machines__find(struct machines *machines, pid_t pid)
 306 {
 307         struct rb_node **p = &machines->guests.rb_root.rb_node;
 308         struct rb_node *parent = NULL;
 309         struct machine *machine;
 310         struct machine *default_machine = NULL;
 311
 312         if (pid == HOST_KERNEL_ID)
 313                 return &machines->host;
 314
 315         while (*p != NULL) {
 316                 parent = *p;
 317                 machine = rb_entry(parent, struct machine, rb_node);
 318                 if (pid < machine->pid)
 319                         p = &(*p)->rb_left;
 320                 else if (pid > machine->pid)
 321                         p = &(*p)->rb_right;
 322                 else
 323                         return machine;
 324                 if (!machine->pid)
 325                         default_machine = machine;
 326         }
 327
 328         return default_machine;
 329 }
 330
 331 struct machine *machines__findnew(struct machines *machines, pid_t pid)
 332 {
 333         char path[PATH_MAX];
 334         const char *root_dir = "";
 335         struct machine *machine = machines__find(machines, pid);
 336
 337         if (machine && (machine->pid == pid))
 338                 goto out;
 339
 340         if ((pid != HOST_KERNEL_ID) &&
 341             (pid != DEFAULT_GUEST_KERNEL_ID) &&
 342             (symbol_conf.guestmount)) {
 343                 sprintf(path, "%s/%d", symbol_conf.guestmount, pid);
 344                 if (access(path, R_OK)) {
 345                         static struct strlist *seen;
 346
 347                         if (!seen)
 348                                 seen = strlist__new(NULL, NULL);
 349
 350                         if (!strlist__has_entry(seen, path)) {
 351                                 pr_err("Can't access file %s\n", path);
 352                                 strlist__add(seen, path);
 353                         }
 354                         machine = NULL;
 355                         goto out;
 356                 }
 357                 root_dir = path;
 358         }
 359
 360         machine = machines__add(machines, pid, root_dir);
 361 out:
 362         return machine;
 363 }
 364
 365 void machines__process_guests(struct machines *machines,
 366                               machine__process_t process, void *data)
 367 {
 368         struct rb_node *nd;
 369
 370         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 371                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
 372                 process(pos, data);
 373         }
 374 }
 375
 376 void machines__set_id_hdr_size(struct machines *machines, u16 id_hdr_size)
 377 {
 378         struct rb_node *node;
 379         struct machine *machine;
 380
 381         machines->host.id_hdr_size = id_hdr_size;
 382
 383         for (node = rb_first_cached(&machines->guests); node;
 384              node = rb_next(node)) {
 385                 machine = rb_entry(node, struct machine, rb_node);
 386                 machine->id_hdr_size = id_hdr_size;
 387         }
 388
 389         return;
 390 }
 391
 392 static void machine__update_thread_pid(struct machine *machine,
 393                                        struct thread *th, pid_t pid)
 394 {
 395         struct thread *leader;
 396
 397         if (pid == th->pid_ || pid == -1 || th->pid_ != -1)
 398                 return;
 399
 400         th->pid_ = pid;
 401
 402         if (th->pid_ == th->tid)
 403                 return;
 404
 405         leader = __machine__findnew_thread(machine, th->pid_, th->pid_);
 406         if (!leader)
 407                 goto out_err;
 408
 409         if (!leader->mg)
 410                 leader->mg = map_groups__new(machine);
 411
 412         if (!leader->mg)
 413                 goto out_err;
 414
 415         if (th->mg == leader->mg)
 416                 return;
 417
 418         if (th->mg) {
 419                 /*
 420                  * Maps are created from MMAP events which provide the pid and
 421                  * tid.  Consequently there never should be any maps on a thread
 422                  * with an unknown pid.  Just print an error if there are.
 423                  */
 424                 if (!map_groups__empty(th->mg))
 425                         pr_err("Discarding thread maps for %d:%d\n",
 426                                th->pid_, th->tid);
 427                 map_groups__put(th->mg);
 428         }
 429
 430         th->mg = map_groups__get(leader->mg);
 431 out_put:
 432         thread__put(leader);
 433         return;
 434 out_err:
 435         pr_err("Failed to join map groups for %d:%d\n", th->pid_, th->tid);
 436         goto out_put;
 437 }
 438
 439 /*
 440  * Front-end cache - TID lookups come in blocks,
 441  * so most of the time we dont have to look up
 442  * the full rbtree:
 443  */
 444 static struct thread*
 445 __threads__get_last_match(struct threads *threads, struct machine *machine,
 446                           int pid, int tid)
 447 {
 448         struct thread *th;
 449
 450         th = threads->last_match;
 451         if (th != NULL) {
 452                 if (th->tid == tid) {
 453                         machine__update_thread_pid(machine, th, pid);
 454                         return thread__get(th);
 455                 }
 456
 457                 threads->last_match = NULL;
 458         }
 459
 460         return NULL;
 461 }
 462
 463 static struct thread*
 464 threads__get_last_match(struct threads *threads, struct machine *machine,
 465                         int pid, int tid)
 466 {
 467         struct thread *th = NULL;
 468
 469         if (perf_singlethreaded)
 470                 th = __threads__get_last_match(threads, machine, pid, tid);
 471
 472         return th;
 473 }
 474
 475 static void
 476 __threads__set_last_match(struct threads *threads, struct thread *th)
 477 {
 478         threads->last_match = th;
 479 }
 480
 481 static void
 482 threads__set_last_match(struct threads *threads, struct thread *th)
 483 {
 484         if (perf_singlethreaded)
 485                 __threads__set_last_match(threads, th);
 486 }
 487
 488 /*
 489  * Caller must eventually drop thread->refcnt returned with a successful
 490  * lookup/new thread inserted.
 491  */
 492 static struct thread *____machine__findnew_thread(struct machine *machine,
 493                                                   struct threads *threads,
 494                                                   pid_t pid, pid_t tid,
 495                                                   bool create)
 496 {
 497         struct rb_node **p = &threads->entries.rb_root.rb_node;
 498         struct rb_node *parent = NULL;
 499         struct thread *th;
 500         bool leftmost = true;
 501
 502         th = threads__get_last_match(threads, machine, pid, tid);
 503         if (th)
 504                 return th;
 505
 506         while (*p != NULL) {
 507                 parent = *p;
 508                 th = rb_entry(parent, struct thread, rb_node);
 509
 510                 if (th->tid == tid) {
 511                         threads__set_last_match(threads, th);
 512                         machine__update_thread_pid(machine, th, pid);
 513                         return thread__get(th);
 514                 }
 515
 516                 if (tid < th->tid)
 517                         p = &(*p)->rb_left;
 518                 else {
 519                         p = &(*p)->rb_right;
 520                         leftmost = false;
 521                 }
 522         }
 523
 524         if (!create)
 525                 return NULL;
 526
 527         th = thread__new(pid, tid);
 528         if (th != NULL) {
 529                 rb_link_node(&th->rb_node, parent, p);
 530                 rb_insert_color_cached(&th->rb_node, &threads->entries, leftmost);
 531
 532                 /*
 533                  * We have to initialize map_groups separately
 534                  * after rb tree is updated.
 535                  *
 536                  * The reason is that we call machine__findnew_thread
 537                  * within thread__init_map_groups to find the thread
 538                  * leader and that would screwed the rb tree.
 539                  */
 540                 if (thread__init_map_groups(th, machine)) {
 541                         rb_erase_cached(&th->rb_node, &threads->entries);
 542                         RB_CLEAR_NODE(&th->rb_node);
 543                         thread__put(th);
 544                         return NULL;
 545                 }
 546                 /*
 547                  * It is now in the rbtree, get a ref
 548                  */
 549                 thread__get(th);
 550                 threads__set_last_match(threads, th);
 551                 ++threads->nr;
 552         }
 553
 554         return th;
 555 }
 556
 557 struct thread *__machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid)
 558 {
 559         return ____machine__findnew_thread(machine, machine__threads(machine, tid), pid, tid, true);
 560 }
 561
 562 struct thread *machine__findnew_thread(struct machine *machine, pid_t pid,
 563                                        pid_t tid)
 564 {
 565         struct threads *threads = machine__threads(machine, tid);
 566         struct thread *th;
 567
 568         down_write(&threads->lock);
 569         th = __machine__findnew_thread(machine, pid, tid);
 570         up_write(&threads->lock);
 571         return th;
 572 }
 573
 574 struct thread *machine__find_thread(struct machine *machine, pid_t pid,
 575                                     pid_t tid)
 576 {
 577         struct threads *threads = machine__threads(machine, tid);
 578         struct thread *th;
 579
 580         down_read(&threads->lock);
 581         th =  ____machine__findnew_thread(machine, threads, pid, tid, false);
 582         up_read(&threads->lock);
 583         return th;
 584 }
 585
 586 struct comm *machine__thread_exec_comm(struct machine *machine,
 587                                        struct thread *thread)
 588 {
 589         if (machine->comm_exec)
 590                 return thread__exec_comm(thread);
 591         else
 592                 return thread__comm(thread);
 593 }
 594
 595 int machine__process_comm_event(struct machine *machine, union perf_event *event,
 596                                 struct perf_sample *sample)
 597 {
 598         struct thread *thread = machine__findnew_thread(machine,
 599                                                         event->comm.pid,
 600                                                         event->comm.tid);
 601         bool exec = event->header.misc & PERF_RECORD_MISC_COMM_EXEC;
 602         int err = 0;
 603
 604         if (exec)
 605                 machine->comm_exec = true;
 606
 607         if (dump_trace)
 608                 perf_event__fprintf_comm(event, stdout);
 609
 610         if (thread == NULL ||
 611             __thread__set_comm(thread, event->comm.comm, sample->time, exec)) {
 612                 dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n");
 613                 err = -1;
 614         }
 615
 616         thread__put(thread);
 617
 618         return err;
 619 }
 620
 621 int machine__process_namespaces_event(struct machine *machine __maybe_unused,
 622                                       union perf_event *event,
 623                                       struct perf_sample *sample __maybe_unused)
 624 {
 625         struct thread *thread = machine__findnew_thread(machine,
 626                                                         event->namespaces.pid,
 627                                                         event->namespaces.tid);
 628         int err = 0;
 629
 630         WARN_ONCE(event->namespaces.nr_namespaces > NR_NAMESPACES,
 631                   "\nWARNING: kernel seems to support more namespaces than perf"
 632                   " tool.\nTry updating the perf tool..\n\n");
 633
 634         WARN_ONCE(event->namespaces.nr_namespaces < NR_NAMESPACES,
 635                   "\nWARNING: perf tool seems to support more namespaces than"
 636                   " the kernel.\nTry updating the kernel..\n\n");
 637
 638         if (dump_trace)
 639                 perf_event__fprintf_namespaces(event, stdout);
 640
 641         if (thread == NULL ||
 642             thread__set_namespaces(thread, sample->time, &event->namespaces)) {
 643                 dump_printf("problem processing PERF_RECORD_NAMESPACES, skipping event.\n");
 644                 err = -1;
 645         }
 646
 647         thread__put(thread);
 648
 649         return err;
 650 }
 651
 652 int machine__process_lost_event(struct machine *machine __maybe_unused,
 653                                 union perf_event *event, struct perf_sample *sample __maybe_unused)
 654 {
 655         dump_printf(": id:%" PRI_lu64 ": lost:%" PRI_lu64 "\n",
 656                     event->lost.id, event->lost.lost);
 657         return 0;
 658 }
 659
 660 int machine__process_lost_samples_event(struct machine *machine __maybe_unused,
 661                                         union perf_event *event, struct perf_sample *sample)
 662 {
 663         dump_printf(": id:%" PRIu64 ": lost samples :%" PRI_lu64 "\n",
 664                     sample->id, event->lost_samples.lost);
 665         return 0;
 666 }
 667
 668 static struct dso *machine__findnew_module_dso(struct machine *machine,
 669                                                struct kmod_path *m,
 670                                                const char *filename)
 671 {
 672         struct dso *dso;
 673
 674         down_write(&machine->dsos.lock);
 675
 676         dso = __dsos__find(&machine->dsos, m->name, true);
 677         if (!dso) {
 678                 dso = __dsos__addnew(&machine->dsos, m->name);
 679                 if (dso == NULL)
 680                         goto out_unlock;
 681
 682                 dso__set_module_info(dso, m, machine);
 683                 dso__set_long_name(dso, strdup(filename), true);
 684         }
 685
 686         dso__get(dso);
 687 out_unlock:
 688         up_write(&machine->dsos.lock);
 689         return dso;
 690 }
 691
 692 int machine__process_aux_event(struct machine *machine __maybe_unused,
 693                                union perf_event *event)
 694 {
 695         if (dump_trace)
 696                 perf_event__fprintf_aux(event, stdout);
 697         return 0;
 698 }
 699
 700 int machine__process_itrace_start_event(struct machine *machine __maybe_unused,
 701                                         union perf_event *event)
 702 {
 703         if (dump_trace)
 704                 perf_event__fprintf_itrace_start(event, stdout);
 705         return 0;
 706 }
 707
 708 int machine__process_switch_event(struct machine *machine __maybe_unused,
 709                                   union perf_event *event)
 710 {
 711         if (dump_trace)
 712                 perf_event__fprintf_switch(event, stdout);
 713         return 0;
 714 }
 715
 716 static int machine__process_ksymbol_register(struct machine *machine,
 717                                              union perf_event *event,
 718                                              struct perf_sample *sample __maybe_unused)
 719 {
 720         struct symbol *sym;
 721         struct map *map;
 722
 723         map = map_groups__find(&machine->kmaps, event->ksymbol.addr);
 724         if (!map) {
 725                 map = dso__new_map(event->ksymbol.name);
 726                 if (!map)
 727                         return -ENOMEM;
 728
 729                 map->start = event->ksymbol.addr;
 730                 map->end = map->start + event->ksymbol.len;
 731                 map_groups__insert(&machine->kmaps, map);
 732         }
 733
 734         sym = symbol__new(map->map_ip(map, map->start),
 735                           event->ksymbol.len,
 736                           0, 0, event->ksymbol.name);
 737         if (!sym)
 738                 return -ENOMEM;
 739         dso__insert_symbol(map->dso, sym);
 740         return 0;
 741 }
 742
 743 static int machine__process_ksymbol_unregister(struct machine *machine,
 744                                                union perf_event *event,
 745                                                struct perf_sample *sample __maybe_unused)
 746 {
 747         struct map *map;
 748
 749         map = map_groups__find(&machine->kmaps, event->ksymbol.addr);
 750         if (map)
 751                 map_groups__remove(&machine->kmaps, map);
 752
 753         return 0;
 754 }
 755
 756 int machine__process_ksymbol(struct machine *machine __maybe_unused,
 757                              union perf_event *event,
 758                              struct perf_sample *sample)
 759 {
 760         if (dump_trace)
 761                 perf_event__fprintf_ksymbol(event, stdout);
 762
 763         if (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER)
 764                 return machine__process_ksymbol_unregister(machine, event,
 765                                                            sample);
 766         return machine__process_ksymbol_register(machine, event, sample);
 767 }
 768
 769 static void dso__adjust_kmod_long_name(struct dso *dso, const char *filename)
 770 {
 771         const char *dup_filename;
 772
 773         if (!filename || !dso || !dso->long_name)
 774                 return;
 775         if (dso->long_name[0] != '[')
 776                 return;
 777         if (!strchr(filename, '/'))
 778                 return;
 779
 780         dup_filename = strdup(filename);
 781         if (!dup_filename)
 782                 return;
 783
 784         dso__set_long_name(dso, dup_filename, true);
 785 }
 786
 787 struct map *machine__findnew_module_map(struct machine *machine, u64 start,
 788                                         const char *filename)
 789 {
 790         struct map *map = NULL;
 791         struct dso *dso = NULL;
 792         struct kmod_path m;
 793
 794         if (kmod_path__parse_name(&m, filename))
 795                 return NULL;
 796
 797         map = map_groups__find_by_name(&machine->kmaps, m.name);
 798         if (map) {
 799                 /*
 800                  * If the map's dso is an offline module, give dso__load()
 801                  * a chance to find the file path of that module by fixing
 802                  * long_name.
 803                  */
 804                 dso__adjust_kmod_long_name(map->dso, filename);
 805                 goto out;
 806         }
 807
 808         dso = machine__findnew_module_dso(machine, &m, filename);
 809         if (dso == NULL)
 810                 goto out;
 811
 812         map = map__new2(start, dso);
 813         if (map == NULL)
 814                 goto out;
 815
 816         map_groups__insert(&machine->kmaps, map);
 817
 818         /* Put the map here because map_groups__insert alread got it */
 819         map__put(map);
 820 out:
 821         /* put the dso here, corresponding to  machine__findnew_module_dso */
 822         dso__put(dso);
 823         zfree(&m.name);
 824         return map;
 825 }
 826
 827 size_t machines__fprintf_dsos(struct machines *machines, FILE *fp)
 828 {
 829         struct rb_node *nd;
 830         size_t ret = __dsos__fprintf(&machines->host.dsos.head, fp);
 831
 832         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 833                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
 834                 ret += __dsos__fprintf(&pos->dsos.head, fp);
 835         }
 836
 837         return ret;
 838 }
 839
 840 size_t machine__fprintf_dsos_buildid(struct machine *m, FILE *fp,
 841                                      bool (skip)(struct dso *dso, int parm), int parm)
 842 {
 843         return __dsos__fprintf_buildid(&m->dsos.head, fp, skip, parm);
 844 }
 845
 846 size_t machines__fprintf_dsos_buildid(struct machines *machines, FILE *fp,
 847                                      bool (skip)(struct dso *dso, int parm), int parm)
 848 {
 849         struct rb_node *nd;
 850         size_t ret = machine__fprintf_dsos_buildid(&machines->host, fp, skip, parm);
 851
 852         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 853                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
 854                 ret += machine__fprintf_dsos_buildid(pos, fp, skip, parm);
 855         }
 856         return ret;
 857 }
 858
 859 size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp)
 860 {
 861         int i;
 862         size_t printed = 0;
 863         struct dso *kdso = machine__kernel_map(machine)->dso;
 864
 865         if (kdso->has_build_id) {
 866                 char filename[PATH_MAX];
 867                 if (dso__build_id_filename(kdso, filename, sizeof(filename),
 868                                            false))
 869                         printed += fprintf(fp, "[0] %s\n", filename);
 870         }
 871
 872         for (i = 0; i < vmlinux_path__nr_entries; ++i)
 873                 printed += fprintf(fp, "[%d] %s\n",
 874                                    i + kdso->has_build_id, vmlinux_path[i]);
 875
 876         return printed;
 877 }
 878
 879 size_t machine__fprintf(struct machine *machine, FILE *fp)
 880 {
 881         struct rb_node *nd;
 882         size_t ret;
 883         int i;
 884
 885         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
 886                 struct threads *threads = &machine->threads[i];
 887
 888                 down_read(&threads->lock);
 889
 890                 ret = fprintf(fp, "Threads: %u\n", threads->nr);
 891
 892                 for (nd = rb_first_cached(&threads->entries); nd;
 893                      nd = rb_next(nd)) {
 894                         struct thread *pos = rb_entry(nd, struct thread, rb_node);
 895
 896                         ret += thread__fprintf(pos, fp);
 897                 }
 898
 899                 up_read(&threads->lock);
 900         }
 901         return ret;
 902 }
 903
 904 static struct dso *machine__get_kernel(struct machine *machine)
 905 {
 906         const char *vmlinux_name = machine->mmap_name;
 907         struct dso *kernel;
 908
 909         if (machine__is_host(machine)) {
 910                 if (symbol_conf.vmlinux_name)
 911                         vmlinux_name = symbol_conf.vmlinux_name;
 912
 913                 kernel = machine__findnew_kernel(machine, vmlinux_name,
 914                                                  "[kernel]", DSO_TYPE_KERNEL);
 915         } else {
 916                 if (symbol_conf.default_guest_vmlinux_name)
 917                         vmlinux_name = symbol_conf.default_guest_vmlinux_name;
 918
 919                 kernel = machine__findnew_kernel(machine, vmlinux_name,
 920                                                  "[guest.kernel]",
 921                                                  DSO_TYPE_GUEST_KERNEL);
 922         }
 923
 924         if (kernel != NULL && (!kernel->has_build_id))
 925                 dso__read_running_kernel_build_id(kernel, machine);
 926
 927         return kernel;
 928 }
 929
 930 struct process_args {
 931         u64 start;
 932 };
 933
 934 void machine__get_kallsyms_filename(struct machine *machine, char *buf,
 935                                     size_t bufsz)
 936 {
 937         if (machine__is_default_guest(machine))
 938                 scnprintf(buf, bufsz, "%s", symbol_conf.default_guest_kallsyms);
 939         else
 940                 scnprintf(buf, bufsz, "%s/proc/kallsyms", machine->root_dir);
 941 }
 942
 943 const char *ref_reloc_sym_names[] = {"_text", "_stext", NULL};
 944
 945 /* Figure out the start address of kernel map from /proc/kallsyms.
 946  * Returns the name of the start symbol in *symbol_name. Pass in NULL as
 947  * symbol_name if it's not that important.
 948  */
 949 static int machine__get_running_kernel_start(struct machine *machine,
 950                                              const char **symbol_name,
 951                                              u64 *start, u64 *end)
 952 {
 953         char filename[PATH_MAX];
 954         int i, err = -1;
 955         const char *name;
 956         u64 addr = 0;
 957
 958         machine__get_kallsyms_filename(machine, filename, PATH_MAX);
 959
 960         if (symbol__restricted_filename(filename, "/proc/kallsyms"))
 961                 return 0;
 962
 963         for (i = 0; (name = ref_reloc_sym_names[i]) != NULL; i++) {
 964                 err = kallsyms__get_function_start(filename, name, &addr);
 965                 if (!err)
 966                         break;
 967         }
 968
 969         if (err)
 970                 return -1;
 971
 972         if (symbol_name)
 973                 *symbol_name = name;
 974
 975         *start = addr;
 976
 977         err = kallsyms__get_function_start(filename, "_etext", &addr);
 978         if (!err)
 979                 *end = addr;
 980
 981         return 0;
 982 }
 983
 984 int machine__create_extra_kernel_map(struct machine *machine,
 985                                      struct dso *kernel,
 986                                      struct extra_kernel_map *xm)
 987 {
 988         struct kmap *kmap;
 989         struct map *map;
 990
 991         map = map__new2(xm->start, kernel);
 992         if (!map)
 993                 return -1;
 994
 995         map->end   = xm->end;
 996         map->pgoff = xm->pgoff;
 997
 998         kmap = map__kmap(map);
 999
1000         kmap->kmaps = &machine->kmaps;
1001         strlcpy(kmap->name, xm->name, KMAP_NAME_LEN);
1002
1003         map_groups__insert(&machine->kmaps, map);
1004
1005         pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n",
1006                   kmap->name, map->start, map->end);
1007
1008         map__put(map);
1009
1010         return 0;
1011 }
1012
1013 static u64 find_entry_trampoline(struct dso *dso)
1014 {
1015         /* Duplicates are removed so lookup all aliases */
1016         const char *syms[] = {
1017                 "_entry_trampoline",
1018                 "__entry_trampoline_start",
1019                 "entry_SYSCALL_64_trampoline",
1020         };
1021         struct symbol *sym = dso__first_symbol(dso);
1022         unsigned int i;
1023
1024         for (; sym; sym = dso__next_symbol(sym)) {
1025                 if (sym->binding != STB_GLOBAL)
1026                         continue;
1027                 for (i = 0; i < ARRAY_SIZE(syms); i++) {
1028                         if (!strcmp(sym->name, syms[i]))
1029                                 return sym->start;
1030                 }
1031         }
1032
1033         return 0;
1034 }
1035
1036 /*
1037  * These values can be used for kernels that do not have symbols for the entry
1038  * trampolines in kallsyms.
1039  */
1040 #define X86_64_CPU_ENTRY_AREA_PER_CPU   0xfffffe0000000000ULL
1041 #define X86_64_CPU_ENTRY_AREA_SIZE      0x2c000
1042 #define X86_64_ENTRY_TRAMPOLINE         0x6000
1043
1044 /* Map x86_64 PTI entry trampolines */
1045 int machine__map_x86_64_entry_trampolines(struct machine *machine,
1046                                           struct dso *kernel)
1047 {
1048         struct map_groups *kmaps = &machine->kmaps;
1049         struct maps *maps = &kmaps->maps;
1050         int nr_cpus_avail, cpu;
1051         bool found = false;
1052         struct map *map;
1053         u64 pgoff;
1054
1055         /*
1056          * In the vmlinux case, pgoff is a virtual address which must now be
1057          * mapped to a vmlinux offset.
1058          */
1059         for (map = maps__first(maps); map; map = map__next(map)) {
1060                 struct kmap *kmap = __map__kmap(map);
1061                 struct map *dest_map;
1062
1063                 if (!kmap || !is_entry_trampoline(kmap->name))
1064                         continue;
1065
1066                 dest_map = map_groups__find(kmaps, map->pgoff);
1067                 if (dest_map != map)
1068                         map->pgoff = dest_map->map_ip(dest_map, map->pgoff);
1069                 found = true;
1070         }
1071         if (found || machine->trampolines_mapped)
1072                 return 0;
1073
1074         pgoff = find_entry_trampoline(kernel);
1075         if (!pgoff)
1076                 return 0;
1077
1078         nr_cpus_avail = machine__nr_cpus_avail(machine);
1079
1080         /* Add a 1 page map for each CPU's entry trampoline */
1081         for (cpu = 0; cpu < nr_cpus_avail; cpu++) {
1082                 u64 va = X86_64_CPU_ENTRY_AREA_PER_CPU +
1083                          cpu * X86_64_CPU_ENTRY_AREA_SIZE +
1084                          X86_64_ENTRY_TRAMPOLINE;
1085                 struct extra_kernel_map xm = {
1086                         .start = va,
1087                         .end   = va + page_size,
1088                         .pgoff = pgoff,
1089                 };
1090
1091                 strlcpy(xm.name, ENTRY_TRAMPOLINE_NAME, KMAP_NAME_LEN);
1092
1093                 if (machine__create_extra_kernel_map(machine, kernel, &xm) < 0)
1094                         return -1;
1095         }
1096
1097         machine->trampolines_mapped = nr_cpus_avail;
1098
1099         return 0;
1100 }
1101
1102 int __weak machine__create_extra_kernel_maps(struct machine *machine __maybe_unused,
1103                                              struct dso *kernel __maybe_unused)
1104 {
1105         return 0;
1106 }
1107
1108 static int
1109 __machine__create_kernel_maps(struct machine *machine, struct dso *kernel)
1110 {
1111         struct kmap *kmap;
1112         struct map *map;
1113
1114         /* In case of renewal the kernel map, destroy previous one */
1115         machine__destroy_kernel_maps(machine);
1116
1117         machine->vmlinux_map = map__new2(0, kernel);
1118         if (machine->vmlinux_map == NULL)
1119                 return -1;
1120
1121         machine->vmlinux_map->map_ip = machine->vmlinux_map->unmap_ip = identity__map_ip;
1122         map = machine__kernel_map(machine);
1123         kmap = map__kmap(map);
1124         if (!kmap)
1125                 return -1;
1126
1127         kmap->kmaps = &machine->kmaps;
1128         map_groups__insert(&machine->kmaps, map);
1129
1130         return 0;
1131 }
1132
1133 void machine__destroy_kernel_maps(struct machine *machine)
1134 {
1135         struct kmap *kmap;
1136         struct map *map = machine__kernel_map(machine);
1137
1138         if (map == NULL)
1139                 return;
1140
1141         kmap = map__kmap(map);
1142         map_groups__remove(&machine->kmaps, map);
1143         if (kmap && kmap->ref_reloc_sym) {
1144                 zfree((char **)&kmap->ref_reloc_sym->name);
1145                 zfree(&kmap->ref_reloc_sym);
1146         }
1147
1148         map__zput(machine->vmlinux_map);
1149 }
1150
1151 int machines__create_guest_kernel_maps(struct machines *machines)
1152 {
1153         int ret = 0;
1154         struct dirent **namelist = NULL;
1155         int i, items = 0;
1156         char path[PATH_MAX];
1157         pid_t pid;
1158         char *endp;
1159
1160         if (symbol_conf.default_guest_vmlinux_name ||
1161             symbol_conf.default_guest_modules ||
1162             symbol_conf.default_guest_kallsyms) {
1163                 machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID);
1164         }
1165
1166         if (symbol_conf.guestmount) {
1167                 items = scandir(symbol_conf.guestmount, &namelist, NULL, NULL);
1168                 if (items <= 0)
1169                         return -ENOENT;
1170                 for (i = 0; i < items; i++) {
1171                         if (!isdigit(namelist[i]->d_name[0])) {
1172                                 /* Filter out . and .. */
1173                                 continue;
1174                         }
1175                         pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10);
1176                         if ((*endp != '\0') ||
1177                             (endp == namelist[i]->d_name) ||
1178                             (errno == ERANGE)) {
1179                                 pr_debug("invalid directory (%s). Skipping.\n",
1180                                          namelist[i]->d_name);
1181                                 continue;
1182                         }
1183                         sprintf(path, "%s/%s/proc/kallsyms",
1184                                 symbol_conf.guestmount,
1185                                 namelist[i]->d_name);
1186                         ret = access(path, R_OK);
1187                         if (ret) {
1188                                 pr_debug("Can't access file %s\n", path);
1189                                 goto failure;
1190                         }
1191                         machines__create_kernel_maps(machines, pid);
1192                 }
1193 failure:
1194                 free(namelist);
1195         }
1196
1197         return ret;
1198 }
1199
1200 void machines__destroy_kernel_maps(struct machines *machines)
1201 {
1202         struct rb_node *next = rb_first_cached(&machines->guests);
1203
1204         machine__destroy_kernel_maps(&machines->host);
1205
1206         while (next) {
1207                 struct machine *pos = rb_entry(next, struct machine, rb_node);
1208
1209                 next = rb_next(&pos->rb_node);
1210                 rb_erase_cached(&pos->rb_node, &machines->guests);
1211                 machine__delete(pos);
1212         }
1213 }
1214
1215 int machines__create_kernel_maps(struct machines *machines, pid_t pid)
1216 {
1217         struct machine *machine = machines__findnew(machines, pid);
1218
1219         if (machine == NULL)
1220                 return -1;
1221
1222         return machine__create_kernel_maps(machine);
1223 }
1224
1225 int machine__load_kallsyms(struct machine *machine, const char *filename)
1226 {
1227         struct map *map = machine__kernel_map(machine);
1228         int ret = __dso__load_kallsyms(map->dso, filename, map, true);
1229
1230         if (ret > 0) {
1231                 dso__set_loaded(map->dso);
1232                 /*
1233                  * Since /proc/kallsyms will have multiple sessions for the
1234                  * kernel, with modules between them, fixup the end of all
1235                  * sections.
1236                  */
1237                 map_groups__fixup_end(&machine->kmaps);
1238         }
1239
1240         return ret;
1241 }
1242
1243 int machine__load_vmlinux_path(struct machine *machine)
1244 {
1245         struct map *map = machine__kernel_map(machine);
1246         int ret = dso__load_vmlinux_path(map->dso, map);
1247
1248         if (ret > 0)
1249                 dso__set_loaded(map->dso);
1250
1251         return ret;
1252 }
1253
1254 static char *get_kernel_version(const char *root_dir)
1255 {
1256         char version[PATH_MAX];
1257         FILE *file;
1258         char *name, *tmp;
1259         const char *prefix = "Linux version ";
1260
1261         sprintf(version, "%s/proc/version", root_dir);
1262         file = fopen(version, "r");
1263         if (!file)
1264                 return NULL;
1265
1266         tmp = fgets(version, sizeof(version), file);
1267         fclose(file);
1268         if (!tmp)
1269                 return NULL;
1270
1271         name = strstr(version, prefix);
1272         if (!name)
1273                 return NULL;
1274         name += strlen(prefix);
1275         tmp = strchr(name, ' ');
1276         if (tmp)
1277                 *tmp = '\0';
1278
1279         return strdup(name);
1280 }
1281
1282 static bool is_kmod_dso(struct dso *dso)
1283 {
1284         return dso->symtab_type == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE ||
1285                dso->symtab_type == DSO_BINARY_TYPE__GUEST_KMODULE;
1286 }
1287
1288 static int map_groups__set_module_path(struct map_groups *mg, const char *path,
1289                                        struct kmod_path *m)
1290 {
1291         char *long_name;
1292         struct map *map = map_groups__find_by_name(mg, m->name);
1293
1294         if (map == NULL)
1295                 return 0;
1296
1297         long_name = strdup(path);
1298         if (long_name == NULL)
1299                 return -ENOMEM;
1300
1301         dso__set_long_name(map->dso, long_name, true);
1302         dso__kernel_module_get_build_id(map->dso, "");
1303
1304         /*
1305          * Full name could reveal us kmod compression, so
1306          * we need to update the symtab_type if needed.
1307          */
1308         if (m->comp && is_kmod_dso(map->dso)) {
1309                 map->dso->symtab_type++;
1310                 map->dso->comp = m->comp;
1311         }
1312
1313         return 0;
1314 }
1315
1316 static int map_groups__set_modules_path_dir(struct map_groups *mg,
1317                                 const char *dir_name, int depth)
1318 {
1319         struct dirent *dent;
1320         DIR *dir = opendir(dir_name);
1321         int ret = 0;
1322
1323         if (!dir) {
1324                 pr_debug("%s: cannot open %s dir\n", __func__, dir_name);
1325                 return -1;
1326         }
1327
1328         while ((dent = readdir(dir)) != NULL) {
1329                 char path[PATH_MAX];
1330                 struct stat st;
1331
1332                 /*sshfs might return bad dent->d_type, so we have to stat*/
1333                 snprintf(path, sizeof(path), "%s/%s", dir_name, dent->d_name);
1334                 if (stat(path, &st))
1335                         continue;
1336
1337                 if (S_ISDIR(st.st_mode)) {
1338                         if (!strcmp(dent->d_name, ".") ||
1339                             !strcmp(dent->d_name, ".."))
1340                                 continue;
1341
1342                         /* Do not follow top-level source and build symlinks */
1343                         if (depth == 0) {
1344                                 if (!strcmp(dent->d_name, "source") ||
1345                                     !strcmp(dent->d_name, "build"))
1346                                         continue;
1347                         }
1348
1349                         ret = map_groups__set_modules_path_dir(mg, path,
1350                                                                depth + 1);
1351                         if (ret < 0)
1352                                 goto out;
1353                 } else {
1354                         struct kmod_path m;
1355
1356                         ret = kmod_path__parse_name(&m, dent->d_name);
1357                         if (ret)
1358                                 goto out;
1359
1360                         if (m.kmod)
1361                                 ret = map_groups__set_module_path(mg, path, &m);
1362
1363                         zfree(&m.name);
1364
1365                         if (ret)
1366                                 goto out;
1367                 }
1368         }
1369
1370 out:
1371         closedir(dir);
1372         return ret;
1373 }
1374
1375 static int machine__set_modules_path(struct machine *machine)
1376 {
1377         char *version;
1378         char modules_path[PATH_MAX];
1379
1380         version = get_kernel_version(machine->root_dir);
1381         if (!version)
1382                 return -1;
1383
1384         snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s",
1385                  machine->root_dir, version);
1386         free(version);
1387
1388         return map_groups__set_modules_path_dir(&machine->kmaps, modules_path, 0);
1389 }
1390 int __weak arch__fix_module_text_start(u64 *start __maybe_unused,
1391                                 u64 *size __maybe_unused,
1392                                 const char *name __maybe_unused)
1393 {
1394         return 0;
1395 }
1396
1397 static int machine__create_module(void *arg, const char *name, u64 start,
1398                                   u64 size)
1399 {
1400         struct machine *machine = arg;
1401         struct map *map;
1402
1403         if (arch__fix_module_text_start(&start, &size, name) < 0)
1404                 return -1;
1405
1406         map = machine__findnew_module_map(machine, start, name);
1407         if (map == NULL)
1408                 return -1;
1409         map->end = start + size;
1410
1411         dso__kernel_module_get_build_id(map->dso, machine->root_dir);
1412
1413         return 0;
1414 }
1415
1416 static int machine__create_modules(struct machine *machine)
1417 {
1418         const char *modules;
1419         char path[PATH_MAX];
1420
1421         if (machine__is_default_guest(machine)) {
1422                 modules = symbol_conf.default_guest_modules;
1423         } else {
1424                 snprintf(path, PATH_MAX, "%s/proc/modules", machine->root_dir);
1425                 modules = path;
1426         }
1427
1428         if (symbol__restricted_filename(modules, "/proc/modules"))
1429                 return -1;
1430
1431         if (modules__parse(modules, machine, machine__create_module))
1432                 return -1;
1433
1434         if (!machine__set_modules_path(machine))
1435                 return 0;
1436
1437         pr_debug("Problems setting modules path maps, continuing anyway...\n");
1438
1439         return 0;
1440 }
1441
1442 static void machine__set_kernel_mmap(struct machine *machine,
1443                                      u64 start, u64 end)
1444 {
1445         machine->vmlinux_map->start = start;
1446         machine->vmlinux_map->end   = end;
1447         /*
1448          * Be a bit paranoid here, some perf.data file came with
1449          * a zero sized synthesized MMAP event for the kernel.
1450          */
1451         if (start == 0 && end == 0)
1452                 machine->vmlinux_map->end = ~0ULL;
1453 }
1454
1455 static void machine__update_kernel_mmap(struct machine *machine,
1456                                      u64 start, u64 end)
1457 {
1458         struct map *map = machine__kernel_map(machine);
1459
1460         map__get(map);
1461         map_groups__remove(&machine->kmaps, map);
1462
1463         machine__set_kernel_mmap(machine, start, end);
1464
1465         map_groups__insert(&machine->kmaps, map);
1466         map__put(map);
1467 }
1468
1469 int machine__create_kernel_maps(struct machine *machine)
1470 {
1471         struct dso *kernel = machine__get_kernel(machine);
1472         const char *name = NULL;
1473         struct map *map;
1474         u64 start = 0, end = ~0ULL;
1475         int ret;
1476
1477         if (kernel == NULL)
1478                 return -1;
1479
1480         ret = __machine__create_kernel_maps(machine, kernel);
1481         if (ret < 0)
1482                 goto out_put;
1483
1484         if (symbol_conf.use_modules && machine__create_modules(machine) < 0) {
1485                 if (machine__is_host(machine))
1486                         pr_debug("Problems creating module maps, "
1487                                  "continuing anyway...\n");
1488                 else
1489                         pr_debug("Problems creating module maps for guest %d, "
1490                                  "continuing anyway...\n", machine->pid);
1491         }
1492
1493         if (!machine__get_running_kernel_start(machine, &name, &start, &end)) {
1494                 if (name &&
1495                     map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, name, start)) {
1496                         machine__destroy_kernel_maps(machine);
1497                         ret = -1;
1498                         goto out_put;
1499                 }
1500
1501                 /*
1502                  * we have a real start address now, so re-order the kmaps
1503                  * assume it's the last in the kmaps
1504                  */
1505                 machine__update_kernel_mmap(machine, start, end);
1506         }
1507
1508         if (machine__create_extra_kernel_maps(machine, kernel))
1509                 pr_debug("Problems creating extra kernel maps, continuing anyway...\n");
1510
1511         if (end == ~0ULL) {
1512                 /* update end address of the kernel map using adjacent module address */
1513                 map = map__next(machine__kernel_map(machine));
1514                 if (map)
1515                         machine__set_kernel_mmap(machine, start, map->start);
1516         }
1517
1518 out_put:
1519         dso__put(kernel);
1520         return ret;
1521 }
1522
1523 static bool machine__uses_kcore(struct machine *machine)
1524 {
1525         struct dso *dso;
1526
1527         list_for_each_entry(dso, &machine->dsos.head, node) {
1528                 if (dso__is_kcore(dso))
1529                         return true;
1530         }
1531
1532         return false;
1533 }
1534
1535 static bool perf_event__is_extra_kernel_mmap(struct machine *machine,
1536                                              union perf_event *event)
1537 {
1538         return machine__is(machine, "x86_64") &&
1539                is_entry_trampoline(event->mmap.filename);
1540 }
1541
1542 static int machine__process_extra_kernel_map(struct machine *machine,
1543                                              union perf_event *event)
1544 {
1545         struct map *kernel_map = machine__kernel_map(machine);
1546         struct dso *kernel = kernel_map ? kernel_map->dso : NULL;
1547         struct extra_kernel_map xm = {
1548                 .start = event->mmap.start,
1549                 .end   = event->mmap.start + event->mmap.len,
1550                 .pgoff = event->mmap.pgoff,
1551         };
1552
1553         if (kernel == NULL)
1554                 return -1;
1555
1556         strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN);
1557
1558         return machine__create_extra_kernel_map(machine, kernel, &xm);
1559 }
1560
1561 static int machine__process_kernel_mmap_event(struct machine *machine,
1562                                               union perf_event *event)
1563 {
1564         struct map *map;
1565         enum dso_kernel_type kernel_type;
1566         bool is_kernel_mmap;
1567
1568         /* If we have maps from kcore then we do not need or want any others */
1569         if (machine__uses_kcore(machine))
1570                 return 0;
1571
1572         if (machine__is_host(machine))
1573                 kernel_type = DSO_TYPE_KERNEL;
1574         else
1575                 kernel_type = DSO_TYPE_GUEST_KERNEL;
1576
1577         is_kernel_mmap = memcmp(event->mmap.filename,
1578                                 machine->mmap_name,
1579                                 strlen(machine->mmap_name) - 1) == 0;
1580         if (event->mmap.filename[0] == '/' ||
1581             (!is_kernel_mmap && event->mmap.filename[0] == '[')) {
1582                 map = machine__findnew_module_map(machine, event->mmap.start,
1583                                                   event->mmap.filename);
1584                 if (map == NULL)
1585                         goto out_problem;
1586
1587                 map->end = map->start + event->mmap.len;
1588         } else if (is_kernel_mmap) {
1589                 const char *symbol_name = (event->mmap.filename +
1590                                 strlen(machine->mmap_name));
1591                 /*
1592                  * Should be there already, from the build-id table in
1593                  * the header.
1594                  */
1595                 struct dso *kernel = NULL;
1596                 struct dso *dso;
1597
1598                 down_read(&machine->dsos.lock);
1599
1600                 list_for_each_entry(dso, &machine->dsos.head, node) {
1601
1602                         /*
1603                          * The cpumode passed to is_kernel_module is not the
1604                          * cpumode of *this* event. If we insist on passing
1605                          * correct cpumode to is_kernel_module, we should
1606                          * record the cpumode when we adding this dso to the
1607                          * linked list.
1608                          *
1609                          * However we don't really need passing correct
1610                          * cpumode.  We know the correct cpumode must be kernel
1611                          * mode (if not, we should not link it onto kernel_dsos
1612                          * list).
1613                          *
1614                          * Therefore, we pass PERF_RECORD_MISC_CPUMODE_UNKNOWN.
1615                          * is_kernel_module() treats it as a kernel cpumode.
1616                          */
1617
1618                         if (!dso->kernel ||
1619                             is_kernel_module(dso->long_name,
1620                                              PERF_RECORD_MISC_CPUMODE_UNKNOWN))
1621                                 continue;
1622
1623
1624                         kernel = dso;
1625                         break;
1626                 }
1627
1628                 up_read(&machine->dsos.lock);
1629
1630                 if (kernel == NULL)
1631                         kernel = machine__findnew_dso(machine, machine->mmap_name);
1632                 if (kernel == NULL)
1633                         goto out_problem;
1634
1635                 kernel->kernel = kernel_type;
1636                 if (__machine__create_kernel_maps(machine, kernel) < 0) {
1637                         dso__put(kernel);
1638                         goto out_problem;
1639                 }
1640
1641                 if (strstr(kernel->long_name, "vmlinux"))
1642                         dso__set_short_name(kernel, "[kernel.vmlinux]", false);
1643
1644                 machine__update_kernel_mmap(machine, event->mmap.start,
1645                                          event->mmap.start + event->mmap.len);
1646
1647                 /*
1648                  * Avoid using a zero address (kptr_restrict) for the ref reloc
1649                  * symbol. Effectively having zero here means that at record
1650                  * time /proc/sys/kernel/kptr_restrict was non zero.
1651                  */
1652                 if (event->mmap.pgoff != 0) {
1653                         map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map,
1654                                                         symbol_name,
1655                                                         event->mmap.pgoff);
1656                 }
1657
1658                 if (machine__is_default_guest(machine)) {
1659                         /*
1660                          * preload dso of guest kernel and modules
1661                          */
1662                         dso__load(kernel, machine__kernel_map(machine));
1663                 }
1664         } else if (perf_event__is_extra_kernel_mmap(machine, event)) {
1665                 return machine__process_extra_kernel_map(machine, event);
1666         }
1667         return 0;
1668 out_problem:
1669         return -1;
1670 }
1671
1672 int machine__process_mmap2_event(struct machine *machine,
1673                                  union perf_event *event,
1674                                  struct perf_sample *sample)
1675 {
1676         struct thread *thread;
1677         struct map *map;
1678         int ret = 0;
1679
1680         if (dump_trace)
1681                 perf_event__fprintf_mmap2(event, stdout);
1682
1683         if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1684             sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1685                 ret = machine__process_kernel_mmap_event(machine, event);
1686                 if (ret < 0)
1687                         goto out_problem;
1688                 return 0;
1689         }
1690
1691         thread = machine__findnew_thread(machine, event->mmap2.pid,
1692                                         event->mmap2.tid);
1693         if (thread == NULL)
1694                 goto out_problem;
1695
1696         map = map__new(machine, event->mmap2.start,
1697                         event->mmap2.len, event->mmap2.pgoff,
1698                         event->mmap2.maj,
1699                         event->mmap2.min, event->mmap2.ino,
1700                         event->mmap2.ino_generation,
1701                         event->mmap2.prot,
1702                         event->mmap2.flags,
1703                         event->mmap2.filename, thread);
1704
1705         if (map == NULL)
1706                 goto out_problem_map;
1707
1708         ret = thread__insert_map(thread, map);
1709         if (ret)
1710                 goto out_problem_insert;
1711
1712         thread__put(thread);
1713         map__put(map);
1714         return 0;
1715
1716 out_problem_insert:
1717         map__put(map);
1718 out_problem_map:
1719         thread__put(thread);
1720 out_problem:
1721         dump_printf("problem processing PERF_RECORD_MMAP2, skipping event.\n");
1722         return 0;
1723 }
1724
1725 int machine__process_mmap_event(struct machine *machine, union perf_event *event,
1726                                 struct perf_sample *sample)
1727 {
1728         struct thread *thread;
1729         struct map *map;
1730         u32 prot = 0;
1731         int ret = 0;
1732
1733         if (dump_trace)
1734                 perf_event__fprintf_mmap(event, stdout);
1735
1736         if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1737             sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1738                 ret = machine__process_kernel_mmap_event(machine, event);
1739                 if (ret < 0)
1740                         goto out_problem;
1741                 return 0;
1742         }
1743
1744         thread = machine__findnew_thread(machine, event->mmap.pid,
1745                                          event->mmap.tid);
1746         if (thread == NULL)
1747                 goto out_problem;
1748
1749         if (!(event->header.misc & PERF_RECORD_MISC_MMAP_DATA))
1750                 prot = PROT_EXEC;
1751
1752         map = map__new(machine, event->mmap.start,
1753                         event->mmap.len, event->mmap.pgoff,
1754                         0, 0, 0, 0, prot, 0,
1755                         event->mmap.filename,
1756                         thread);
1757
1758         if (map == NULL)
1759                 goto out_problem_map;
1760
1761         ret = thread__insert_map(thread, map);
1762         if (ret)
1763                 goto out_problem_insert;
1764
1765         thread__put(thread);
1766         map__put(map);
1767         return 0;
1768
1769 out_problem_insert:
1770         map__put(map);
1771 out_problem_map:
1772         thread__put(thread);
1773 out_problem:
1774         dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n");
1775         return 0;
1776 }
1777
1778 static void __machine__remove_thread(struct machine *machine, struct thread *th, bool lock)
1779 {
1780         struct threads *threads = machine__threads(machine, th->tid);
1781
1782         if (threads->last_match == th)
1783                 threads__set_last_match(threads, NULL);
1784
1785         if (lock)
1786                 down_write(&threads->lock);
1787
1788         BUG_ON(refcount_read(&th->refcnt) == 0);
1789
1790         rb_erase_cached(&th->rb_node, &threads->entries);
1791         RB_CLEAR_NODE(&th->rb_node);
1792         --threads->nr;
1793         /*
1794          * Move it first to the dead_threads list, then drop the reference,
1795          * if this is the last reference, then the thread__delete destructor
1796          * will be called and we will remove it from the dead_threads list.
1797          */
1798         list_add_tail(&th->node, &threads->dead);
1799
1800         /*
1801          * We need to do the put here because if this is the last refcount,
1802          * then we will be touching the threads->dead head when removing the
1803          * thread.
1804          */
1805         thread__put(th);
1806
1807         if (lock)
1808                 up_write(&threads->lock);
1809 }
1810
1811 void machine__remove_thread(struct machine *machine, struct thread *th)
1812 {
1813         return __machine__remove_thread(machine, th, true);
1814 }
1815
1816 int machine__process_fork_event(struct machine *machine, union perf_event *event,
1817                                 struct perf_sample *sample)
1818 {
1819         struct thread *thread = machine__find_thread(machine,
1820                                                      event->fork.pid,
1821                                                      event->fork.tid);
1822         struct thread *parent = machine__findnew_thread(machine,
1823                                                         event->fork.ppid,
1824                                                         event->fork.ptid);
1825         bool do_maps_clone = true;
1826         int err = 0;
1827
1828         if (dump_trace)
1829                 perf_event__fprintf_task(event, stdout);
1830
1831         /*
1832          * There may be an existing thread that is not actually the parent,
1833          * either because we are processing events out of order, or because the
1834          * (fork) event that would have removed the thread was lost. Assume the
1835          * latter case and continue on as best we can.
1836          */
1837         if (parent->pid_ != (pid_t)event->fork.ppid) {
1838                 dump_printf("removing erroneous parent thread %d/%d\n",
1839                             parent->pid_, parent->tid);
1840                 machine__remove_thread(machine, parent);
1841                 thread__put(parent);
1842                 parent = machine__findnew_thread(machine, event->fork.ppid,
1843                                                  event->fork.ptid);
1844         }
1845
1846         /* if a thread currently exists for the thread id remove it */
1847         if (thread != NULL) {
1848                 machine__remove_thread(machine, thread);
1849                 thread__put(thread);
1850         }
1851
1852         thread = machine__findnew_thread(machine, event->fork.pid,
1853                                          event->fork.tid);
1854         /*
1855          * When synthesizing FORK events, we are trying to create thread
1856          * objects for the already running tasks on the machine.
1857          *
1858          * Normally, for a kernel FORK event, we want to clone the parent's
1859          * maps because that is what the kernel just did.
1860          *
1861          * But when synthesizing, this should not be done.  If we do, we end up
1862          * with overlapping maps as we process the sythesized MMAP2 events that
1863          * get delivered shortly thereafter.
1864          *
1865          * Use the FORK event misc flags in an internal way to signal this
1866          * situation, so we can elide the map clone when appropriate.
1867          */
1868         if (event->fork.header.misc & PERF_RECORD_MISC_FORK_EXEC)
1869                 do_maps_clone = false;
1870
1871         if (thread == NULL || parent == NULL ||
1872             thread__fork(thread, parent, sample->time, do_maps_clone) < 0) {
1873                 dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
1874                 err = -1;
1875         }
1876         thread__put(thread);
1877         thread__put(parent);
1878
1879         return err;
1880 }
1881
1882 int machine__process_exit_event(struct machine *machine, union perf_event *event,
1883                                 struct perf_sample *sample __maybe_unused)
1884 {
1885         struct thread *thread = machine__find_thread(machine,
1886                                                      event->fork.pid,
1887                                                      event->fork.tid);
1888
1889         if (dump_trace)
1890                 perf_event__fprintf_task(event, stdout);
1891
1892         if (thread != NULL) {
1893                 thread__exited(thread);
1894                 thread__put(thread);
1895         }
1896
1897         return 0;
1898 }
1899
1900 int machine__process_event(struct machine *machine, union perf_event *event,
1901                            struct perf_sample *sample)
1902 {
1903         int ret;
1904
1905         switch (event->header.type) {
1906         case PERF_RECORD_COMM:
1907                 ret = machine__process_comm_event(machine, event, sample); break;
1908         case PERF_RECORD_MMAP:
1909                 ret = machine__process_mmap_event(machine, event, sample); break;
1910         case PERF_RECORD_NAMESPACES:
1911                 ret = machine__process_namespaces_event(machine, event, sample); break;
1912         case PERF_RECORD_MMAP2:
1913                 ret = machine__process_mmap2_event(machine, event, sample); break;
1914         case PERF_RECORD_FORK:
1915                 ret = machine__process_fork_event(machine, event, sample); break;
1916         case PERF_RECORD_EXIT:
1917                 ret = machine__process_exit_event(machine, event, sample); break;
1918         case PERF_RECORD_LOST:
1919                 ret = machine__process_lost_event(machine, event, sample); break;
1920         case PERF_RECORD_AUX:
1921                 ret = machine__process_aux_event(machine, event); break;
1922         case PERF_RECORD_ITRACE_START:
1923                 ret = machine__process_itrace_start_event(machine, event); break;
1924         case PERF_RECORD_LOST_SAMPLES:
1925                 ret = machine__process_lost_samples_event(machine, event, sample); break;
1926         case PERF_RECORD_SWITCH:
1927         case PERF_RECORD_SWITCH_CPU_WIDE:
1928                 ret = machine__process_switch_event(machine, event); break;
1929         case PERF_RECORD_KSYMBOL:
1930                 ret = machine__process_ksymbol(machine, event, sample); break;
1931         case PERF_RECORD_BPF_EVENT:
1932                 ret = machine__process_bpf(machine, event, sample); break;
1933         default:
1934                 ret = -1;
1935                 break;
1936         }
1937
1938         return ret;
1939 }
1940
1941 static bool symbol__match_regex(struct symbol *sym, regex_t *regex)
1942 {
1943         if (!regexec(regex, sym->name, 0, NULL, 0))
1944                 return 1;
1945         return 0;
1946 }
1947
1948 static void ip__resolve_ams(struct thread *thread,
1949                             struct addr_map_symbol *ams,
1950                             u64 ip)
1951 {
1952         struct addr_location al;
1953
1954         memset(&al, 0, sizeof(al));
1955         /*
1956          * We cannot use the header.misc hint to determine whether a
1957          * branch stack address is user, kernel, guest, hypervisor.
1958          * Branches may straddle the kernel/user/hypervisor boundaries.
1959          * Thus, we have to try consecutively until we find a match
1960          * or else, the symbol is unknown
1961          */
1962         thread__find_cpumode_addr_location(thread, ip, &al);
1963
1964         ams->addr = ip;
1965         ams->al_addr = al.addr;
1966         ams->sym = al.sym;
1967         ams->map = al.map;
1968         ams->phys_addr = 0;
1969 }
1970
1971 static void ip__resolve_data(struct thread *thread,
1972                              u8 m, struct addr_map_symbol *ams,
1973                              u64 addr, u64 phys_addr)
1974 {
1975         struct addr_location al;
1976
1977         memset(&al, 0, sizeof(al));
1978
1979         thread__find_symbol(thread, m, addr, &al);
1980
1981         ams->addr = addr;
1982         ams->al_addr = al.addr;
1983         ams->sym = al.sym;
1984         ams->map = al.map;
1985         ams->phys_addr = phys_addr;
1986 }
1987
1988 struct mem_info *sample__resolve_mem(struct perf_sample *sample,
1989                                      struct addr_location *al)
1990 {
1991         struct mem_info *mi = mem_info__new();
1992
1993         if (!mi)
1994                 return NULL;
1995
1996         ip__resolve_ams(al->thread, &mi->iaddr, sample->ip);
1997         ip__resolve_data(al->thread, al->cpumode, &mi->daddr,
1998                          sample->addr, sample->phys_addr);
1999         mi->data_src.val = sample->data_src;
2000
2001         return mi;
2002 }
2003
2004 static char *callchain_srcline(struct map *map, struct symbol *sym, u64 ip)
2005 {
2006         char *srcline = NULL;
2007
2008         if (!map || callchain_param.key == CCKEY_FUNCTION)
2009                 return srcline;
2010
2011         srcline = srcline__tree_find(&map->dso->srclines, ip);
2012         if (!srcline) {
2013                 bool show_sym = false;
2014                 bool show_addr = callchain_param.key == CCKEY_ADDRESS;
2015
2016                 srcline = get_srcline(map->dso, map__rip_2objdump(map, ip),
2017                                       sym, show_sym, show_addr, ip);
2018                 srcline__tree_insert(&map->dso->srclines, ip, srcline);
2019         }
2020
2021         return srcline;
2022 }
2023
2024 struct iterations {
2025         int nr_loop_iter;
2026         u64 cycles;
2027 };
2028
2029 static int add_callchain_ip(struct thread *thread,
2030                             struct callchain_cursor *cursor,
2031                             struct symbol **parent,
2032                             struct addr_location *root_al,
2033                             u8 *cpumode,
2034                             u64 ip,
2035                             bool branch,
2036                             struct branch_flags *flags,
2037                             struct iterations *iter,
2038                             u64 branch_from)
2039 {
2040         struct addr_location al;
2041         int nr_loop_iter = 0;
2042         u64 iter_cycles = 0;
2043         const char *srcline = NULL;
2044
2045         al.filtered = 0;
2046         al.sym = NULL;
2047         if (!cpumode) {
2048                 thread__find_cpumode_addr_location(thread, ip, &al);
2049         } else {
2050                 if (ip >= PERF_CONTEXT_MAX) {
2051                         switch (ip) {
2052                         case PERF_CONTEXT_HV:
2053                                 *cpumode = PERF_RECORD_MISC_HYPERVISOR;
2054                                 break;
2055                         case PERF_CONTEXT_KERNEL:
2056                                 *cpumode = PERF_RECORD_MISC_KERNEL;
2057                                 break;
2058                         case PERF_CONTEXT_USER:
2059                                 *cpumode = PERF_RECORD_MISC_USER;
2060                                 break;
2061                         default:
2062                                 pr_debug("invalid callchain context: "
2063                                          "%"PRId64"\n", (s64) ip);
2064                                 /*
2065                                  * It seems the callchain is corrupted.
2066                                  * Discard all.
2067                                  */
2068                                 callchain_cursor_reset(cursor);
2069                                 return 1;
2070                         }
2071                         return 0;
2072                 }
2073                 thread__find_symbol(thread, *cpumode, ip, &al);
2074         }
2075
2076         if (al.sym != NULL) {
2077                 if (perf_hpp_list.parent && !*parent &&
2078                     symbol__match_regex(al.sym, &parent_regex))
2079                         *parent = al.sym;
2080                 else if (have_ignore_callees && root_al &&
2081                   symbol__match_regex(al.sym, &ignore_callees_regex)) {
2082                         /* Treat this symbol as the root,
2083                            forgetting its callees. */
2084                         *root_al = al;
2085                         callchain_cursor_reset(cursor);
2086                 }
2087         }
2088
2089         if (symbol_conf.hide_unresolved && al.sym == NULL)
2090                 return 0;
2091
2092         if (iter) {
2093                 nr_loop_iter = iter->nr_loop_iter;
2094                 iter_cycles = iter->cycles;
2095         }
2096
2097         srcline = callchain_srcline(al.map, al.sym, al.addr);
2098         return callchain_cursor_append(cursor, ip, al.map, al.sym,
2099                                        branch, flags, nr_loop_iter,
2100                                        iter_cycles, branch_from, srcline);
2101 }
2102
2103 struct branch_info *sample__resolve_bstack(struct perf_sample *sample,
2104                                            struct addr_location *al)
2105 {
2106         unsigned int i;
2107         const struct branch_stack *bs = sample->branch_stack;
2108         struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info));
2109
2110         if (!bi)
2111                 return NULL;
2112
2113         for (i = 0; i < bs->nr; i++) {
2114                 ip__resolve_ams(al->thread, &bi[i].to, bs->entries[i].to);
2115                 ip__resolve_ams(al->thread, &bi[i].from, bs->entries[i].from);
2116                 bi[i].flags = bs->entries[i].flags;
2117         }
2118         return bi;
2119 }
2120
2121 static void save_iterations(struct iterations *iter,
2122                             struct branch_entry *be, int nr)
2123 {
2124         int i;
2125
2126         iter->nr_loop_iter++;
2127         iter->cycles = 0;
2128
2129         for (i = 0; i < nr; i++)
2130                 iter->cycles += be[i].flags.cycles;
2131 }
2132
2133 #define CHASHSZ 127
2134 #define CHASHBITS 7
2135 #define NO_ENTRY 0xff
2136
2137 #define PERF_MAX_BRANCH_DEPTH 127
2138
2139 /* Remove loops. */
2140 static int remove_loops(struct branch_entry *l, int nr,
2141                         struct iterations *iter)
2142 {
2143         int i, j, off;
2144         unsigned char chash[CHASHSZ];
2145
2146         memset(chash, NO_ENTRY, sizeof(chash));
2147
2148         BUG_ON(PERF_MAX_BRANCH_DEPTH > 255);
2149
2150         for (i = 0; i < nr; i++) {
2151                 int h = hash_64(l[i].from, CHASHBITS) % CHASHSZ;
2152
2153                 /* no collision handling for now */
2154                 if (chash[h] == NO_ENTRY) {
2155                         chash[h] = i;
2156                 } else if (l[chash[h]].from == l[i].from) {
2157                         bool is_loop = true;
2158                         /* check if it is a real loop */
2159                         off = 0;
2160                         for (j = chash[h]; j < i && i + off < nr; j++, off++)
2161                                 if (l[j].from != l[i + off].from) {
2162                                         is_loop = false;
2163                                         break;
2164                                 }
2165                         if (is_loop) {
2166                                 j = nr - (i + off);
2167                                 if (j > 0) {
2168                                         save_iterations(iter + i + off,
2169                                                 l + i, off);
2170
2171                                         memmove(iter + i, iter + i + off,
2172                                                 j * sizeof(*iter));
2173
2174                                         memmove(l + i, l + i + off,
2175                                                 j * sizeof(*l));
2176                                 }
2177
2178                                 nr -= off;
2179                         }
2180                 }
2181         }
2182         return nr;
2183 }
2184
2185 /*
2186  * Recolve LBR callstack chain sample
2187  * Return:
2188  * 1 on success get LBR callchain information
2189  * 0 no available LBR callchain information, should try fp
2190  * negative error code on other errors.
2191  */
2192 static int resolve_lbr_callchain_sample(struct thread *thread,
2193                                         struct callchain_cursor *cursor,
2194                                         struct perf_sample *sample,
2195                                         struct symbol **parent,
2196                                         struct addr_location *root_al,
2197                                         int max_stack)
2198 {
2199         struct ip_callchain *chain = sample->callchain;
2200         int chain_nr = min(max_stack, (int)chain->nr), i;
2201         u8 cpumode = PERF_RECORD_MISC_USER;
2202         u64 ip, branch_from = 0;
2203
2204         for (i = 0; i < chain_nr; i++) {
2205                 if (chain->ips[i] == PERF_CONTEXT_USER)
2206                         break;
2207         }
2208
2209         /* LBR only affects the user callchain */
2210         if (i != chain_nr) {
2211                 struct branch_stack *lbr_stack = sample->branch_stack;
2212                 int lbr_nr = lbr_stack->nr, j, k;
2213                 bool branch;
2214                 struct branch_flags *flags;
2215                 /*
2216                  * LBR callstack can only get user call chain.
2217                  * The mix_chain_nr is kernel call chain
2218                  * number plus LBR user call chain number.
2219                  * i is kernel call chain number,
2220                  * 1 is PERF_CONTEXT_USER,
2221                  * lbr_nr + 1 is the user call chain number.
2222                  * For details, please refer to the comments
2223                  * in callchain__printf
2224                  */
2225                 int mix_chain_nr = i + 1 + lbr_nr + 1;
2226
2227                 for (j = 0; j < mix_chain_nr; j++) {
2228                         int err;
2229                         branch = false;
2230                         flags = NULL;
2231
2232                         if (callchain_param.order == ORDER_CALLEE) {
2233                                 if (j < i + 1)
2234                                         ip = chain->ips[j];
2235                                 else if (j > i + 1) {
2236                                         k = j - i - 2;
2237                                         ip = lbr_stack->entries[k].from;
2238                                         branch = true;
2239                                         flags = &lbr_stack->entries[k].flags;
2240                                 } else {
2241                                         ip = lbr_stack->entries[0].to;
2242                                         branch = true;
2243                                         flags = &lbr_stack->entries[0].flags;
2244                                         branch_from =
2245                                                 lbr_stack->entries[0].from;
2246                                 }
2247                         } else {
2248                                 if (j < lbr_nr) {
2249                                         k = lbr_nr - j - 1;
2250                                         ip = lbr_stack->entries[k].from;
2251                                         branch = true;
2252                                         flags = &lbr_stack->entries[k].flags;
2253                                 }
2254                                 else if (j > lbr_nr)
2255                                         ip = chain->ips[i + 1 - (j - lbr_nr)];
2256                                 else {
2257                                         ip = lbr_stack->entries[0].to;
2258                                         branch = true;
2259                                         flags = &lbr_stack->entries[0].flags;
2260                                         branch_from =
2261                                                 lbr_stack->entries[0].from;
2262                                 }
2263                         }
2264
2265                         err = add_callchain_ip(thread, cursor, parent,
2266                                                root_al, &cpumode, ip,
2267                                                branch, flags, NULL,
2268                                                branch_from);
2269                         if (err)
2270                                 return (err < 0) ? err : 0;
2271                 }
2272                 return 1;
2273         }
2274
2275         return 0;
2276 }
2277
2278 static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread,
2279                              struct callchain_cursor *cursor,
2280                              struct symbol **parent,
2281                              struct addr_location *root_al,
2282                              u8 *cpumode, int ent)
2283 {
2284         int err = 0;
2285
2286         while (--ent >= 0) {
2287                 u64 ip = chain->ips[ent];
2288
2289                 if (ip >= PERF_CONTEXT_MAX) {
2290                         err = add_callchain_ip(thread, cursor, parent,
2291                                                root_al, cpumode, ip,
2292                                                false, NULL, NULL, 0);
2293                         break;
2294                 }
2295         }
2296         return err;
2297 }
2298
2299 static int thread__resolve_callchain_sample(struct thread *thread,
2300                                             struct callchain_cursor *cursor,
2301                                             struct evsel *evsel,
2302                                             struct perf_sample *sample,
2303                                             struct symbol **parent,
2304                                             struct addr_location *root_al,
2305                                             int max_stack)
2306 {
2307         struct branch_stack *branch = sample->branch_stack;
2308         struct ip_callchain *chain = sample->callchain;
2309         int chain_nr = 0;
2310         u8 cpumode = PERF_RECORD_MISC_USER;
2311         int i, j, err, nr_entries;
2312         int skip_idx = -1;
2313         int first_call = 0;
2314
2315         if (chain)
2316                 chain_nr = chain->nr;
2317
2318         if (perf_evsel__has_branch_callstack(evsel)) {
2319                 err = resolve_lbr_callchain_sample(thread, cursor, sample, parent,
2320                                                    root_al, max_stack);
2321                 if (err)
2322                         return (err < 0) ? err : 0;
2323         }
2324
2325         /*
2326          * Based on DWARF debug information, some architectures skip
2327          * a callchain entry saved by the kernel.
2328          */
2329         skip_idx = arch_skip_callchain_idx(thread, chain);
2330
2331         /*
2332          * Add branches to call stack for easier browsing. This gives
2333          * more context for a sample than just the callers.
2334          *
2335          * This uses individual histograms of paths compared to the
2336          * aggregated histograms the normal LBR mode uses.
2337          *
2338          * Limitations for now:
2339          * - No extra filters
2340          * - No annotations (should annotate somehow)
2341          */
2342
2343         if (branch && callchain_param.branch_callstack) {
2344                 int nr = min(max_stack, (int)branch->nr);
2345                 struct branch_entry be[nr];
2346                 struct iterations iter[nr];
2347
2348                 if (branch->nr > PERF_MAX_BRANCH_DEPTH) {
2349                         pr_warning("corrupted branch chain. skipping...\n");
2350                         goto check_calls;
2351                 }
2352
2353                 for (i = 0; i < nr; i++) {
2354                         if (callchain_param.order == ORDER_CALLEE) {
2355                                 be[i] = branch->entries[i];
2356
2357                                 if (chain == NULL)
2358                                         continue;
2359
2360                                 /*
2361                                  * Check for overlap into the callchain.
2362                                  * The return address is one off compared to
2363                                  * the branch entry. To adjust for this
2364                                  * assume the calling instruction is not longer
2365                                  * than 8 bytes.
2366                                  */
2367                                 if (i == skip_idx ||
2368                                     chain->ips[first_call] >= PERF_CONTEXT_MAX)
2369                                         first_call++;
2370                                 else if (be[i].from < chain->ips[first_call] &&
2371                                     be[i].from >= chain->ips[first_call] - 8)
2372                                         first_call++;
2373                         } else
2374                                 be[i] = branch->entries[branch->nr - i - 1];
2375                 }
2376
2377                 memset(iter, 0, sizeof(struct iterations) * nr);
2378                 nr = remove_loops(be, nr, iter);
2379
2380                 for (i = 0; i < nr; i++) {
2381                         err = add_callchain_ip(thread, cursor, parent,
2382                                                root_al,
2383                                                NULL, be[i].to,
2384                                                true, &be[i].flags,
2385                                                NULL, be[i].from);
2386
2387                         if (!err)
2388                                 err = add_callchain_ip(thread, cursor, parent, root_al,
2389                                                        NULL, be[i].from,
2390                                                        true, &be[i].flags,
2391                                                        &iter[i], 0);
2392                         if (err == -EINVAL)
2393                                 break;
2394                         if (err)
2395                                 return err;
2396                 }
2397
2398                 if (chain_nr == 0)
2399                         return 0;
2400
2401                 chain_nr -= nr;
2402         }
2403
2404 check_calls:
2405         if (callchain_param.order != ORDER_CALLEE) {
2406                 err = find_prev_cpumode(chain, thread, cursor, parent, root_al,
2407                                         &cpumode, chain->nr - first_call);
2408                 if (err)
2409                         return (err < 0) ? err : 0;
2410         }
2411         for (i = first_call, nr_entries = 0;
2412              i < chain_nr && nr_entries < max_stack; i++) {
2413                 u64 ip;
2414
2415                 if (callchain_param.order == ORDER_CALLEE)
2416                         j = i;
2417                 else
2418                         j = chain->nr - i - 1;
2419
2420 #ifdef HAVE_SKIP_CALLCHAIN_IDX
2421                 if (j == skip_idx)
2422                         continue;
2423 #endif
2424                 ip = chain->ips[j];
2425                 if (ip < PERF_CONTEXT_MAX)
2426                        ++nr_entries;
2427                 else if (callchain_param.order != ORDER_CALLEE) {
2428                         err = find_prev_cpumode(chain, thread, cursor, parent,
2429                                                 root_al, &cpumode, j);
2430                         if (err)
2431                                 return (err < 0) ? err : 0;
2432                         continue;
2433                 }
2434
2435                 err = add_callchain_ip(thread, cursor, parent,
2436                                        root_al, &cpumode, ip,
2437                                        false, NULL, NULL, 0);
2438
2439                 if (err)
2440                         return (err < 0) ? err : 0;
2441         }
2442
2443         return 0;
2444 }
2445
2446 static int append_inlines(struct callchain_cursor *cursor,
2447                           struct map *map, struct symbol *sym, u64 ip)
2448 {
2449         struct inline_node *inline_node;
2450         struct inline_list *ilist;
2451         u64 addr;
2452         int ret = 1;
2453
2454         if (!symbol_conf.inline_name || !map || !sym)
2455                 return ret;
2456
2457         addr = map__map_ip(map, ip);
2458         addr = map__rip_2objdump(map, addr);
2459
2460         inline_node = inlines__tree_find(&map->dso->inlined_nodes, addr);
2461         if (!inline_node) {
2462                 inline_node = dso__parse_addr_inlines(map->dso, addr, sym);
2463                 if (!inline_node)
2464                         return ret;
2465                 inlines__tree_insert(&map->dso->inlined_nodes, inline_node);
2466         }
2467
2468         list_for_each_entry(ilist, &inline_node->val, list) {
2469                 ret = callchain_cursor_append(cursor, ip, map,
2470                                               ilist->symbol, false,
2471                                               NULL, 0, 0, 0, ilist->srcline);
2472
2473                 if (ret != 0)
2474                         return ret;
2475         }
2476
2477         return ret;
2478 }
2479
2480 static int unwind_entry(struct unwind_entry *entry, void *arg)
2481 {
2482         struct callchain_cursor *cursor = arg;
2483         const char *srcline = NULL;
2484         u64 addr = entry->ip;
2485
2486         if (symbol_conf.hide_unresolved && entry->sym == NULL)
2487                 return 0;
2488
2489         if (append_inlines(cursor, entry->map, entry->sym, entry->ip) == 0)
2490                 return 0;
2491
2492         /*
2493          * Convert entry->ip from a virtual address to an offset in
2494          * its corresponding binary.
2495          */
2496         if (entry->map)
2497                 addr = map__map_ip(entry->map, entry->ip);
2498
2499         srcline = callchain_srcline(entry->map, entry->sym, addr);
2500         return callchain_cursor_append(cursor, entry->ip,
2501                                        entry->map, entry->sym,
2502                                        false, NULL, 0, 0, 0, srcline);
2503 }
2504
2505 static int thread__resolve_callchain_unwind(struct thread *thread,
2506                                             struct callchain_cursor *cursor,
2507                                             struct evsel *evsel,
2508                                             struct perf_sample *sample,
2509                                             int max_stack)
2510 {
2511         /* Can we do dwarf post unwind? */
2512         if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) &&
2513               (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER)))
2514                 return 0;
2515
2516         /* Bail out if nothing was captured. */
2517         if ((!sample->user_regs.regs) ||
2518             (!sample->user_stack.size))
2519                 return 0;
2520
2521         return unwind__get_entries(unwind_entry, cursor,
2522                                    thread, sample, max_stack);
2523 }
2524
2525 int thread__resolve_callchain(struct thread *thread,
2526                               struct callchain_cursor *cursor,
2527                               struct evsel *evsel,
2528                               struct perf_sample *sample,
2529                               struct symbol **parent,
2530                               struct addr_location *root_al,
2531                               int max_stack)
2532 {
2533         int ret = 0;
2534
2535         callchain_cursor_reset(cursor);
2536
2537         if (callchain_param.order == ORDER_CALLEE) {
2538                 ret = thread__resolve_callchain_sample(thread, cursor,
2539                                                        evsel, sample,
2540                                                        parent, root_al,
2541                                                        max_stack);
2542                 if (ret)
2543                         return ret;
2544                 ret = thread__resolve_callchain_unwind(thread, cursor,
2545                                                        evsel, sample,
2546                                                        max_stack);
2547         } else {
2548                 ret = thread__resolve_callchain_unwind(thread, cursor,
2549                                                        evsel, sample,
2550                                                        max_stack);
2551                 if (ret)
2552                         return ret;
2553                 ret = thread__resolve_callchain_sample(thread, cursor,
2554                                                        evsel, sample,
2555                                                        parent, root_al,
2556                                                        max_stack);
2557         }
2558
2559         return ret;
2560 }
2561
2562 int machine__for_each_thread(struct machine *machine,
2563                              int (*fn)(struct thread *thread, void *p),
2564                              void *priv)
2565 {
2566         struct threads *threads;
2567         struct rb_node *nd;
2568         struct thread *thread;
2569         int rc = 0;
2570         int i;
2571
2572         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
2573                 threads = &machine->threads[i];
2574                 for (nd = rb_first_cached(&threads->entries); nd;
2575                      nd = rb_next(nd)) {
2576                         thread = rb_entry(nd, struct thread, rb_node);
2577                         rc = fn(thread, priv);
2578                         if (rc != 0)
2579                                 return rc;
2580                 }
2581
2582                 list_for_each_entry(thread, &threads->dead, node) {
2583                         rc = fn(thread, priv);
2584                         if (rc != 0)
2585                                 return rc;
2586                 }
2587         }
2588         return rc;
2589 }
2590
2591 int machines__for_each_thread(struct machines *machines,
2592                               int (*fn)(struct thread *thread, void *p),
2593                               void *priv)
2594 {
2595         struct rb_node *nd;
2596         int rc = 0;
2597
2598         rc = machine__for_each_thread(&machines->host, fn, priv);
2599         if (rc != 0)
2600                 return rc;
2601
2602         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
2603                 struct machine *machine = rb_entry(nd, struct machine, rb_node);
2604
2605                 rc = machine__for_each_thread(machine, fn, priv);
2606                 if (rc != 0)
2607                         return rc;
2608         }
2609         return rc;
2610 }
2611
2612 int __machine__synthesize_threads(struct machine *machine, struct perf_tool *tool,
2613                                   struct target *target, struct perf_thread_map *threads,
2614                                   perf_event__handler_t process, bool data_mmap,
2615                                   unsigned int nr_threads_synthesize)
2616 {
2617         if (target__has_task(target))
2618                 return perf_event__synthesize_thread_map(tool, threads, process, machine, data_mmap);
2619         else if (target__has_cpu(target))
2620                 return perf_event__synthesize_threads(tool, process,
2621                                                       machine, data_mmap,
2622                                                       nr_threads_synthesize);
2623         /* command specified */
2624         return 0;
2625 }
2626
2627 pid_t machine__get_current_tid(struct machine *machine, int cpu)
2628 {
2629         int nr_cpus = min(machine->env->nr_cpus_online, MAX_NR_CPUS);
2630
2631         if (cpu < 0 || cpu >= nr_cpus || !machine->current_tid)
2632                 return -1;
2633
2634         return machine->current_tid[cpu];
2635 }
2636
2637 int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid,
2638                              pid_t tid)
2639 {
2640         struct thread *thread;
2641         int nr_cpus = min(machine->env->nr_cpus_online, MAX_NR_CPUS);
2642
2643         if (cpu < 0)
2644                 return -EINVAL;
2645
2646         if (!machine->current_tid) {
2647                 int i;
2648
2649                 machine->current_tid = calloc(nr_cpus, sizeof(pid_t));
2650                 if (!machine->current_tid)
2651                         return -ENOMEM;
2652                 for (i = 0; i < nr_cpus; i++)
2653                         machine->current_tid[i] = -1;
2654         }
2655
2656         if (cpu >= nr_cpus) {
2657                 pr_err("Requested CPU %d too large. ", cpu);
2658                 pr_err("Consider raising MAX_NR_CPUS\n");
2659                 return -EINVAL;
2660         }
2661
2662         machine->current_tid[cpu] = tid;
2663
2664         thread = machine__findnew_thread(machine, pid, tid);
2665         if (!thread)
2666                 return -ENOMEM;
2667
2668         thread->cpu = cpu;
2669         thread__put(thread);
2670
2671         return 0;
2672 }
2673
2674 /*
2675  * Compares the raw arch string. N.B. see instead perf_env__arch() if a
2676  * normalized arch is needed.
2677  */
2678 bool machine__is(struct machine *machine, const char *arch)
2679 {
2680         return machine && !strcmp(perf_env__raw_arch(machine->env), arch);
2681 }
2682
2683 int machine__nr_cpus_avail(struct machine *machine)
2684 {
2685         return machine ? perf_env__nr_cpus_avail(machine->env) : 0;
2686 }
2687
2688 int machine__get_kernel_start(struct machine *machine)
2689 {
2690         struct map *map = machine__kernel_map(machine);
2691         int err = 0;
2692
2693         /*
2694          * The only addresses above 2^63 are kernel addresses of a 64-bit
2695          * kernel.  Note that addresses are unsigned so that on a 32-bit system
2696          * all addresses including kernel addresses are less than 2^32.  In
2697          * that case (32-bit system), if the kernel mapping is unknown, all
2698          * addresses will be assumed to be in user space - see
2699          * machine__kernel_ip().
2700          */
2701         machine->kernel_start = 1ULL << 63;
2702         if (map) {
2703                 err = map__load(map);
2704                 /*
2705                  * On x86_64, PTI entry trampolines are less than the
2706                  * start of kernel text, but still above 2^63. So leave
2707                  * kernel_start = 1ULL << 63 for x86_64.
2708                  */
2709                 if (!err && !machine__is(machine, "x86_64"))
2710                         machine->kernel_start = map->start;
2711         }
2712         return err;
2713 }
2714
2715 u8 machine__addr_cpumode(struct machine *machine, u8 cpumode, u64 addr)
2716 {
2717         u8 addr_cpumode = cpumode;
2718         bool kernel_ip;
2719
2720         if (!machine->single_address_space)
2721                 goto out;
2722
2723         kernel_ip = machine__kernel_ip(machine, addr);
2724         switch (cpumode) {
2725         case PERF_RECORD_MISC_KERNEL:
2726         case PERF_RECORD_MISC_USER:
2727                 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_KERNEL :
2728                                            PERF_RECORD_MISC_USER;
2729                 break;
2730         case PERF_RECORD_MISC_GUEST_KERNEL:
2731         case PERF_RECORD_MISC_GUEST_USER:
2732                 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_GUEST_KERNEL :
2733                                            PERF_RECORD_MISC_GUEST_USER;
2734                 break;
2735         default:
2736                 break;
2737         }
2738 out:
2739         return addr_cpumode;
2740 }
2741
2742 struct dso *machine__findnew_dso(struct machine *machine, const char *filename)
2743 {
2744         return dsos__findnew(&machine->dsos, filename);
2745 }
2746
2747 char *machine__resolve_kernel_addr(void *vmachine, unsigned long long *addrp, char **modp)
2748 {
2749         struct machine *machine = vmachine;
2750         struct map *map;
2751         struct symbol *sym = machine__find_kernel_symbol(machine, *addrp, &map);
2752
2753         if (sym == NULL)
2754                 return NULL;
2755
2756         *modp = __map__is_kmodule(map) ? (char *)map->dso->short_name : NULL;
2757         *addrp = map->unmap_ip(map, sym->start);
2758         return sym->name;
2759 }