tools/perf/util/machine.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <dirent.h>
   3 #include <errno.h>
   4 #include <inttypes.h>
   5 #include <regex.h>
   6 #include <stdlib.h>
   7 #include "callchain.h"
   8 #include "debug.h"
   9 #include "dso.h"
  10 #include "env.h"
  11 #include "event.h"
  12 #include "evsel.h"
  13 #include "hist.h"
  14 #include "machine.h"
  15 #include "map.h"
  16 #include "map_symbol.h"
  17 #include "branch.h"
  18 #include "mem-events.h"
  19 #include "path.h"
  20 #include "srcline.h"
  21 #include "symbol.h"
  22 #include "sort.h"
  23 #include "strlist.h"
  24 #include "target.h"
  25 #include "thread.h"
  26 #include "util.h"
  27 #include "vdso.h"
  28 #include <stdbool.h>
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include "unwind.h"
  33 #include "linux/hash.h"
  34 #include "asm/bug.h"
  35 #include "bpf-event.h"
  36 #include <internal/lib.h> // page_size
  37 #include "cgroup.h"
  38 #include "arm64-frame-pointer-unwind-support.h"
  39
  40 #include <linux/ctype.h>
  41 #include <symbol/kallsyms.h>
  42 #include <linux/mman.h>
  43 #include <linux/string.h>
  44 #include <linux/zalloc.h>
  45
  46 static void __machine__remove_thread(struct machine *machine, struct thread *th, bool lock);
  47
  48 static struct dso *machine__kernel_dso(struct machine *machine)
  49 {
  50         return machine->vmlinux_map->dso;
  51 }
  52
  53 static void dsos__init(struct dsos *dsos)
  54 {
  55         INIT_LIST_HEAD(&dsos->head);
  56         dsos->root = RB_ROOT;
  57         init_rwsem(&dsos->lock);
  58 }
  59
  60 static void machine__threads_init(struct machine *machine)
  61 {
  62         int i;
  63
  64         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
  65                 struct threads *threads = &machine->threads[i];
  66                 threads->entries = RB_ROOT_CACHED;
  67                 init_rwsem(&threads->lock);
  68                 threads->nr = 0;
  69                 INIT_LIST_HEAD(&threads->dead);
  70                 threads->last_match = NULL;
  71         }
  72 }
  73
  74 static int machine__set_mmap_name(struct machine *machine)
  75 {
  76         if (machine__is_host(machine))
  77                 machine->mmap_name = strdup("[kernel.kallsyms]");
  78         else if (machine__is_default_guest(machine))
  79                 machine->mmap_name = strdup("[guest.kernel.kallsyms]");
  80         else if (asprintf(&machine->mmap_name, "[guest.kernel.kallsyms.%d]",
  81                           machine->pid) < 0)
  82                 machine->mmap_name = NULL;
  83
  84         return machine->mmap_name ? 0 : -ENOMEM;
  85 }
  86
  87 int machine__init(struct machine *machine, const char *root_dir, pid_t pid)
  88 {
  89         int err = -ENOMEM;
  90
  91         memset(machine, 0, sizeof(*machine));
  92         machine->kmaps = maps__new(machine);
  93         if (machine->kmaps == NULL)
  94                 return -ENOMEM;
  95
  96         RB_CLEAR_NODE(&machine->rb_node);
  97         dsos__init(&machine->dsos);
  98
  99         machine__threads_init(machine);
 100
 101         machine->vdso_info = NULL;
 102         machine->env = NULL;
 103
 104         machine->pid = pid;
 105
 106         machine->id_hdr_size = 0;
 107         machine->kptr_restrict_warned = false;
 108         machine->comm_exec = false;
 109         machine->kernel_start = 0;
 110         machine->vmlinux_map = NULL;
 111
 112         machine->root_dir = strdup(root_dir);
 113         if (machine->root_dir == NULL)
 114                 goto out;
 115
 116         if (machine__set_mmap_name(machine))
 117                 goto out;
 118
 119         if (pid != HOST_KERNEL_ID) {
 120                 struct thread *thread = machine__findnew_thread(machine, -1,
 121                                                                 pid);
 122                 char comm[64];
 123
 124                 if (thread == NULL)
 125                         goto out;
 126
 127                 snprintf(comm, sizeof(comm), "[guest/%d]", pid);
 128                 thread__set_comm(thread, comm, 0);
 129                 thread__put(thread);
 130         }
 131
 132         machine->current_tid = NULL;
 133         err = 0;
 134
 135 out:
 136         if (err) {
 137                 zfree(&machine->kmaps);
 138                 zfree(&machine->root_dir);
 139                 zfree(&machine->mmap_name);
 140         }
 141         return 0;
 142 }
 143
 144 struct machine *machine__new_host(void)
 145 {
 146         struct machine *machine = malloc(sizeof(*machine));
 147
 148         if (machine != NULL) {
 149                 machine__init(machine, "", HOST_KERNEL_ID);
 150
 151                 if (machine__create_kernel_maps(machine) < 0)
 152                         goto out_delete;
 153         }
 154
 155         return machine;
 156 out_delete:
 157         free(machine);
 158         return NULL;
 159 }
 160
 161 struct machine *machine__new_kallsyms(void)
 162 {
 163         struct machine *machine = machine__new_host();
 164         /*
 165          * FIXME:
 166          * 1) We should switch to machine__load_kallsyms(), i.e. not explicitly
 167          *    ask for not using the kcore parsing code, once this one is fixed
 168          *    to create a map per module.
 169          */
 170         if (machine && machine__load_kallsyms(machine, "/proc/kallsyms") <= 0) {
 171                 machine__delete(machine);
 172                 machine = NULL;
 173         }
 174
 175         return machine;
 176 }
 177
 178 static void dsos__purge(struct dsos *dsos)
 179 {
 180         struct dso *pos, *n;
 181
 182         down_write(&dsos->lock);
 183
 184         list_for_each_entry_safe(pos, n, &dsos->head, node) {
 185                 RB_CLEAR_NODE(&pos->rb_node);
 186                 pos->root = NULL;
 187                 list_del_init(&pos->node);
 188                 dso__put(pos);
 189         }
 190
 191         up_write(&dsos->lock);
 192 }
 193
 194 static void dsos__exit(struct dsos *dsos)
 195 {
 196         dsos__purge(dsos);
 197         exit_rwsem(&dsos->lock);
 198 }
 199
 200 void machine__delete_threads(struct machine *machine)
 201 {
 202         struct rb_node *nd;
 203         int i;
 204
 205         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
 206                 struct threads *threads = &machine->threads[i];
 207                 down_write(&threads->lock);
 208                 nd = rb_first_cached(&threads->entries);
 209                 while (nd) {
 210                         struct thread *t = rb_entry(nd, struct thread, rb_node);
 211
 212                         nd = rb_next(nd);
 213                         __machine__remove_thread(machine, t, false);
 214                 }
 215                 up_write(&threads->lock);
 216         }
 217 }
 218
 219 void machine__exit(struct machine *machine)
 220 {
 221         int i;
 222
 223         if (machine == NULL)
 224                 return;
 225
 226         machine__destroy_kernel_maps(machine);
 227         maps__delete(machine->kmaps);
 228         dsos__exit(&machine->dsos);
 229         machine__exit_vdso(machine);
 230         zfree(&machine->root_dir);
 231         zfree(&machine->mmap_name);
 232         zfree(&machine->current_tid);
 233
 234         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
 235                 struct threads *threads = &machine->threads[i];
 236                 struct thread *thread, *n;
 237                 /*
 238                  * Forget about the dead, at this point whatever threads were
 239                  * left in the dead lists better have a reference count taken
 240                  * by who is using them, and then, when they drop those references
 241                  * and it finally hits zero, thread__put() will check and see that
 242                  * its not in the dead threads list and will not try to remove it
 243                  * from there, just calling thread__delete() straight away.
 244                  */
 245                 list_for_each_entry_safe(thread, n, &threads->dead, node)
 246                         list_del_init(&thread->node);
 247
 248                 exit_rwsem(&threads->lock);
 249         }
 250 }
 251
 252 void machine__delete(struct machine *machine)
 253 {
 254         if (machine) {
 255                 machine__exit(machine);
 256                 free(machine);
 257         }
 258 }
 259
 260 void machines__init(struct machines *machines)
 261 {
 262         machine__init(&machines->host, "", HOST_KERNEL_ID);
 263         machines->guests = RB_ROOT_CACHED;
 264 }
 265
 266 void machines__exit(struct machines *machines)
 267 {
 268         machine__exit(&machines->host);
 269         /* XXX exit guest */
 270 }
 271
 272 struct machine *machines__add(struct machines *machines, pid_t pid,
 273                               const char *root_dir)
 274 {
 275         struct rb_node **p = &machines->guests.rb_root.rb_node;
 276         struct rb_node *parent = NULL;
 277         struct machine *pos, *machine = malloc(sizeof(*machine));
 278         bool leftmost = true;
 279
 280         if (machine == NULL)
 281                 return NULL;
 282
 283         if (machine__init(machine, root_dir, pid) != 0) {
 284                 free(machine);
 285                 return NULL;
 286         }
 287
 288         while (*p != NULL) {
 289                 parent = *p;
 290                 pos = rb_entry(parent, struct machine, rb_node);
 291                 if (pid < pos->pid)
 292                         p = &(*p)->rb_left;
 293                 else {
 294                         p = &(*p)->rb_right;
 295                         leftmost = false;
 296                 }
 297         }
 298
 299         rb_link_node(&machine->rb_node, parent, p);
 300         rb_insert_color_cached(&machine->rb_node, &machines->guests, leftmost);
 301
 302         machine->machines = machines;
 303
 304         return machine;
 305 }
 306
 307 void machines__set_comm_exec(struct machines *machines, bool comm_exec)
 308 {
 309         struct rb_node *nd;
 310
 311         machines->host.comm_exec = comm_exec;
 312
 313         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 314                 struct machine *machine = rb_entry(nd, struct machine, rb_node);
 315
 316                 machine->comm_exec = comm_exec;
 317         }
 318 }
 319
 320 struct machine *machines__find(struct machines *machines, pid_t pid)
 321 {
 322         struct rb_node **p = &machines->guests.rb_root.rb_node;
 323         struct rb_node *parent = NULL;
 324         struct machine *machine;
 325         struct machine *default_machine = NULL;
 326
 327         if (pid == HOST_KERNEL_ID)
 328                 return &machines->host;
 329
 330         while (*p != NULL) {
 331                 parent = *p;
 332                 machine = rb_entry(parent, struct machine, rb_node);
 333                 if (pid < machine->pid)
 334                         p = &(*p)->rb_left;
 335                 else if (pid > machine->pid)
 336                         p = &(*p)->rb_right;
 337                 else
 338                         return machine;
 339                 if (!machine->pid)
 340                         default_machine = machine;
 341         }
 342
 343         return default_machine;
 344 }
 345
 346 struct machine *machines__findnew(struct machines *machines, pid_t pid)
 347 {
 348         char path[PATH_MAX];
 349         const char *root_dir = "";
 350         struct machine *machine = machines__find(machines, pid);
 351
 352         if (machine && (machine->pid == pid))
 353                 goto out;
 354
 355         if ((pid != HOST_KERNEL_ID) &&
 356             (pid != DEFAULT_GUEST_KERNEL_ID) &&
 357             (symbol_conf.guestmount)) {
 358                 sprintf(path, "%s/%d", symbol_conf.guestmount, pid);
 359                 if (access(path, R_OK)) {
 360                         static struct strlist *seen;
 361
 362                         if (!seen)
 363                                 seen = strlist__new(NULL, NULL);
 364
 365                         if (!strlist__has_entry(seen, path)) {
 366                                 pr_err("Can't access file %s\n", path);
 367                                 strlist__add(seen, path);
 368                         }
 369                         machine = NULL;
 370                         goto out;
 371                 }
 372                 root_dir = path;
 373         }
 374
 375         machine = machines__add(machines, pid, root_dir);
 376 out:
 377         return machine;
 378 }
 379
 380 struct machine *machines__find_guest(struct machines *machines, pid_t pid)
 381 {
 382         struct machine *machine = machines__find(machines, pid);
 383
 384         if (!machine)
 385                 machine = machines__findnew(machines, DEFAULT_GUEST_KERNEL_ID);
 386         return machine;
 387 }
 388
 389 void machines__process_guests(struct machines *machines,
 390                               machine__process_t process, void *data)
 391 {
 392         struct rb_node *nd;
 393
 394         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 395                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
 396                 process(pos, data);
 397         }
 398 }
 399
 400 void machines__set_id_hdr_size(struct machines *machines, u16 id_hdr_size)
 401 {
 402         struct rb_node *node;
 403         struct machine *machine;
 404
 405         machines->host.id_hdr_size = id_hdr_size;
 406
 407         for (node = rb_first_cached(&machines->guests); node;
 408              node = rb_next(node)) {
 409                 machine = rb_entry(node, struct machine, rb_node);
 410                 machine->id_hdr_size = id_hdr_size;
 411         }
 412
 413         return;
 414 }
 415
 416 static void machine__update_thread_pid(struct machine *machine,
 417                                        struct thread *th, pid_t pid)
 418 {
 419         struct thread *leader;
 420
 421         if (pid == th->pid_ || pid == -1 || th->pid_ != -1)
 422                 return;
 423
 424         th->pid_ = pid;
 425
 426         if (th->pid_ == th->tid)
 427                 return;
 428
 429         leader = __machine__findnew_thread(machine, th->pid_, th->pid_);
 430         if (!leader)
 431                 goto out_err;
 432
 433         if (!leader->maps)
 434                 leader->maps = maps__new(machine);
 435
 436         if (!leader->maps)
 437                 goto out_err;
 438
 439         if (th->maps == leader->maps)
 440                 return;
 441
 442         if (th->maps) {
 443                 /*
 444                  * Maps are created from MMAP events which provide the pid and
 445                  * tid.  Consequently there never should be any maps on a thread
 446                  * with an unknown pid.  Just print an error if there are.
 447                  */
 448                 if (!maps__empty(th->maps))
 449                         pr_err("Discarding thread maps for %d:%d\n",
 450                                th->pid_, th->tid);
 451                 maps__put(th->maps);
 452         }
 453
 454         th->maps = maps__get(leader->maps);
 455 out_put:
 456         thread__put(leader);
 457         return;
 458 out_err:
 459         pr_err("Failed to join map groups for %d:%d\n", th->pid_, th->tid);
 460         goto out_put;
 461 }
 462
 463 /*
 464  * Front-end cache - TID lookups come in blocks,
 465  * so most of the time we dont have to look up
 466  * the full rbtree:
 467  */
 468 static struct thread*
 469 __threads__get_last_match(struct threads *threads, struct machine *machine,
 470                           int pid, int tid)
 471 {
 472         struct thread *th;
 473
 474         th = threads->last_match;
 475         if (th != NULL) {
 476                 if (th->tid == tid) {
 477                         machine__update_thread_pid(machine, th, pid);
 478                         return thread__get(th);
 479                 }
 480
 481                 threads->last_match = NULL;
 482         }
 483
 484         return NULL;
 485 }
 486
 487 static struct thread*
 488 threads__get_last_match(struct threads *threads, struct machine *machine,
 489                         int pid, int tid)
 490 {
 491         struct thread *th = NULL;
 492
 493         if (perf_singlethreaded)
 494                 th = __threads__get_last_match(threads, machine, pid, tid);
 495
 496         return th;
 497 }
 498
 499 static void
 500 __threads__set_last_match(struct threads *threads, struct thread *th)
 501 {
 502         threads->last_match = th;
 503 }
 504
 505 static void
 506 threads__set_last_match(struct threads *threads, struct thread *th)
 507 {
 508         if (perf_singlethreaded)
 509                 __threads__set_last_match(threads, th);
 510 }
 511
 512 /*
 513  * Caller must eventually drop thread->refcnt returned with a successful
 514  * lookup/new thread inserted.
 515  */
 516 static struct thread *____machine__findnew_thread(struct machine *machine,
 517                                                   struct threads *threads,
 518                                                   pid_t pid, pid_t tid,
 519                                                   bool create)
 520 {
 521         struct rb_node **p = &threads->entries.rb_root.rb_node;
 522         struct rb_node *parent = NULL;
 523         struct thread *th;
 524         bool leftmost = true;
 525
 526         th = threads__get_last_match(threads, machine, pid, tid);
 527         if (th)
 528                 return th;
 529
 530         while (*p != NULL) {
 531                 parent = *p;
 532                 th = rb_entry(parent, struct thread, rb_node);
 533
 534                 if (th->tid == tid) {
 535                         threads__set_last_match(threads, th);
 536                         machine__update_thread_pid(machine, th, pid);
 537                         return thread__get(th);
 538                 }
 539
 540                 if (tid < th->tid)
 541                         p = &(*p)->rb_left;
 542                 else {
 543                         p = &(*p)->rb_right;
 544                         leftmost = false;
 545                 }
 546         }
 547
 548         if (!create)
 549                 return NULL;
 550
 551         th = thread__new(pid, tid);
 552         if (th != NULL) {
 553                 rb_link_node(&th->rb_node, parent, p);
 554                 rb_insert_color_cached(&th->rb_node, &threads->entries, leftmost);
 555
 556                 /*
 557                  * We have to initialize maps separately after rb tree is updated.
 558                  *
 559                  * The reason is that we call machine__findnew_thread
 560                  * within thread__init_maps to find the thread
 561                  * leader and that would screwed the rb tree.
 562                  */
 563                 if (thread__init_maps(th, machine)) {
 564                         rb_erase_cached(&th->rb_node, &threads->entries);
 565                         RB_CLEAR_NODE(&th->rb_node);
 566                         thread__put(th);
 567                         return NULL;
 568                 }
 569                 /*
 570                  * It is now in the rbtree, get a ref
 571                  */
 572                 thread__get(th);
 573                 threads__set_last_match(threads, th);
 574                 ++threads->nr;
 575         }
 576
 577         return th;
 578 }
 579
 580 struct thread *__machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid)
 581 {
 582         return ____machine__findnew_thread(machine, machine__threads(machine, tid), pid, tid, true);
 583 }
 584
 585 struct thread *machine__findnew_thread(struct machine *machine, pid_t pid,
 586                                        pid_t tid)
 587 {
 588         struct threads *threads = machine__threads(machine, tid);
 589         struct thread *th;
 590
 591         down_write(&threads->lock);
 592         th = __machine__findnew_thread(machine, pid, tid);
 593         up_write(&threads->lock);
 594         return th;
 595 }
 596
 597 struct thread *machine__find_thread(struct machine *machine, pid_t pid,
 598                                     pid_t tid)
 599 {
 600         struct threads *threads = machine__threads(machine, tid);
 601         struct thread *th;
 602
 603         down_read(&threads->lock);
 604         th =  ____machine__findnew_thread(machine, threads, pid, tid, false);
 605         up_read(&threads->lock);
 606         return th;
 607 }
 608
 609 /*
 610  * Threads are identified by pid and tid, and the idle task has pid == tid == 0.
 611  * So here a single thread is created for that, but actually there is a separate
 612  * idle task per cpu, so there should be one 'struct thread' per cpu, but there
 613  * is only 1. That causes problems for some tools, requiring workarounds. For
 614  * example get_idle_thread() in builtin-sched.c, or thread_stack__per_cpu().
 615  */
 616 struct thread *machine__idle_thread(struct machine *machine)
 617 {
 618         struct thread *thread = machine__findnew_thread(machine, 0, 0);
 619
 620         if (!thread || thread__set_comm(thread, "swapper", 0) ||
 621             thread__set_namespaces(thread, 0, NULL))
 622                 pr_err("problem inserting idle task for machine pid %d\n", machine->pid);
 623
 624         return thread;
 625 }
 626
 627 struct comm *machine__thread_exec_comm(struct machine *machine,
 628                                        struct thread *thread)
 629 {
 630         if (machine->comm_exec)
 631                 return thread__exec_comm(thread);
 632         else
 633                 return thread__comm(thread);
 634 }
 635
 636 int machine__process_comm_event(struct machine *machine, union perf_event *event,
 637                                 struct perf_sample *sample)
 638 {
 639         struct thread *thread = machine__findnew_thread(machine,
 640                                                         event->comm.pid,
 641                                                         event->comm.tid);
 642         bool exec = event->header.misc & PERF_RECORD_MISC_COMM_EXEC;
 643         int err = 0;
 644
 645         if (exec)
 646                 machine->comm_exec = true;
 647
 648         if (dump_trace)
 649                 perf_event__fprintf_comm(event, stdout);
 650
 651         if (thread == NULL ||
 652             __thread__set_comm(thread, event->comm.comm, sample->time, exec)) {
 653                 dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n");
 654                 err = -1;
 655         }
 656
 657         thread__put(thread);
 658
 659         return err;
 660 }
 661
 662 int machine__process_namespaces_event(struct machine *machine __maybe_unused,
 663                                       union perf_event *event,
 664                                       struct perf_sample *sample __maybe_unused)
 665 {
 666         struct thread *thread = machine__findnew_thread(machine,
 667                                                         event->namespaces.pid,
 668                                                         event->namespaces.tid);
 669         int err = 0;
 670
 671         WARN_ONCE(event->namespaces.nr_namespaces > NR_NAMESPACES,
 672                   "\nWARNING: kernel seems to support more namespaces than perf"
 673                   " tool.\nTry updating the perf tool..\n\n");
 674
 675         WARN_ONCE(event->namespaces.nr_namespaces < NR_NAMESPACES,
 676                   "\nWARNING: perf tool seems to support more namespaces than"
 677                   " the kernel.\nTry updating the kernel..\n\n");
 678
 679         if (dump_trace)
 680                 perf_event__fprintf_namespaces(event, stdout);
 681
 682         if (thread == NULL ||
 683             thread__set_namespaces(thread, sample->time, &event->namespaces)) {
 684                 dump_printf("problem processing PERF_RECORD_NAMESPACES, skipping event.\n");
 685                 err = -1;
 686         }
 687
 688         thread__put(thread);
 689
 690         return err;
 691 }
 692
 693 int machine__process_cgroup_event(struct machine *machine,
 694                                   union perf_event *event,
 695                                   struct perf_sample *sample __maybe_unused)
 696 {
 697         struct cgroup *cgrp;
 698
 699         if (dump_trace)
 700                 perf_event__fprintf_cgroup(event, stdout);
 701
 702         cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path);
 703         if (cgrp == NULL)
 704                 return -ENOMEM;
 705
 706         return 0;
 707 }
 708
 709 int machine__process_lost_event(struct machine *machine __maybe_unused,
 710                                 union perf_event *event, struct perf_sample *sample __maybe_unused)
 711 {
 712         dump_printf(": id:%" PRI_lu64 ": lost:%" PRI_lu64 "\n",
 713                     event->lost.id, event->lost.lost);
 714         return 0;
 715 }
 716
 717 int machine__process_lost_samples_event(struct machine *machine __maybe_unused,
 718                                         union perf_event *event, struct perf_sample *sample)
 719 {
 720         dump_printf(": id:%" PRIu64 ": lost samples :%" PRI_lu64 "\n",
 721                     sample->id, event->lost_samples.lost);
 722         return 0;
 723 }
 724
 725 static struct dso *machine__findnew_module_dso(struct machine *machine,
 726                                                struct kmod_path *m,
 727                                                const char *filename)
 728 {
 729         struct dso *dso;
 730
 731         down_write(&machine->dsos.lock);
 732
 733         dso = __dsos__find(&machine->dsos, m->name, true);
 734         if (!dso) {
 735                 dso = __dsos__addnew(&machine->dsos, m->name);
 736                 if (dso == NULL)
 737                         goto out_unlock;
 738
 739                 dso__set_module_info(dso, m, machine);
 740                 dso__set_long_name(dso, strdup(filename), true);
 741                 dso->kernel = DSO_SPACE__KERNEL;
 742         }
 743
 744         dso__get(dso);
 745 out_unlock:
 746         up_write(&machine->dsos.lock);
 747         return dso;
 748 }
 749
 750 int machine__process_aux_event(struct machine *machine __maybe_unused,
 751                                union perf_event *event)
 752 {
 753         if (dump_trace)
 754                 perf_event__fprintf_aux(event, stdout);
 755         return 0;
 756 }
 757
 758 int machine__process_itrace_start_event(struct machine *machine __maybe_unused,
 759                                         union perf_event *event)
 760 {
 761         if (dump_trace)
 762                 perf_event__fprintf_itrace_start(event, stdout);
 763         return 0;
 764 }
 765
 766 int machine__process_aux_output_hw_id_event(struct machine *machine __maybe_unused,
 767                                             union perf_event *event)
 768 {
 769         if (dump_trace)
 770                 perf_event__fprintf_aux_output_hw_id(event, stdout);
 771         return 0;
 772 }
 773
 774 int machine__process_switch_event(struct machine *machine __maybe_unused,
 775                                   union perf_event *event)
 776 {
 777         if (dump_trace)
 778                 perf_event__fprintf_switch(event, stdout);
 779         return 0;
 780 }
 781
 782 static int machine__process_ksymbol_register(struct machine *machine,
 783                                              union perf_event *event,
 784                                              struct perf_sample *sample __maybe_unused)
 785 {
 786         struct symbol *sym;
 787         struct map *map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr);
 788
 789         if (!map) {
 790                 struct dso *dso = dso__new(event->ksymbol.name);
 791
 792                 if (dso) {
 793                         dso->kernel = DSO_SPACE__KERNEL;
 794                         map = map__new2(0, dso);
 795                         dso__put(dso);
 796                 }
 797
 798                 if (!dso || !map) {
 799                         return -ENOMEM;
 800                 }
 801
 802                 if (event->ksymbol.ksym_type == PERF_RECORD_KSYMBOL_TYPE_OOL) {
 803                         map->dso->binary_type = DSO_BINARY_TYPE__OOL;
 804                         map->dso->data.file_size = event->ksymbol.len;
 805                         dso__set_loaded(map->dso);
 806                 }
 807
 808                 map->start = event->ksymbol.addr;
 809                 map->end = map->start + event->ksymbol.len;
 810                 maps__insert(machine__kernel_maps(machine), map);
 811                 map__put(map);
 812                 dso__set_loaded(dso);
 813
 814                 if (is_bpf_image(event->ksymbol.name)) {
 815                         dso->binary_type = DSO_BINARY_TYPE__BPF_IMAGE;
 816                         dso__set_long_name(dso, "", false);
 817                 }
 818         }
 819
 820         sym = symbol__new(map->map_ip(map, map->start),
 821                           event->ksymbol.len,
 822                           0, 0, event->ksymbol.name);
 823         if (!sym)
 824                 return -ENOMEM;
 825         dso__insert_symbol(map->dso, sym);
 826         return 0;
 827 }
 828
 829 static int machine__process_ksymbol_unregister(struct machine *machine,
 830                                                union perf_event *event,
 831                                                struct perf_sample *sample __maybe_unused)
 832 {
 833         struct symbol *sym;
 834         struct map *map;
 835
 836         map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr);
 837         if (!map)
 838                 return 0;
 839
 840         if (map != machine->vmlinux_map)
 841                 maps__remove(machine__kernel_maps(machine), map);
 842         else {
 843                 sym = dso__find_symbol(map->dso, map->map_ip(map, map->start));
 844                 if (sym)
 845                         dso__delete_symbol(map->dso, sym);
 846         }
 847
 848         return 0;
 849 }
 850
 851 int machine__process_ksymbol(struct machine *machine __maybe_unused,
 852                              union perf_event *event,
 853                              struct perf_sample *sample)
 854 {
 855         if (dump_trace)
 856                 perf_event__fprintf_ksymbol(event, stdout);
 857
 858         if (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER)
 859                 return machine__process_ksymbol_unregister(machine, event,
 860                                                            sample);
 861         return machine__process_ksymbol_register(machine, event, sample);
 862 }
 863
 864 int machine__process_text_poke(struct machine *machine, union perf_event *event,
 865                                struct perf_sample *sample __maybe_unused)
 866 {
 867         struct map *map = maps__find(machine__kernel_maps(machine), event->text_poke.addr);
 868         u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
 869
 870         if (dump_trace)
 871                 perf_event__fprintf_text_poke(event, machine, stdout);
 872
 873         if (!event->text_poke.new_len)
 874                 return 0;
 875
 876         if (cpumode != PERF_RECORD_MISC_KERNEL) {
 877                 pr_debug("%s: unsupported cpumode - ignoring\n", __func__);
 878                 return 0;
 879         }
 880
 881         if (map && map->dso) {
 882                 u8 *new_bytes = event->text_poke.bytes + event->text_poke.old_len;
 883                 int ret;
 884
 885                 /*
 886                  * Kernel maps might be changed when loading symbols so loading
 887                  * must be done prior to using kernel maps.
 888                  */
 889                 map__load(map);
 890                 ret = dso__data_write_cache_addr(map->dso, map, machine,
 891                                                  event->text_poke.addr,
 892                                                  new_bytes,
 893                                                  event->text_poke.new_len);
 894                 if (ret != event->text_poke.new_len)
 895                         pr_debug("Failed to write kernel text poke at %#" PRI_lx64 "\n",
 896                                  event->text_poke.addr);
 897         } else {
 898                 pr_debug("Failed to find kernel text poke address map for %#" PRI_lx64 "\n",
 899                          event->text_poke.addr);
 900         }
 901
 902         return 0;
 903 }
 904
 905 static struct map *machine__addnew_module_map(struct machine *machine, u64 start,
 906                                               const char *filename)
 907 {
 908         struct map *map = NULL;
 909         struct kmod_path m;
 910         struct dso *dso;
 911
 912         if (kmod_path__parse_name(&m, filename))
 913                 return NULL;
 914
 915         dso = machine__findnew_module_dso(machine, &m, filename);
 916         if (dso == NULL)
 917                 goto out;
 918
 919         map = map__new2(start, dso);
 920         if (map == NULL)
 921                 goto out;
 922
 923         maps__insert(machine__kernel_maps(machine), map);
 924
 925         /* Put the map here because maps__insert already got it */
 926         map__put(map);
 927 out:
 928         /* put the dso here, corresponding to  machine__findnew_module_dso */
 929         dso__put(dso);
 930         zfree(&m.name);
 931         return map;
 932 }
 933
 934 size_t machines__fprintf_dsos(struct machines *machines, FILE *fp)
 935 {
 936         struct rb_node *nd;
 937         size_t ret = __dsos__fprintf(&machines->host.dsos.head, fp);
 938
 939         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 940                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
 941                 ret += __dsos__fprintf(&pos->dsos.head, fp);
 942         }
 943
 944         return ret;
 945 }
 946
 947 size_t machine__fprintf_dsos_buildid(struct machine *m, FILE *fp,
 948                                      bool (skip)(struct dso *dso, int parm), int parm)
 949 {
 950         return __dsos__fprintf_buildid(&m->dsos.head, fp, skip, parm);
 951 }
 952
 953 size_t machines__fprintf_dsos_buildid(struct machines *machines, FILE *fp,
 954                                      bool (skip)(struct dso *dso, int parm), int parm)
 955 {
 956         struct rb_node *nd;
 957         size_t ret = machine__fprintf_dsos_buildid(&machines->host, fp, skip, parm);
 958
 959         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
 960                 struct machine *pos = rb_entry(nd, struct machine, rb_node);
 961                 ret += machine__fprintf_dsos_buildid(pos, fp, skip, parm);
 962         }
 963         return ret;
 964 }
 965
 966 size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp)
 967 {
 968         int i;
 969         size_t printed = 0;
 970         struct dso *kdso = machine__kernel_dso(machine);
 971
 972         if (kdso->has_build_id) {
 973                 char filename[PATH_MAX];
 974                 if (dso__build_id_filename(kdso, filename, sizeof(filename),
 975                                            false))
 976                         printed += fprintf(fp, "[0] %s\n", filename);
 977         }
 978
 979         for (i = 0; i < vmlinux_path__nr_entries; ++i)
 980                 printed += fprintf(fp, "[%d] %s\n",
 981                                    i + kdso->has_build_id, vmlinux_path[i]);
 982
 983         return printed;
 984 }
 985
 986 size_t machine__fprintf(struct machine *machine, FILE *fp)
 987 {
 988         struct rb_node *nd;
 989         size_t ret;
 990         int i;
 991
 992         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
 993                 struct threads *threads = &machine->threads[i];
 994
 995                 down_read(&threads->lock);
 996
 997                 ret = fprintf(fp, "Threads: %u\n", threads->nr);
 998
 999                 for (nd = rb_first_cached(&threads->entries); nd;
1000                      nd = rb_next(nd)) {
1001                         struct thread *pos = rb_entry(nd, struct thread, rb_node);
1002
1003                         ret += thread__fprintf(pos, fp);
1004                 }
1005
1006                 up_read(&threads->lock);
1007         }
1008         return ret;
1009 }
1010
1011 static struct dso *machine__get_kernel(struct machine *machine)
1012 {
1013         const char *vmlinux_name = machine->mmap_name;
1014         struct dso *kernel;
1015
1016         if (machine__is_host(machine)) {
1017                 if (symbol_conf.vmlinux_name)
1018                         vmlinux_name = symbol_conf.vmlinux_name;
1019
1020                 kernel = machine__findnew_kernel(machine, vmlinux_name,
1021                                                  "[kernel]", DSO_SPACE__KERNEL);
1022         } else {
1023                 if (symbol_conf.default_guest_vmlinux_name)
1024                         vmlinux_name = symbol_conf.default_guest_vmlinux_name;
1025
1026                 kernel = machine__findnew_kernel(machine, vmlinux_name,
1027                                                  "[guest.kernel]",
1028                                                  DSO_SPACE__KERNEL_GUEST);
1029         }
1030
1031         if (kernel != NULL && (!kernel->has_build_id))
1032                 dso__read_running_kernel_build_id(kernel, machine);
1033
1034         return kernel;
1035 }
1036
1037 struct process_args {
1038         u64 start;
1039 };
1040
1041 void machine__get_kallsyms_filename(struct machine *machine, char *buf,
1042                                     size_t bufsz)
1043 {
1044         if (machine__is_default_guest(machine))
1045                 scnprintf(buf, bufsz, "%s", symbol_conf.default_guest_kallsyms);
1046         else
1047                 scnprintf(buf, bufsz, "%s/proc/kallsyms", machine->root_dir);
1048 }
1049
1050 const char *ref_reloc_sym_names[] = {"_text", "_stext", NULL};
1051
1052 /* Figure out the start address of kernel map from /proc/kallsyms.
1053  * Returns the name of the start symbol in *symbol_name. Pass in NULL as
1054  * symbol_name if it's not that important.
1055  */
1056 static int machine__get_running_kernel_start(struct machine *machine,
1057                                              const char **symbol_name,
1058                                              u64 *start, u64 *end)
1059 {
1060         char filename[PATH_MAX];
1061         int i, err = -1;
1062         const char *name;
1063         u64 addr = 0;
1064
1065         machine__get_kallsyms_filename(machine, filename, PATH_MAX);
1066
1067         if (symbol__restricted_filename(filename, "/proc/kallsyms"))
1068                 return 0;
1069
1070         for (i = 0; (name = ref_reloc_sym_names[i]) != NULL; i++) {
1071                 err = kallsyms__get_function_start(filename, name, &addr);
1072                 if (!err)
1073                         break;
1074         }
1075
1076         if (err)
1077                 return -1;
1078
1079         if (symbol_name)
1080                 *symbol_name = name;
1081
1082         *start = addr;
1083
1084         err = kallsyms__get_function_start(filename, "_etext", &addr);
1085         if (!err)
1086                 *end = addr;
1087
1088         return 0;
1089 }
1090
1091 int machine__create_extra_kernel_map(struct machine *machine,
1092                                      struct dso *kernel,
1093                                      struct extra_kernel_map *xm)
1094 {
1095         struct kmap *kmap;
1096         struct map *map;
1097
1098         map = map__new2(xm->start, kernel);
1099         if (!map)
1100                 return -1;
1101
1102         map->end   = xm->end;
1103         map->pgoff = xm->pgoff;
1104
1105         kmap = map__kmap(map);
1106
1107         strlcpy(kmap->name, xm->name, KMAP_NAME_LEN);
1108
1109         maps__insert(machine__kernel_maps(machine), map);
1110
1111         pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n",
1112                   kmap->name, map->start, map->end);
1113
1114         map__put(map);
1115
1116         return 0;
1117 }
1118
1119 static u64 find_entry_trampoline(struct dso *dso)
1120 {
1121         /* Duplicates are removed so lookup all aliases */
1122         const char *syms[] = {
1123                 "_entry_trampoline",
1124                 "__entry_trampoline_start",
1125                 "entry_SYSCALL_64_trampoline",
1126         };
1127         struct symbol *sym = dso__first_symbol(dso);
1128         unsigned int i;
1129
1130         for (; sym; sym = dso__next_symbol(sym)) {
1131                 if (sym->binding != STB_GLOBAL)
1132                         continue;
1133                 for (i = 0; i < ARRAY_SIZE(syms); i++) {
1134                         if (!strcmp(sym->name, syms[i]))
1135                                 return sym->start;
1136                 }
1137         }
1138
1139         return 0;
1140 }
1141
1142 /*
1143  * These values can be used for kernels that do not have symbols for the entry
1144  * trampolines in kallsyms.
1145  */
1146 #define X86_64_CPU_ENTRY_AREA_PER_CPU   0xfffffe0000000000ULL
1147 #define X86_64_CPU_ENTRY_AREA_SIZE      0x2c000
1148 #define X86_64_ENTRY_TRAMPOLINE         0x6000
1149
1150 /* Map x86_64 PTI entry trampolines */
1151 int machine__map_x86_64_entry_trampolines(struct machine *machine,
1152                                           struct dso *kernel)
1153 {
1154         struct maps *kmaps = machine__kernel_maps(machine);
1155         int nr_cpus_avail, cpu;
1156         bool found = false;
1157         struct map *map;
1158         u64 pgoff;
1159
1160         /*
1161          * In the vmlinux case, pgoff is a virtual address which must now be
1162          * mapped to a vmlinux offset.
1163          */
1164         maps__for_each_entry(kmaps, map) {
1165                 struct kmap *kmap = __map__kmap(map);
1166                 struct map *dest_map;
1167
1168                 if (!kmap || !is_entry_trampoline(kmap->name))
1169                         continue;
1170
1171                 dest_map = maps__find(kmaps, map->pgoff);
1172                 if (dest_map != map)
1173                         map->pgoff = dest_map->map_ip(dest_map, map->pgoff);
1174                 found = true;
1175         }
1176         if (found || machine->trampolines_mapped)
1177                 return 0;
1178
1179         pgoff = find_entry_trampoline(kernel);
1180         if (!pgoff)
1181                 return 0;
1182
1183         nr_cpus_avail = machine__nr_cpus_avail(machine);
1184
1185         /* Add a 1 page map for each CPU's entry trampoline */
1186         for (cpu = 0; cpu < nr_cpus_avail; cpu++) {
1187                 u64 va = X86_64_CPU_ENTRY_AREA_PER_CPU +
1188                          cpu * X86_64_CPU_ENTRY_AREA_SIZE +
1189                          X86_64_ENTRY_TRAMPOLINE;
1190                 struct extra_kernel_map xm = {
1191                         .start = va,
1192                         .end   = va + page_size,
1193                         .pgoff = pgoff,
1194                 };
1195
1196                 strlcpy(xm.name, ENTRY_TRAMPOLINE_NAME, KMAP_NAME_LEN);
1197
1198                 if (machine__create_extra_kernel_map(machine, kernel, &xm) < 0)
1199                         return -1;
1200         }
1201
1202         machine->trampolines_mapped = nr_cpus_avail;
1203
1204         return 0;
1205 }
1206
1207 int __weak machine__create_extra_kernel_maps(struct machine *machine __maybe_unused,
1208                                              struct dso *kernel __maybe_unused)
1209 {
1210         return 0;
1211 }
1212
1213 static int
1214 __machine__create_kernel_maps(struct machine *machine, struct dso *kernel)
1215 {
1216         /* In case of renewal the kernel map, destroy previous one */
1217         machine__destroy_kernel_maps(machine);
1218
1219         machine->vmlinux_map = map__new2(0, kernel);
1220         if (machine->vmlinux_map == NULL)
1221                 return -1;
1222
1223         machine->vmlinux_map->map_ip = machine->vmlinux_map->unmap_ip = identity__map_ip;
1224         maps__insert(machine__kernel_maps(machine), machine->vmlinux_map);
1225         return 0;
1226 }
1227
1228 void machine__destroy_kernel_maps(struct machine *machine)
1229 {
1230         struct kmap *kmap;
1231         struct map *map = machine__kernel_map(machine);
1232
1233         if (map == NULL)
1234                 return;
1235
1236         kmap = map__kmap(map);
1237         maps__remove(machine__kernel_maps(machine), map);
1238         if (kmap && kmap->ref_reloc_sym) {
1239                 zfree((char **)&kmap->ref_reloc_sym->name);
1240                 zfree(&kmap->ref_reloc_sym);
1241         }
1242
1243         map__zput(machine->vmlinux_map);
1244 }
1245
1246 int machines__create_guest_kernel_maps(struct machines *machines)
1247 {
1248         int ret = 0;
1249         struct dirent **namelist = NULL;
1250         int i, items = 0;
1251         char path[PATH_MAX];
1252         pid_t pid;
1253         char *endp;
1254
1255         if (symbol_conf.default_guest_vmlinux_name ||
1256             symbol_conf.default_guest_modules ||
1257             symbol_conf.default_guest_kallsyms) {
1258                 machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID);
1259         }
1260
1261         if (symbol_conf.guestmount) {
1262                 items = scandir(symbol_conf.guestmount, &namelist, NULL, NULL);
1263                 if (items <= 0)
1264                         return -ENOENT;
1265                 for (i = 0; i < items; i++) {
1266                         if (!isdigit(namelist[i]->d_name[0])) {
1267                                 /* Filter out . and .. */
1268                                 continue;
1269                         }
1270                         pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10);
1271                         if ((*endp != '\0') ||
1272                             (endp == namelist[i]->d_name) ||
1273                             (errno == ERANGE)) {
1274                                 pr_debug("invalid directory (%s). Skipping.\n",
1275                                          namelist[i]->d_name);
1276                                 continue;
1277                         }
1278                         sprintf(path, "%s/%s/proc/kallsyms",
1279                                 symbol_conf.guestmount,
1280                                 namelist[i]->d_name);
1281                         ret = access(path, R_OK);
1282                         if (ret) {
1283                                 pr_debug("Can't access file %s\n", path);
1284                                 goto failure;
1285                         }
1286                         machines__create_kernel_maps(machines, pid);
1287                 }
1288 failure:
1289                 free(namelist);
1290         }
1291
1292         return ret;
1293 }
1294
1295 void machines__destroy_kernel_maps(struct machines *machines)
1296 {
1297         struct rb_node *next = rb_first_cached(&machines->guests);
1298
1299         machine__destroy_kernel_maps(&machines->host);
1300
1301         while (next) {
1302                 struct machine *pos = rb_entry(next, struct machine, rb_node);
1303
1304                 next = rb_next(&pos->rb_node);
1305                 rb_erase_cached(&pos->rb_node, &machines->guests);
1306                 machine__delete(pos);
1307         }
1308 }
1309
1310 int machines__create_kernel_maps(struct machines *machines, pid_t pid)
1311 {
1312         struct machine *machine = machines__findnew(machines, pid);
1313
1314         if (machine == NULL)
1315                 return -1;
1316
1317         return machine__create_kernel_maps(machine);
1318 }
1319
1320 int machine__load_kallsyms(struct machine *machine, const char *filename)
1321 {
1322         struct map *map = machine__kernel_map(machine);
1323         int ret = __dso__load_kallsyms(map->dso, filename, map, true);
1324
1325         if (ret > 0) {
1326                 dso__set_loaded(map->dso);
1327                 /*
1328                  * Since /proc/kallsyms will have multiple sessions for the
1329                  * kernel, with modules between them, fixup the end of all
1330                  * sections.
1331                  */
1332                 maps__fixup_end(machine__kernel_maps(machine));
1333         }
1334
1335         return ret;
1336 }
1337
1338 int machine__load_vmlinux_path(struct machine *machine)
1339 {
1340         struct map *map = machine__kernel_map(machine);
1341         int ret = dso__load_vmlinux_path(map->dso, map);
1342
1343         if (ret > 0)
1344                 dso__set_loaded(map->dso);
1345
1346         return ret;
1347 }
1348
1349 static char *get_kernel_version(const char *root_dir)
1350 {
1351         char version[PATH_MAX];
1352         FILE *file;
1353         char *name, *tmp;
1354         const char *prefix = "Linux version ";
1355
1356         sprintf(version, "%s/proc/version", root_dir);
1357         file = fopen(version, "r");
1358         if (!file)
1359                 return NULL;
1360
1361         tmp = fgets(version, sizeof(version), file);
1362         fclose(file);
1363         if (!tmp)
1364                 return NULL;
1365
1366         name = strstr(version, prefix);
1367         if (!name)
1368                 return NULL;
1369         name += strlen(prefix);
1370         tmp = strchr(name, ' ');
1371         if (tmp)
1372                 *tmp = '\0';
1373
1374         return strdup(name);
1375 }
1376
1377 static bool is_kmod_dso(struct dso *dso)
1378 {
1379         return dso->symtab_type == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE ||
1380                dso->symtab_type == DSO_BINARY_TYPE__GUEST_KMODULE;
1381 }
1382
1383 static int maps__set_module_path(struct maps *maps, const char *path, struct kmod_path *m)
1384 {
1385         char *long_name;
1386         struct map *map = maps__find_by_name(maps, m->name);
1387
1388         if (map == NULL)
1389                 return 0;
1390
1391         long_name = strdup(path);
1392         if (long_name == NULL)
1393                 return -ENOMEM;
1394
1395         dso__set_long_name(map->dso, long_name, true);
1396         dso__kernel_module_get_build_id(map->dso, "");
1397
1398         /*
1399          * Full name could reveal us kmod compression, so
1400          * we need to update the symtab_type if needed.
1401          */
1402         if (m->comp && is_kmod_dso(map->dso)) {
1403                 map->dso->symtab_type++;
1404                 map->dso->comp = m->comp;
1405         }
1406
1407         return 0;
1408 }
1409
1410 static int maps__set_modules_path_dir(struct maps *maps, const char *dir_name, int depth)
1411 {
1412         struct dirent *dent;
1413         DIR *dir = opendir(dir_name);
1414         int ret = 0;
1415
1416         if (!dir) {
1417                 pr_debug("%s: cannot open %s dir\n", __func__, dir_name);
1418                 return -1;
1419         }
1420
1421         while ((dent = readdir(dir)) != NULL) {
1422                 char path[PATH_MAX];
1423                 struct stat st;
1424
1425                 /*sshfs might return bad dent->d_type, so we have to stat*/
1426                 path__join(path, sizeof(path), dir_name, dent->d_name);
1427                 if (stat(path, &st))
1428                         continue;
1429
1430                 if (S_ISDIR(st.st_mode)) {
1431                         if (!strcmp(dent->d_name, ".") ||
1432                             !strcmp(dent->d_name, ".."))
1433                                 continue;
1434
1435                         /* Do not follow top-level source and build symlinks */
1436                         if (depth == 0) {
1437                                 if (!strcmp(dent->d_name, "source") ||
1438                                     !strcmp(dent->d_name, "build"))
1439                                         continue;
1440                         }
1441
1442                         ret = maps__set_modules_path_dir(maps, path, depth + 1);
1443                         if (ret < 0)
1444                                 goto out;
1445                 } else {
1446                         struct kmod_path m;
1447
1448                         ret = kmod_path__parse_name(&m, dent->d_name);
1449                         if (ret)
1450                                 goto out;
1451
1452                         if (m.kmod)
1453                                 ret = maps__set_module_path(maps, path, &m);
1454
1455                         zfree(&m.name);
1456
1457                         if (ret)
1458                                 goto out;
1459                 }
1460         }
1461
1462 out:
1463         closedir(dir);
1464         return ret;
1465 }
1466
1467 static int machine__set_modules_path(struct machine *machine)
1468 {
1469         char *version;
1470         char modules_path[PATH_MAX];
1471
1472         version = get_kernel_version(machine->root_dir);
1473         if (!version)
1474                 return -1;
1475
1476         snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s",
1477                  machine->root_dir, version);
1478         free(version);
1479
1480         return maps__set_modules_path_dir(machine__kernel_maps(machine), modules_path, 0);
1481 }
1482 int __weak arch__fix_module_text_start(u64 *start __maybe_unused,
1483                                 u64 *size __maybe_unused,
1484                                 const char *name __maybe_unused)
1485 {
1486         return 0;
1487 }
1488
1489 static int machine__create_module(void *arg, const char *name, u64 start,
1490                                   u64 size)
1491 {
1492         struct machine *machine = arg;
1493         struct map *map;
1494
1495         if (arch__fix_module_text_start(&start, &size, name) < 0)
1496                 return -1;
1497
1498         map = machine__addnew_module_map(machine, start, name);
1499         if (map == NULL)
1500                 return -1;
1501         map->end = start + size;
1502
1503         dso__kernel_module_get_build_id(map->dso, machine->root_dir);
1504
1505         return 0;
1506 }
1507
1508 static int machine__create_modules(struct machine *machine)
1509 {
1510         const char *modules;
1511         char path[PATH_MAX];
1512
1513         if (machine__is_default_guest(machine)) {
1514                 modules = symbol_conf.default_guest_modules;
1515         } else {
1516                 snprintf(path, PATH_MAX, "%s/proc/modules", machine->root_dir);
1517                 modules = path;
1518         }
1519
1520         if (symbol__restricted_filename(modules, "/proc/modules"))
1521                 return -1;
1522
1523         if (modules__parse(modules, machine, machine__create_module))
1524                 return -1;
1525
1526         if (!machine__set_modules_path(machine))
1527                 return 0;
1528
1529         pr_debug("Problems setting modules path maps, continuing anyway...\n");
1530
1531         return 0;
1532 }
1533
1534 static void machine__set_kernel_mmap(struct machine *machine,
1535                                      u64 start, u64 end)
1536 {
1537         machine->vmlinux_map->start = start;
1538         machine->vmlinux_map->end   = end;
1539         /*
1540          * Be a bit paranoid here, some perf.data file came with
1541          * a zero sized synthesized MMAP event for the kernel.
1542          */
1543         if (start == 0 && end == 0)
1544                 machine->vmlinux_map->end = ~0ULL;
1545 }
1546
1547 static void machine__update_kernel_mmap(struct machine *machine,
1548                                      u64 start, u64 end)
1549 {
1550         struct map *map = machine__kernel_map(machine);
1551
1552         map__get(map);
1553         maps__remove(machine__kernel_maps(machine), map);
1554
1555         machine__set_kernel_mmap(machine, start, end);
1556
1557         maps__insert(machine__kernel_maps(machine), map);
1558         map__put(map);
1559 }
1560
1561 int machine__create_kernel_maps(struct machine *machine)
1562 {
1563         struct dso *kernel = machine__get_kernel(machine);
1564         const char *name = NULL;
1565         struct map *map;
1566         u64 start = 0, end = ~0ULL;
1567         int ret;
1568
1569         if (kernel == NULL)
1570                 return -1;
1571
1572         ret = __machine__create_kernel_maps(machine, kernel);
1573         if (ret < 0)
1574                 goto out_put;
1575
1576         if (symbol_conf.use_modules && machine__create_modules(machine) < 0) {
1577                 if (machine__is_host(machine))
1578                         pr_debug("Problems creating module maps, "
1579                                  "continuing anyway...\n");
1580                 else
1581                         pr_debug("Problems creating module maps for guest %d, "
1582                                  "continuing anyway...\n", machine->pid);
1583         }
1584
1585         if (!machine__get_running_kernel_start(machine, &name, &start, &end)) {
1586                 if (name &&
1587                     map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, name, start)) {
1588                         machine__destroy_kernel_maps(machine);
1589                         ret = -1;
1590                         goto out_put;
1591                 }
1592
1593                 /*
1594                  * we have a real start address now, so re-order the kmaps
1595                  * assume it's the last in the kmaps
1596                  */
1597                 machine__update_kernel_mmap(machine, start, end);
1598         }
1599
1600         if (machine__create_extra_kernel_maps(machine, kernel))
1601                 pr_debug("Problems creating extra kernel maps, continuing anyway...\n");
1602
1603         if (end == ~0ULL) {
1604                 /* update end address of the kernel map using adjacent module address */
1605                 map = map__next(machine__kernel_map(machine));
1606                 if (map)
1607                         machine__set_kernel_mmap(machine, start, map->start);
1608         }
1609
1610 out_put:
1611         dso__put(kernel);
1612         return ret;
1613 }
1614
1615 static bool machine__uses_kcore(struct machine *machine)
1616 {
1617         struct dso *dso;
1618
1619         list_for_each_entry(dso, &machine->dsos.head, node) {
1620                 if (dso__is_kcore(dso))
1621                         return true;
1622         }
1623
1624         return false;
1625 }
1626
1627 static bool perf_event__is_extra_kernel_mmap(struct machine *machine,
1628                                              struct extra_kernel_map *xm)
1629 {
1630         return machine__is(machine, "x86_64") &&
1631                is_entry_trampoline(xm->name);
1632 }
1633
1634 static int machine__process_extra_kernel_map(struct machine *machine,
1635                                              struct extra_kernel_map *xm)
1636 {
1637         struct dso *kernel = machine__kernel_dso(machine);
1638
1639         if (kernel == NULL)
1640                 return -1;
1641
1642         return machine__create_extra_kernel_map(machine, kernel, xm);
1643 }
1644
1645 static int machine__process_kernel_mmap_event(struct machine *machine,
1646                                               struct extra_kernel_map *xm,
1647                                               struct build_id *bid)
1648 {
1649         struct map *map;
1650         enum dso_space_type dso_space;
1651         bool is_kernel_mmap;
1652
1653         /* If we have maps from kcore then we do not need or want any others */
1654         if (machine__uses_kcore(machine))
1655                 return 0;
1656
1657         if (machine__is_host(machine))
1658                 dso_space = DSO_SPACE__KERNEL;
1659         else
1660                 dso_space = DSO_SPACE__KERNEL_GUEST;
1661
1662         is_kernel_mmap = memcmp(xm->name, machine->mmap_name,
1663                                 strlen(machine->mmap_name) - 1) == 0;
1664         if (xm->name[0] == '/' ||
1665             (!is_kernel_mmap && xm->name[0] == '[')) {
1666                 map = machine__addnew_module_map(machine, xm->start,
1667                                                  xm->name);
1668                 if (map == NULL)
1669                         goto out_problem;
1670
1671                 map->end = map->start + xm->end - xm->start;
1672
1673                 if (build_id__is_defined(bid))
1674                         dso__set_build_id(map->dso, bid);
1675
1676         } else if (is_kernel_mmap) {
1677                 const char *symbol_name = (xm->name + strlen(machine->mmap_name));
1678                 /*
1679                  * Should be there already, from the build-id table in
1680                  * the header.
1681                  */
1682                 struct dso *kernel = NULL;
1683                 struct dso *dso;
1684
1685                 down_read(&machine->dsos.lock);
1686
1687                 list_for_each_entry(dso, &machine->dsos.head, node) {
1688
1689                         /*
1690                          * The cpumode passed to is_kernel_module is not the
1691                          * cpumode of *this* event. If we insist on passing
1692                          * correct cpumode to is_kernel_module, we should
1693                          * record the cpumode when we adding this dso to the
1694                          * linked list.
1695                          *
1696                          * However we don't really need passing correct
1697                          * cpumode.  We know the correct cpumode must be kernel
1698                          * mode (if not, we should not link it onto kernel_dsos
1699                          * list).
1700                          *
1701                          * Therefore, we pass PERF_RECORD_MISC_CPUMODE_UNKNOWN.
1702                          * is_kernel_module() treats it as a kernel cpumode.
1703                          */
1704
1705                         if (!dso->kernel ||
1706                             is_kernel_module(dso->long_name,
1707                                              PERF_RECORD_MISC_CPUMODE_UNKNOWN))
1708                                 continue;
1709
1710
1711                         kernel = dso;
1712                         break;
1713                 }
1714
1715                 up_read(&machine->dsos.lock);
1716
1717                 if (kernel == NULL)
1718                         kernel = machine__findnew_dso(machine, machine->mmap_name);
1719                 if (kernel == NULL)
1720                         goto out_problem;
1721
1722                 kernel->kernel = dso_space;
1723                 if (__machine__create_kernel_maps(machine, kernel) < 0) {
1724                         dso__put(kernel);
1725                         goto out_problem;
1726                 }
1727
1728                 if (strstr(kernel->long_name, "vmlinux"))
1729                         dso__set_short_name(kernel, "[kernel.vmlinux]", false);
1730
1731                 machine__update_kernel_mmap(machine, xm->start, xm->end);
1732
1733                 if (build_id__is_defined(bid))
1734                         dso__set_build_id(kernel, bid);
1735
1736                 /*
1737                  * Avoid using a zero address (kptr_restrict) for the ref reloc
1738                  * symbol. Effectively having zero here means that at record
1739                  * time /proc/sys/kernel/kptr_restrict was non zero.
1740                  */
1741                 if (xm->pgoff != 0) {
1742                         map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map,
1743                                                         symbol_name,
1744                                                         xm->pgoff);
1745                 }
1746
1747                 if (machine__is_default_guest(machine)) {
1748                         /*
1749                          * preload dso of guest kernel and modules
1750                          */
1751                         dso__load(kernel, machine__kernel_map(machine));
1752                 }
1753         } else if (perf_event__is_extra_kernel_mmap(machine, xm)) {
1754                 return machine__process_extra_kernel_map(machine, xm);
1755         }
1756         return 0;
1757 out_problem:
1758         return -1;
1759 }
1760
1761 int machine__process_mmap2_event(struct machine *machine,
1762                                  union perf_event *event,
1763                                  struct perf_sample *sample)
1764 {
1765         struct thread *thread;
1766         struct map *map;
1767         struct dso_id dso_id = {
1768                 .maj = event->mmap2.maj,
1769                 .min = event->mmap2.min,
1770                 .ino = event->mmap2.ino,
1771                 .ino_generation = event->mmap2.ino_generation,
1772         };
1773         struct build_id __bid, *bid = NULL;
1774         int ret = 0;
1775
1776         if (dump_trace)
1777                 perf_event__fprintf_mmap2(event, stdout);
1778
1779         if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
1780                 bid = &__bid;
1781                 build_id__init(bid, event->mmap2.build_id, event->mmap2.build_id_size);
1782         }
1783
1784         if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1785             sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1786                 struct extra_kernel_map xm = {
1787                         .start = event->mmap2.start,
1788                         .end   = event->mmap2.start + event->mmap2.len,
1789                         .pgoff = event->mmap2.pgoff,
1790                 };
1791
1792                 strlcpy(xm.name, event->mmap2.filename, KMAP_NAME_LEN);
1793                 ret = machine__process_kernel_mmap_event(machine, &xm, bid);
1794                 if (ret < 0)
1795                         goto out_problem;
1796                 return 0;
1797         }
1798
1799         thread = machine__findnew_thread(machine, event->mmap2.pid,
1800                                         event->mmap2.tid);
1801         if (thread == NULL)
1802                 goto out_problem;
1803
1804         map = map__new(machine, event->mmap2.start,
1805                         event->mmap2.len, event->mmap2.pgoff,
1806                         &dso_id, event->mmap2.prot,
1807                         event->mmap2.flags, bid,
1808                         event->mmap2.filename, thread);
1809
1810         if (map == NULL)
1811                 goto out_problem_map;
1812
1813         ret = thread__insert_map(thread, map);
1814         if (ret)
1815                 goto out_problem_insert;
1816
1817         thread__put(thread);
1818         map__put(map);
1819         return 0;
1820
1821 out_problem_insert:
1822         map__put(map);
1823 out_problem_map:
1824         thread__put(thread);
1825 out_problem:
1826         dump_printf("problem processing PERF_RECORD_MMAP2, skipping event.\n");
1827         return 0;
1828 }
1829
1830 int machine__process_mmap_event(struct machine *machine, union perf_event *event,
1831                                 struct perf_sample *sample)
1832 {
1833         struct thread *thread;
1834         struct map *map;
1835         u32 prot = 0;
1836         int ret = 0;
1837
1838         if (dump_trace)
1839                 perf_event__fprintf_mmap(event, stdout);
1840
1841         if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1842             sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1843                 struct extra_kernel_map xm = {
1844                         .start = event->mmap.start,
1845                         .end   = event->mmap.start + event->mmap.len,
1846                         .pgoff = event->mmap.pgoff,
1847                 };
1848
1849                 strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN);
1850                 ret = machine__process_kernel_mmap_event(machine, &xm, NULL);
1851                 if (ret < 0)
1852                         goto out_problem;
1853                 return 0;
1854         }
1855
1856         thread = machine__findnew_thread(machine, event->mmap.pid,
1857                                          event->mmap.tid);
1858         if (thread == NULL)
1859                 goto out_problem;
1860
1861         if (!(event->header.misc & PERF_RECORD_MISC_MMAP_DATA))
1862                 prot = PROT_EXEC;
1863
1864         map = map__new(machine, event->mmap.start,
1865                         event->mmap.len, event->mmap.pgoff,
1866                         NULL, prot, 0, NULL, event->mmap.filename, thread);
1867
1868         if (map == NULL)
1869                 goto out_problem_map;
1870
1871         ret = thread__insert_map(thread, map);
1872         if (ret)
1873                 goto out_problem_insert;
1874
1875         thread__put(thread);
1876         map__put(map);
1877         return 0;
1878
1879 out_problem_insert:
1880         map__put(map);
1881 out_problem_map:
1882         thread__put(thread);
1883 out_problem:
1884         dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n");
1885         return 0;
1886 }
1887
1888 static void __machine__remove_thread(struct machine *machine, struct thread *th, bool lock)
1889 {
1890         struct threads *threads = machine__threads(machine, th->tid);
1891
1892         if (threads->last_match == th)
1893                 threads__set_last_match(threads, NULL);
1894
1895         if (lock)
1896                 down_write(&threads->lock);
1897
1898         BUG_ON(refcount_read(&th->refcnt) == 0);
1899
1900         rb_erase_cached(&th->rb_node, &threads->entries);
1901         RB_CLEAR_NODE(&th->rb_node);
1902         --threads->nr;
1903         /*
1904          * Move it first to the dead_threads list, then drop the reference,
1905          * if this is the last reference, then the thread__delete destructor
1906          * will be called and we will remove it from the dead_threads list.
1907          */
1908         list_add_tail(&th->node, &threads->dead);
1909
1910         /*
1911          * We need to do the put here because if this is the last refcount,
1912          * then we will be touching the threads->dead head when removing the
1913          * thread.
1914          */
1915         thread__put(th);
1916
1917         if (lock)
1918                 up_write(&threads->lock);
1919 }
1920
1921 void machine__remove_thread(struct machine *machine, struct thread *th)
1922 {
1923         return __machine__remove_thread(machine, th, true);
1924 }
1925
1926 int machine__process_fork_event(struct machine *machine, union perf_event *event,
1927                                 struct perf_sample *sample)
1928 {
1929         struct thread *thread = machine__find_thread(machine,
1930                                                      event->fork.pid,
1931                                                      event->fork.tid);
1932         struct thread *parent = machine__findnew_thread(machine,
1933                                                         event->fork.ppid,
1934                                                         event->fork.ptid);
1935         bool do_maps_clone = true;
1936         int err = 0;
1937
1938         if (dump_trace)
1939                 perf_event__fprintf_task(event, stdout);
1940
1941         /*
1942          * There may be an existing thread that is not actually the parent,
1943          * either because we are processing events out of order, or because the
1944          * (fork) event that would have removed the thread was lost. Assume the
1945          * latter case and continue on as best we can.
1946          */
1947         if (parent->pid_ != (pid_t)event->fork.ppid) {
1948                 dump_printf("removing erroneous parent thread %d/%d\n",
1949                             parent->pid_, parent->tid);
1950                 machine__remove_thread(machine, parent);
1951                 thread__put(parent);
1952                 parent = machine__findnew_thread(machine, event->fork.ppid,
1953                                                  event->fork.ptid);
1954         }
1955
1956         /* if a thread currently exists for the thread id remove it */
1957         if (thread != NULL) {
1958                 machine__remove_thread(machine, thread);
1959                 thread__put(thread);
1960         }
1961
1962         thread = machine__findnew_thread(machine, event->fork.pid,
1963                                          event->fork.tid);
1964         /*
1965          * When synthesizing FORK events, we are trying to create thread
1966          * objects for the already running tasks on the machine.
1967          *
1968          * Normally, for a kernel FORK event, we want to clone the parent's
1969          * maps because that is what the kernel just did.
1970          *
1971          * But when synthesizing, this should not be done.  If we do, we end up
1972          * with overlapping maps as we process the synthesized MMAP2 events that
1973          * get delivered shortly thereafter.
1974          *
1975          * Use the FORK event misc flags in an internal way to signal this
1976          * situation, so we can elide the map clone when appropriate.
1977          */
1978         if (event->fork.header.misc & PERF_RECORD_MISC_FORK_EXEC)
1979                 do_maps_clone = false;
1980
1981         if (thread == NULL || parent == NULL ||
1982             thread__fork(thread, parent, sample->time, do_maps_clone) < 0) {
1983                 dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
1984                 err = -1;
1985         }
1986         thread__put(thread);
1987         thread__put(parent);
1988
1989         return err;
1990 }
1991
1992 int machine__process_exit_event(struct machine *machine, union perf_event *event,
1993                                 struct perf_sample *sample __maybe_unused)
1994 {
1995         struct thread *thread = machine__find_thread(machine,
1996                                                      event->fork.pid,
1997                                                      event->fork.tid);
1998
1999         if (dump_trace)
2000                 perf_event__fprintf_task(event, stdout);
2001
2002         if (thread != NULL) {
2003                 thread__exited(thread);
2004                 thread__put(thread);
2005         }
2006
2007         return 0;
2008 }
2009
2010 int machine__process_event(struct machine *machine, union perf_event *event,
2011                            struct perf_sample *sample)
2012 {
2013         int ret;
2014
2015         switch (event->header.type) {
2016         case PERF_RECORD_COMM:
2017                 ret = machine__process_comm_event(machine, event, sample); break;
2018         case PERF_RECORD_MMAP:
2019                 ret = machine__process_mmap_event(machine, event, sample); break;
2020         case PERF_RECORD_NAMESPACES:
2021                 ret = machine__process_namespaces_event(machine, event, sample); break;
2022         case PERF_RECORD_CGROUP:
2023                 ret = machine__process_cgroup_event(machine, event, sample); break;
2024         case PERF_RECORD_MMAP2:
2025                 ret = machine__process_mmap2_event(machine, event, sample); break;
2026         case PERF_RECORD_FORK:
2027                 ret = machine__process_fork_event(machine, event, sample); break;
2028         case PERF_RECORD_EXIT:
2029                 ret = machine__process_exit_event(machine, event, sample); break;
2030         case PERF_RECORD_LOST:
2031                 ret = machine__process_lost_event(machine, event, sample); break;
2032         case PERF_RECORD_AUX:
2033                 ret = machine__process_aux_event(machine, event); break;
2034         case PERF_RECORD_ITRACE_START:
2035                 ret = machine__process_itrace_start_event(machine, event); break;
2036         case PERF_RECORD_LOST_SAMPLES:
2037                 ret = machine__process_lost_samples_event(machine, event, sample); break;
2038         case PERF_RECORD_SWITCH:
2039         case PERF_RECORD_SWITCH_CPU_WIDE:
2040                 ret = machine__process_switch_event(machine, event); break;
2041         case PERF_RECORD_KSYMBOL:
2042                 ret = machine__process_ksymbol(machine, event, sample); break;
2043         case PERF_RECORD_BPF_EVENT:
2044                 ret = machine__process_bpf(machine, event, sample); break;
2045         case PERF_RECORD_TEXT_POKE:
2046                 ret = machine__process_text_poke(machine, event, sample); break;
2047         case PERF_RECORD_AUX_OUTPUT_HW_ID:
2048                 ret = machine__process_aux_output_hw_id_event(machine, event); break;
2049         default:
2050                 ret = -1;
2051                 break;
2052         }
2053
2054         return ret;
2055 }
2056
2057 static bool symbol__match_regex(struct symbol *sym, regex_t *regex)
2058 {
2059         if (!regexec(regex, sym->name, 0, NULL, 0))
2060                 return true;
2061         return false;
2062 }
2063
2064 static void ip__resolve_ams(struct thread *thread,
2065                             struct addr_map_symbol *ams,
2066                             u64 ip)
2067 {
2068         struct addr_location al;
2069
2070         memset(&al, 0, sizeof(al));
2071         /*
2072          * We cannot use the header.misc hint to determine whether a
2073          * branch stack address is user, kernel, guest, hypervisor.
2074          * Branches may straddle the kernel/user/hypervisor boundaries.
2075          * Thus, we have to try consecutively until we find a match
2076          * or else, the symbol is unknown
2077          */
2078         thread__find_cpumode_addr_location(thread, ip, &al);
2079
2080         ams->addr = ip;
2081         ams->al_addr = al.addr;
2082         ams->al_level = al.level;
2083         ams->ms.maps = al.maps;
2084         ams->ms.sym = al.sym;
2085         ams->ms.map = al.map;
2086         ams->phys_addr = 0;
2087         ams->data_page_size = 0;
2088 }
2089
2090 static void ip__resolve_data(struct thread *thread,
2091                              u8 m, struct addr_map_symbol *ams,
2092                              u64 addr, u64 phys_addr, u64 daddr_page_size)
2093 {
2094         struct addr_location al;
2095
2096         memset(&al, 0, sizeof(al));
2097
2098         thread__find_symbol(thread, m, addr, &al);
2099
2100         ams->addr = addr;
2101         ams->al_addr = al.addr;
2102         ams->al_level = al.level;
2103         ams->ms.maps = al.maps;
2104         ams->ms.sym = al.sym;
2105         ams->ms.map = al.map;
2106         ams->phys_addr = phys_addr;
2107         ams->data_page_size = daddr_page_size;
2108 }
2109
2110 struct mem_info *sample__resolve_mem(struct perf_sample *sample,
2111                                      struct addr_location *al)
2112 {
2113         struct mem_info *mi = mem_info__new();
2114
2115         if (!mi)
2116                 return NULL;
2117
2118         ip__resolve_ams(al->thread, &mi->iaddr, sample->ip);
2119         ip__resolve_data(al->thread, al->cpumode, &mi->daddr,
2120                          sample->addr, sample->phys_addr,
2121                          sample->data_page_size);
2122         mi->data_src.val = sample->data_src;
2123
2124         return mi;
2125 }
2126
2127 static char *callchain_srcline(struct map_symbol *ms, u64 ip)
2128 {
2129         struct map *map = ms->map;
2130         char *srcline = NULL;
2131
2132         if (!map || callchain_param.key == CCKEY_FUNCTION)
2133                 return srcline;
2134
2135         srcline = srcline__tree_find(&map->dso->srclines, ip);
2136         if (!srcline) {
2137                 bool show_sym = false;
2138                 bool show_addr = callchain_param.key == CCKEY_ADDRESS;
2139
2140                 srcline = get_srcline(map->dso, map__rip_2objdump(map, ip),
2141                                       ms->sym, show_sym, show_addr, ip);
2142                 srcline__tree_insert(&map->dso->srclines, ip, srcline);
2143         }
2144
2145         return srcline;
2146 }
2147
2148 struct iterations {
2149         int nr_loop_iter;
2150         u64 cycles;
2151 };
2152
2153 static int add_callchain_ip(struct thread *thread,
2154                             struct callchain_cursor *cursor,
2155                             struct symbol **parent,
2156                             struct addr_location *root_al,
2157                             u8 *cpumode,
2158                             u64 ip,
2159                             bool branch,
2160                             struct branch_flags *flags,
2161                             struct iterations *iter,
2162                             u64 branch_from)
2163 {
2164         struct map_symbol ms;
2165         struct addr_location al;
2166         int nr_loop_iter = 0;
2167         u64 iter_cycles = 0;
2168         const char *srcline = NULL;
2169
2170         al.filtered = 0;
2171         al.sym = NULL;
2172         al.srcline = NULL;
2173         if (!cpumode) {
2174                 thread__find_cpumode_addr_location(thread, ip, &al);
2175         } else {
2176                 if (ip >= PERF_CONTEXT_MAX) {
2177                         switch (ip) {
2178                         case PERF_CONTEXT_HV:
2179                                 *cpumode = PERF_RECORD_MISC_HYPERVISOR;
2180                                 break;
2181                         case PERF_CONTEXT_KERNEL:
2182                                 *cpumode = PERF_RECORD_MISC_KERNEL;
2183                                 break;
2184                         case PERF_CONTEXT_USER:
2185                                 *cpumode = PERF_RECORD_MISC_USER;
2186                                 break;
2187                         default:
2188                                 pr_debug("invalid callchain context: "
2189                                          "%"PRId64"\n", (s64) ip);
2190                                 /*
2191                                  * It seems the callchain is corrupted.
2192                                  * Discard all.
2193                                  */
2194                                 callchain_cursor_reset(cursor);
2195                                 return 1;
2196                         }
2197                         return 0;
2198                 }
2199                 thread__find_symbol(thread, *cpumode, ip, &al);
2200         }
2201
2202         if (al.sym != NULL) {
2203                 if (perf_hpp_list.parent && !*parent &&
2204                     symbol__match_regex(al.sym, &parent_regex))
2205                         *parent = al.sym;
2206                 else if (have_ignore_callees && root_al &&
2207                   symbol__match_regex(al.sym, &ignore_callees_regex)) {
2208                         /* Treat this symbol as the root,
2209                            forgetting its callees. */
2210                         *root_al = al;
2211                         callchain_cursor_reset(cursor);
2212                 }
2213         }
2214
2215         if (symbol_conf.hide_unresolved && al.sym == NULL)
2216                 return 0;
2217
2218         if (iter) {
2219                 nr_loop_iter = iter->nr_loop_iter;
2220                 iter_cycles = iter->cycles;
2221         }
2222
2223         ms.maps = al.maps;
2224         ms.map = al.map;
2225         ms.sym = al.sym;
2226         srcline = callchain_srcline(&ms, al.addr);
2227         return callchain_cursor_append(cursor, ip, &ms,
2228                                        branch, flags, nr_loop_iter,
2229                                        iter_cycles, branch_from, srcline);
2230 }
2231
2232 struct branch_info *sample__resolve_bstack(struct perf_sample *sample,
2233                                            struct addr_location *al)
2234 {
2235         unsigned int i;
2236         const struct branch_stack *bs = sample->branch_stack;
2237         struct branch_entry *entries = perf_sample__branch_entries(sample);
2238         struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info));
2239
2240         if (!bi)
2241                 return NULL;
2242
2243         for (i = 0; i < bs->nr; i++) {
2244                 ip__resolve_ams(al->thread, &bi[i].to, entries[i].to);
2245                 ip__resolve_ams(al->thread, &bi[i].from, entries[i].from);
2246                 bi[i].flags = entries[i].flags;
2247         }
2248         return bi;
2249 }
2250
2251 static void save_iterations(struct iterations *iter,
2252                             struct branch_entry *be, int nr)
2253 {
2254         int i;
2255
2256         iter->nr_loop_iter++;
2257         iter->cycles = 0;
2258
2259         for (i = 0; i < nr; i++)
2260                 iter->cycles += be[i].flags.cycles;
2261 }
2262
2263 #define CHASHSZ 127
2264 #define CHASHBITS 7
2265 #define NO_ENTRY 0xff
2266
2267 #define PERF_MAX_BRANCH_DEPTH 127
2268
2269 /* Remove loops. */
2270 static int remove_loops(struct branch_entry *l, int nr,
2271                         struct iterations *iter)
2272 {
2273         int i, j, off;
2274         unsigned char chash[CHASHSZ];
2275
2276         memset(chash, NO_ENTRY, sizeof(chash));
2277
2278         BUG_ON(PERF_MAX_BRANCH_DEPTH > 255);
2279
2280         for (i = 0; i < nr; i++) {
2281                 int h = hash_64(l[i].from, CHASHBITS) % CHASHSZ;
2282
2283                 /* no collision handling for now */
2284                 if (chash[h] == NO_ENTRY) {
2285                         chash[h] = i;
2286                 } else if (l[chash[h]].from == l[i].from) {
2287                         bool is_loop = true;
2288                         /* check if it is a real loop */
2289                         off = 0;
2290                         for (j = chash[h]; j < i && i + off < nr; j++, off++)
2291                                 if (l[j].from != l[i + off].from) {
2292                                         is_loop = false;
2293                                         break;
2294                                 }
2295                         if (is_loop) {
2296                                 j = nr - (i + off);
2297                                 if (j > 0) {
2298                                         save_iterations(iter + i + off,
2299                                                 l + i, off);
2300
2301                                         memmove(iter + i, iter + i + off,
2302                                                 j * sizeof(*iter));
2303
2304                                         memmove(l + i, l + i + off,
2305                                                 j * sizeof(*l));
2306                                 }
2307
2308                                 nr -= off;
2309                         }
2310                 }
2311         }
2312         return nr;
2313 }
2314
2315 static int lbr_callchain_add_kernel_ip(struct thread *thread,
2316                                        struct callchain_cursor *cursor,
2317                                        struct perf_sample *sample,
2318                                        struct symbol **parent,
2319                                        struct addr_location *root_al,
2320                                        u64 branch_from,
2321                                        bool callee, int end)
2322 {
2323         struct ip_callchain *chain = sample->callchain;
2324         u8 cpumode = PERF_RECORD_MISC_USER;
2325         int err, i;
2326
2327         if (callee) {
2328                 for (i = 0; i < end + 1; i++) {
2329                         err = add_callchain_ip(thread, cursor, parent,
2330                                                root_al, &cpumode, chain->ips[i],
2331                                                false, NULL, NULL, branch_from);
2332                         if (err)
2333                                 return err;
2334                 }
2335                 return 0;
2336         }
2337
2338         for (i = end; i >= 0; i--) {
2339                 err = add_callchain_ip(thread, cursor, parent,
2340                                        root_al, &cpumode, chain->ips[i],
2341                                        false, NULL, NULL, branch_from);
2342                 if (err)
2343                         return err;
2344         }
2345
2346         return 0;
2347 }
2348
2349 static void save_lbr_cursor_node(struct thread *thread,
2350                                  struct callchain_cursor *cursor,
2351                                  int idx)
2352 {
2353         struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
2354
2355         if (!lbr_stitch)
2356                 return;
2357
2358         if (cursor->pos == cursor->nr) {
2359                 lbr_stitch->prev_lbr_cursor[idx].valid = false;
2360                 return;
2361         }
2362
2363         if (!cursor->curr)
2364                 cursor->curr = cursor->first;
2365         else
2366                 cursor->curr = cursor->curr->next;
2367         memcpy(&lbr_stitch->prev_lbr_cursor[idx], cursor->curr,
2368                sizeof(struct callchain_cursor_node));
2369
2370         lbr_stitch->prev_lbr_cursor[idx].valid = true;
2371         cursor->pos++;
2372 }
2373
2374 static int lbr_callchain_add_lbr_ip(struct thread *thread,
2375                                     struct callchain_cursor *cursor,
2376                                     struct perf_sample *sample,
2377                                     struct symbol **parent,
2378                                     struct addr_location *root_al,
2379                                     u64 *branch_from,
2380                                     bool callee)
2381 {
2382         struct branch_stack *lbr_stack = sample->branch_stack;
2383         struct branch_entry *entries = perf_sample__branch_entries(sample);
2384         u8 cpumode = PERF_RECORD_MISC_USER;
2385         int lbr_nr = lbr_stack->nr;
2386         struct branch_flags *flags;
2387         int err, i;
2388         u64 ip;
2389
2390         /*
2391          * The curr and pos are not used in writing session. They are cleared
2392          * in callchain_cursor_commit() when the writing session is closed.
2393          * Using curr and pos to track the current cursor node.
2394          */
2395         if (thread->lbr_stitch) {
2396                 cursor->curr = NULL;
2397                 cursor->pos = cursor->nr;
2398                 if (cursor->nr) {
2399                         cursor->curr = cursor->first;
2400                         for (i = 0; i < (int)(cursor->nr - 1); i++)
2401                                 cursor->curr = cursor->curr->next;
2402                 }
2403         }
2404
2405         if (callee) {
2406                 /* Add LBR ip from first entries.to */
2407                 ip = entries[0].to;
2408                 flags = &entries[0].flags;
2409                 *branch_from = entries[0].from;
2410                 err = add_callchain_ip(thread, cursor, parent,
2411                                        root_al, &cpumode, ip,
2412                                        true, flags, NULL,
2413                                        *branch_from);
2414                 if (err)
2415                         return err;
2416
2417                 /*
2418                  * The number of cursor node increases.
2419                  * Move the current cursor node.
2420                  * But does not need to save current cursor node for entry 0.
2421                  * It's impossible to stitch the whole LBRs of previous sample.
2422                  */
2423                 if (thread->lbr_stitch && (cursor->pos != cursor->nr)) {
2424                         if (!cursor->curr)
2425                                 cursor->curr = cursor->first;
2426                         else
2427                                 cursor->curr = cursor->curr->next;
2428                         cursor->pos++;
2429                 }
2430
2431                 /* Add LBR ip from entries.from one by one. */
2432                 for (i = 0; i < lbr_nr; i++) {
2433                         ip = entries[i].from;
2434                         flags = &entries[i].flags;
2435                         err = add_callchain_ip(thread, cursor, parent,
2436                                                root_al, &cpumode, ip,
2437                                                true, flags, NULL,
2438                                                *branch_from);
2439                         if (err)
2440                                 return err;
2441                         save_lbr_cursor_node(thread, cursor, i);
2442                 }
2443                 return 0;
2444         }
2445
2446         /* Add LBR ip from entries.from one by one. */
2447         for (i = lbr_nr - 1; i >= 0; i--) {
2448                 ip = entries[i].from;
2449                 flags = &entries[i].flags;
2450                 err = add_callchain_ip(thread, cursor, parent,
2451                                        root_al, &cpumode, ip,
2452                                        true, flags, NULL,
2453                                        *branch_from);
2454                 if (err)
2455                         return err;
2456                 save_lbr_cursor_node(thread, cursor, i);
2457         }
2458
2459         /* Add LBR ip from first entries.to */
2460         ip = entries[0].to;
2461         flags = &entries[0].flags;
2462         *branch_from = entries[0].from;
2463         err = add_callchain_ip(thread, cursor, parent,
2464                                root_al, &cpumode, ip,
2465                                true, flags, NULL,
2466                                *branch_from);
2467         if (err)
2468                 return err;
2469
2470         return 0;
2471 }
2472
2473 static int lbr_callchain_add_stitched_lbr_ip(struct thread *thread,
2474                                              struct callchain_cursor *cursor)
2475 {
2476         struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
2477         struct callchain_cursor_node *cnode;
2478         struct stitch_list *stitch_node;
2479         int err;
2480
2481         list_for_each_entry(stitch_node, &lbr_stitch->lists, node) {
2482                 cnode = &stitch_node->cursor;
2483
2484                 err = callchain_cursor_append(cursor, cnode->ip,
2485                                               &cnode->ms,
2486                                               cnode->branch,
2487                                               &cnode->branch_flags,
2488                                               cnode->nr_loop_iter,
2489                                               cnode->iter_cycles,
2490                                               cnode->branch_from,
2491                                               cnode->srcline);
2492                 if (err)
2493                         return err;
2494         }
2495         return 0;
2496 }
2497
2498 static struct stitch_list *get_stitch_node(struct thread *thread)
2499 {
2500         struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
2501         struct stitch_list *stitch_node;
2502
2503         if (!list_empty(&lbr_stitch->free_lists)) {
2504                 stitch_node = list_first_entry(&lbr_stitch->free_lists,
2505                                                struct stitch_list, node);
2506                 list_del(&stitch_node->node);
2507
2508                 return stitch_node;
2509         }
2510
2511         return malloc(sizeof(struct stitch_list));
2512 }
2513
2514 static bool has_stitched_lbr(struct thread *thread,
2515                              struct perf_sample *cur,
2516                              struct perf_sample *prev,
2517                              unsigned int max_lbr,
2518                              bool callee)
2519 {
2520         struct branch_stack *cur_stack = cur->branch_stack;
2521         struct branch_entry *cur_entries = perf_sample__branch_entries(cur);
2522         struct branch_stack *prev_stack = prev->branch_stack;
2523         struct branch_entry *prev_entries = perf_sample__branch_entries(prev);
2524         struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
2525         int i, j, nr_identical_branches = 0;
2526         struct stitch_list *stitch_node;
2527         u64 cur_base, distance;
2528
2529         if (!cur_stack || !prev_stack)
2530                 return false;
2531
2532         /* Find the physical index of the base-of-stack for current sample. */
2533         cur_base = max_lbr - cur_stack->nr + cur_stack->hw_idx + 1;
2534
2535         distance = (prev_stack->hw_idx > cur_base) ? (prev_stack->hw_idx - cur_base) :
2536                                                      (max_lbr + prev_stack->hw_idx - cur_base);
2537         /* Previous sample has shorter stack. Nothing can be stitched. */
2538         if (distance + 1 > prev_stack->nr)
2539                 return false;
2540
2541         /*
2542          * Check if there are identical LBRs between two samples.
2543          * Identical LBRs must have same from, to and flags values. Also,
2544          * they have to be saved in the same LBR registers (same physical
2545          * index).
2546          *
2547          * Starts from the base-of-stack of current sample.
2548          */
2549         for (i = distance, j = cur_stack->nr - 1; (i >= 0) && (j >= 0); i--, j--) {
2550                 if ((prev_entries[i].from != cur_entries[j].from) ||
2551                     (prev_entries[i].to != cur_entries[j].to) ||
2552                     (prev_entries[i].flags.value != cur_entries[j].flags.value))
2553                         break;
2554                 nr_identical_branches++;
2555         }
2556
2557         if (!nr_identical_branches)
2558                 return false;
2559
2560         /*
2561          * Save the LBRs between the base-of-stack of previous sample
2562          * and the base-of-stack of current sample into lbr_stitch->lists.
2563          * These LBRs will be stitched later.
2564          */
2565         for (i = prev_stack->nr - 1; i > (int)distance; i--) {
2566
2567                 if (!lbr_stitch->prev_lbr_cursor[i].valid)
2568                         continue;
2569
2570                 stitch_node = get_stitch_node(thread);
2571                 if (!stitch_node)
2572                         return false;
2573
2574                 memcpy(&stitch_node->cursor, &lbr_stitch->prev_lbr_cursor[i],
2575                        sizeof(struct callchain_cursor_node));
2576
2577                 if (callee)
2578                         list_add(&stitch_node->node, &lbr_stitch->lists);
2579                 else
2580                         list_add_tail(&stitch_node->node, &lbr_stitch->lists);
2581         }
2582
2583         return true;
2584 }
2585
2586 static bool alloc_lbr_stitch(struct thread *thread, unsigned int max_lbr)
2587 {
2588         if (thread->lbr_stitch)
2589                 return true;
2590
2591         thread->lbr_stitch = zalloc(sizeof(*thread->lbr_stitch));
2592         if (!thread->lbr_stitch)
2593                 goto err;
2594
2595         thread->lbr_stitch->prev_lbr_cursor = calloc(max_lbr + 1, sizeof(struct callchain_cursor_node));
2596         if (!thread->lbr_stitch->prev_lbr_cursor)
2597                 goto free_lbr_stitch;
2598
2599         INIT_LIST_HEAD(&thread->lbr_stitch->lists);
2600         INIT_LIST_HEAD(&thread->lbr_stitch->free_lists);
2601
2602         return true;
2603
2604 free_lbr_stitch:
2605         zfree(&thread->lbr_stitch);
2606 err:
2607         pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n");
2608         thread->lbr_stitch_enable = false;
2609         return false;
2610 }
2611
2612 /*
2613  * Resolve LBR callstack chain sample
2614  * Return:
2615  * 1 on success get LBR callchain information
2616  * 0 no available LBR callchain information, should try fp
2617  * negative error code on other errors.
2618  */
2619 static int resolve_lbr_callchain_sample(struct thread *thread,
2620                                         struct callchain_cursor *cursor,
2621                                         struct perf_sample *sample,
2622                                         struct symbol **parent,
2623                                         struct addr_location *root_al,
2624                                         int max_stack,
2625                                         unsigned int max_lbr)
2626 {
2627         bool callee = (callchain_param.order == ORDER_CALLEE);
2628         struct ip_callchain *chain = sample->callchain;
2629         int chain_nr = min(max_stack, (int)chain->nr), i;
2630         struct lbr_stitch *lbr_stitch;
2631         bool stitched_lbr = false;
2632         u64 branch_from = 0;
2633         int err;
2634
2635         for (i = 0; i < chain_nr; i++) {
2636                 if (chain->ips[i] == PERF_CONTEXT_USER)
2637                         break;
2638         }
2639
2640         /* LBR only affects the user callchain */
2641         if (i == chain_nr)
2642                 return 0;
2643
2644         if (thread->lbr_stitch_enable && !sample->no_hw_idx &&
2645             (max_lbr > 0) && alloc_lbr_stitch(thread, max_lbr)) {
2646                 lbr_stitch = thread->lbr_stitch;
2647
2648                 stitched_lbr = has_stitched_lbr(thread, sample,
2649                                                 &lbr_stitch->prev_sample,
2650                                                 max_lbr, callee);
2651
2652                 if (!stitched_lbr && !list_empty(&lbr_stitch->lists)) {
2653                         list_replace_init(&lbr_stitch->lists,
2654                                           &lbr_stitch->free_lists);
2655                 }
2656                 memcpy(&lbr_stitch->prev_sample, sample, sizeof(*sample));
2657         }
2658
2659         if (callee) {
2660                 /* Add kernel ip */
2661                 err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
2662                                                   parent, root_al, branch_from,
2663                                                   true, i);
2664                 if (err)
2665                         goto error;
2666
2667                 err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
2668                                                root_al, &branch_from, true);
2669                 if (err)
2670                         goto error;
2671
2672                 if (stitched_lbr) {
2673                         err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
2674                         if (err)
2675                                 goto error;
2676                 }
2677
2678         } else {
2679                 if (stitched_lbr) {
2680                         err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
2681                         if (err)
2682                                 goto error;
2683                 }
2684                 err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
2685                                                root_al, &branch_from, false);
2686                 if (err)
2687                         goto error;
2688
2689                 /* Add kernel ip */
2690                 err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
2691                                                   parent, root_al, branch_from,
2692                                                   false, i);
2693                 if (err)
2694                         goto error;
2695         }
2696         return 1;
2697
2698 error:
2699         return (err < 0) ? err : 0;
2700 }
2701
2702 static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread,
2703                              struct callchain_cursor *cursor,
2704                              struct symbol **parent,
2705                              struct addr_location *root_al,
2706                              u8 *cpumode, int ent)
2707 {
2708         int err = 0;
2709
2710         while (--ent >= 0) {
2711                 u64 ip = chain->ips[ent];
2712
2713                 if (ip >= PERF_CONTEXT_MAX) {
2714                         err = add_callchain_ip(thread, cursor, parent,
2715                                                root_al, cpumode, ip,
2716                                                false, NULL, NULL, 0);
2717                         break;
2718                 }
2719         }
2720         return err;
2721 }
2722
2723 static u64 get_leaf_frame_caller(struct perf_sample *sample,
2724                 struct thread *thread, int usr_idx)
2725 {
2726         if (machine__normalized_is(thread->maps->machine, "arm64"))
2727                 return get_leaf_frame_caller_aarch64(sample, thread, usr_idx);
2728         else
2729                 return 0;
2730 }
2731
2732 static int thread__resolve_callchain_sample(struct thread *thread,
2733                                             struct callchain_cursor *cursor,
2734                                             struct evsel *evsel,
2735                                             struct perf_sample *sample,
2736                                             struct symbol **parent,
2737                                             struct addr_location *root_al,
2738                                             int max_stack)
2739 {
2740         struct branch_stack *branch = sample->branch_stack;
2741         struct branch_entry *entries = perf_sample__branch_entries(sample);
2742         struct ip_callchain *chain = sample->callchain;
2743         int chain_nr = 0;
2744         u8 cpumode = PERF_RECORD_MISC_USER;
2745         int i, j, err, nr_entries, usr_idx;
2746         int skip_idx = -1;
2747         int first_call = 0;
2748         u64 leaf_frame_caller;
2749
2750         if (chain)
2751                 chain_nr = chain->nr;
2752
2753         if (evsel__has_branch_callstack(evsel)) {
2754                 struct perf_env *env = evsel__env(evsel);
2755
2756                 err = resolve_lbr_callchain_sample(thread, cursor, sample, parent,
2757                                                    root_al, max_stack,
2758                                                    !env ? 0 : env->max_branches);
2759                 if (err)
2760                         return (err < 0) ? err : 0;
2761         }
2762
2763         /*
2764          * Based on DWARF debug information, some architectures skip
2765          * a callchain entry saved by the kernel.
2766          */
2767         skip_idx = arch_skip_callchain_idx(thread, chain);
2768
2769         /*
2770          * Add branches to call stack for easier browsing. This gives
2771          * more context for a sample than just the callers.
2772          *
2773          * This uses individual histograms of paths compared to the
2774          * aggregated histograms the normal LBR mode uses.
2775          *
2776          * Limitations for now:
2777          * - No extra filters
2778          * - No annotations (should annotate somehow)
2779          */
2780
2781         if (branch && callchain_param.branch_callstack) {
2782                 int nr = min(max_stack, (int)branch->nr);
2783                 struct branch_entry be[nr];
2784                 struct iterations iter[nr];
2785
2786                 if (branch->nr > PERF_MAX_BRANCH_DEPTH) {
2787                         pr_warning("corrupted branch chain. skipping...\n");
2788                         goto check_calls;
2789                 }
2790
2791                 for (i = 0; i < nr; i++) {
2792                         if (callchain_param.order == ORDER_CALLEE) {
2793                                 be[i] = entries[i];
2794
2795                                 if (chain == NULL)
2796                                         continue;
2797
2798                                 /*
2799                                  * Check for overlap into the callchain.
2800                                  * The return address is one off compared to
2801                                  * the branch entry. To adjust for this
2802                                  * assume the calling instruction is not longer
2803                                  * than 8 bytes.
2804                                  */
2805                                 if (i == skip_idx ||
2806                                     chain->ips[first_call] >= PERF_CONTEXT_MAX)
2807                                         first_call++;
2808                                 else if (be[i].from < chain->ips[first_call] &&
2809                                     be[i].from >= chain->ips[first_call] - 8)
2810                                         first_call++;
2811                         } else
2812                                 be[i] = entries[branch->nr - i - 1];
2813                 }
2814
2815                 memset(iter, 0, sizeof(struct iterations) * nr);
2816                 nr = remove_loops(be, nr, iter);
2817
2818                 for (i = 0; i < nr; i++) {
2819                         err = add_callchain_ip(thread, cursor, parent,
2820                                                root_al,
2821                                                NULL, be[i].to,
2822                                                true, &be[i].flags,
2823                                                NULL, be[i].from);
2824
2825                         if (!err)
2826                                 err = add_callchain_ip(thread, cursor, parent, root_al,
2827                                                        NULL, be[i].from,
2828                                                        true, &be[i].flags,
2829                                                        &iter[i], 0);
2830                         if (err == -EINVAL)
2831                                 break;
2832                         if (err)
2833                                 return err;
2834                 }
2835
2836                 if (chain_nr == 0)
2837                         return 0;
2838
2839                 chain_nr -= nr;
2840         }
2841
2842 check_calls:
2843         if (chain && callchain_param.order != ORDER_CALLEE) {
2844                 err = find_prev_cpumode(chain, thread, cursor, parent, root_al,
2845                                         &cpumode, chain->nr - first_call);
2846                 if (err)
2847                         return (err < 0) ? err : 0;
2848         }
2849         for (i = first_call, nr_entries = 0;
2850              i < chain_nr && nr_entries < max_stack; i++) {
2851                 u64 ip;
2852
2853                 if (callchain_param.order == ORDER_CALLEE)
2854                         j = i;
2855                 else
2856                         j = chain->nr - i - 1;
2857
2858 #ifdef HAVE_SKIP_CALLCHAIN_IDX
2859                 if (j == skip_idx)
2860                         continue;
2861 #endif
2862                 ip = chain->ips[j];
2863                 if (ip < PERF_CONTEXT_MAX)
2864                        ++nr_entries;
2865                 else if (callchain_param.order != ORDER_CALLEE) {
2866                         err = find_prev_cpumode(chain, thread, cursor, parent,
2867                                                 root_al, &cpumode, j);
2868                         if (err)
2869                                 return (err < 0) ? err : 0;
2870                         continue;
2871                 }
2872
2873                 /*
2874                  * PERF_CONTEXT_USER allows us to locate where the user stack ends.
2875                  * Depending on callchain_param.order and the position of PERF_CONTEXT_USER,
2876                  * the index will be different in order to add the missing frame
2877                  * at the right place.
2878                  */
2879
2880                 usr_idx = callchain_param.order == ORDER_CALLEE ? j-2 : j-1;
2881
2882                 if (usr_idx >= 0 && chain->ips[usr_idx] == PERF_CONTEXT_USER) {
2883
2884                         leaf_frame_caller = get_leaf_frame_caller(sample, thread, usr_idx);
2885
2886                         /*
2887                          * check if leaf_frame_Caller != ip to not add the same
2888                          * value twice.
2889                          */
2890
2891                         if (leaf_frame_caller && leaf_frame_caller != ip) {
2892
2893                                 err = add_callchain_ip(thread, cursor, parent,
2894                                                root_al, &cpumode, leaf_frame_caller,
2895                                                false, NULL, NULL, 0);
2896                                 if (err)
2897                                         return (err < 0) ? err : 0;
2898                         }
2899                 }
2900
2901                 err = add_callchain_ip(thread, cursor, parent,
2902                                        root_al, &cpumode, ip,
2903                                        false, NULL, NULL, 0);
2904
2905                 if (err)
2906                         return (err < 0) ? err : 0;
2907         }
2908
2909         return 0;
2910 }
2911
2912 static int append_inlines(struct callchain_cursor *cursor, struct map_symbol *ms, u64 ip)
2913 {
2914         struct symbol *sym = ms->sym;
2915         struct map *map = ms->map;
2916         struct inline_node *inline_node;
2917         struct inline_list *ilist;
2918         u64 addr;
2919         int ret = 1;
2920
2921         if (!symbol_conf.inline_name || !map || !sym)
2922                 return ret;
2923
2924         addr = map__map_ip(map, ip);
2925         addr = map__rip_2objdump(map, addr);
2926
2927         inline_node = inlines__tree_find(&map->dso->inlined_nodes, addr);
2928         if (!inline_node) {
2929                 inline_node = dso__parse_addr_inlines(map->dso, addr, sym);
2930                 if (!inline_node)
2931                         return ret;
2932                 inlines__tree_insert(&map->dso->inlined_nodes, inline_node);
2933         }
2934
2935         list_for_each_entry(ilist, &inline_node->val, list) {
2936                 struct map_symbol ilist_ms = {
2937                         .maps = ms->maps,
2938                         .map = map,
2939                         .sym = ilist->symbol,
2940                 };
2941                 ret = callchain_cursor_append(cursor, ip, &ilist_ms, false,
2942                                               NULL, 0, 0, 0, ilist->srcline);
2943
2944                 if (ret != 0)
2945                         return ret;
2946         }
2947
2948         return ret;
2949 }
2950
2951 static int unwind_entry(struct unwind_entry *entry, void *arg)
2952 {
2953         struct callchain_cursor *cursor = arg;
2954         const char *srcline = NULL;
2955         u64 addr = entry->ip;
2956
2957         if (symbol_conf.hide_unresolved && entry->ms.sym == NULL)
2958                 return 0;
2959
2960         if (append_inlines(cursor, &entry->ms, entry->ip) == 0)
2961                 return 0;
2962
2963         /*
2964          * Convert entry->ip from a virtual address to an offset in
2965          * its corresponding binary.
2966          */
2967         if (entry->ms.map)
2968                 addr = map__map_ip(entry->ms.map, entry->ip);
2969
2970         srcline = callchain_srcline(&entry->ms, addr);
2971         return callchain_cursor_append(cursor, entry->ip, &entry->ms,
2972                                        false, NULL, 0, 0, 0, srcline);
2973 }
2974
2975 static int thread__resolve_callchain_unwind(struct thread *thread,
2976                                             struct callchain_cursor *cursor,
2977                                             struct evsel *evsel,
2978                                             struct perf_sample *sample,
2979                                             int max_stack)
2980 {
2981         /* Can we do dwarf post unwind? */
2982         if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) &&
2983               (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER)))
2984                 return 0;
2985
2986         /* Bail out if nothing was captured. */
2987         if ((!sample->user_regs.regs) ||
2988             (!sample->user_stack.size))
2989                 return 0;
2990
2991         return unwind__get_entries(unwind_entry, cursor,
2992                                    thread, sample, max_stack, false);
2993 }
2994
2995 int thread__resolve_callchain(struct thread *thread,
2996                               struct callchain_cursor *cursor,
2997                               struct evsel *evsel,
2998                               struct perf_sample *sample,
2999                               struct symbol **parent,
3000                               struct addr_location *root_al,
3001                               int max_stack)
3002 {
3003         int ret = 0;
3004
3005         callchain_cursor_reset(cursor);
3006
3007         if (callchain_param.order == ORDER_CALLEE) {
3008                 ret = thread__resolve_callchain_sample(thread, cursor,
3009                                                        evsel, sample,
3010                                                        parent, root_al,
3011                                                        max_stack);
3012                 if (ret)
3013                         return ret;
3014                 ret = thread__resolve_callchain_unwind(thread, cursor,
3015                                                        evsel, sample,
3016                                                        max_stack);
3017         } else {
3018                 ret = thread__resolve_callchain_unwind(thread, cursor,
3019                                                        evsel, sample,
3020                                                        max_stack);
3021                 if (ret)
3022                         return ret;
3023                 ret = thread__resolve_callchain_sample(thread, cursor,
3024                                                        evsel, sample,
3025                                                        parent, root_al,
3026                                                        max_stack);
3027         }
3028
3029         return ret;
3030 }
3031
3032 int machine__for_each_thread(struct machine *machine,
3033                              int (*fn)(struct thread *thread, void *p),
3034                              void *priv)
3035 {
3036         struct threads *threads;
3037         struct rb_node *nd;
3038         struct thread *thread;
3039         int rc = 0;
3040         int i;
3041
3042         for (i = 0; i < THREADS__TABLE_SIZE; i++) {
3043                 threads = &machine->threads[i];
3044                 for (nd = rb_first_cached(&threads->entries); nd;
3045                      nd = rb_next(nd)) {
3046                         thread = rb_entry(nd, struct thread, rb_node);
3047                         rc = fn(thread, priv);
3048                         if (rc != 0)
3049                                 return rc;
3050                 }
3051
3052                 list_for_each_entry(thread, &threads->dead, node) {
3053                         rc = fn(thread, priv);
3054                         if (rc != 0)
3055                                 return rc;
3056                 }
3057         }
3058         return rc;
3059 }
3060
3061 int machines__for_each_thread(struct machines *machines,
3062                               int (*fn)(struct thread *thread, void *p),
3063                               void *priv)
3064 {
3065         struct rb_node *nd;
3066         int rc = 0;
3067
3068         rc = machine__for_each_thread(&machines->host, fn, priv);
3069         if (rc != 0)
3070                 return rc;
3071
3072         for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
3073                 struct machine *machine = rb_entry(nd, struct machine, rb_node);
3074
3075                 rc = machine__for_each_thread(machine, fn, priv);
3076                 if (rc != 0)
3077                         return rc;
3078         }
3079         return rc;
3080 }
3081
3082 pid_t machine__get_current_tid(struct machine *machine, int cpu)
3083 {
3084         int nr_cpus = min(machine->env->nr_cpus_avail, MAX_NR_CPUS);
3085
3086         if (cpu < 0 || cpu >= nr_cpus || !machine->current_tid)
3087                 return -1;
3088
3089         return machine->current_tid[cpu];
3090 }
3091
3092 int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid,
3093                              pid_t tid)
3094 {
3095         struct thread *thread;
3096         int nr_cpus = min(machine->env->nr_cpus_avail, MAX_NR_CPUS);
3097
3098         if (cpu < 0)
3099                 return -EINVAL;
3100
3101         if (!machine->current_tid) {
3102                 int i;
3103
3104                 machine->current_tid = calloc(nr_cpus, sizeof(pid_t));
3105                 if (!machine->current_tid)
3106                         return -ENOMEM;
3107                 for (i = 0; i < nr_cpus; i++)
3108                         machine->current_tid[i] = -1;
3109         }
3110
3111         if (cpu >= nr_cpus) {
3112                 pr_err("Requested CPU %d too large. ", cpu);
3113                 pr_err("Consider raising MAX_NR_CPUS\n");
3114                 return -EINVAL;
3115         }
3116
3117         machine->current_tid[cpu] = tid;
3118
3119         thread = machine__findnew_thread(machine, pid, tid);
3120         if (!thread)
3121                 return -ENOMEM;
3122
3123         thread->cpu = cpu;
3124         thread__put(thread);
3125
3126         return 0;
3127 }
3128
3129 /*
3130  * Compares the raw arch string. N.B. see instead perf_env__arch() or
3131  * machine__normalized_is() if a normalized arch is needed.
3132  */
3133 bool machine__is(struct machine *machine, const char *arch)
3134 {
3135         return machine && !strcmp(perf_env__raw_arch(machine->env), arch);
3136 }
3137
3138 bool machine__normalized_is(struct machine *machine, const char *arch)
3139 {
3140         return machine && !strcmp(perf_env__arch(machine->env), arch);
3141 }
3142
3143 int machine__nr_cpus_avail(struct machine *machine)
3144 {
3145         return machine ? perf_env__nr_cpus_avail(machine->env) : 0;
3146 }
3147
3148 int machine__get_kernel_start(struct machine *machine)
3149 {
3150         struct map *map = machine__kernel_map(machine);
3151         int err = 0;
3152
3153         /*
3154          * The only addresses above 2^63 are kernel addresses of a 64-bit
3155          * kernel.  Note that addresses are unsigned so that on a 32-bit system
3156          * all addresses including kernel addresses are less than 2^32.  In
3157          * that case (32-bit system), if the kernel mapping is unknown, all
3158          * addresses will be assumed to be in user space - see
3159          * machine__kernel_ip().
3160          */
3161         machine->kernel_start = 1ULL << 63;
3162         if (map) {
3163                 err = map__load(map);
3164                 /*
3165                  * On x86_64, PTI entry trampolines are less than the
3166                  * start of kernel text, but still above 2^63. So leave
3167                  * kernel_start = 1ULL << 63 for x86_64.
3168                  */
3169                 if (!err && !machine__is(machine, "x86_64"))
3170                         machine->kernel_start = map->start;
3171         }
3172         return err;
3173 }
3174
3175 u8 machine__addr_cpumode(struct machine *machine, u8 cpumode, u64 addr)
3176 {
3177         u8 addr_cpumode = cpumode;
3178         bool kernel_ip;
3179
3180         if (!machine->single_address_space)
3181                 goto out;
3182
3183         kernel_ip = machine__kernel_ip(machine, addr);
3184         switch (cpumode) {
3185         case PERF_RECORD_MISC_KERNEL:
3186         case PERF_RECORD_MISC_USER:
3187                 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_KERNEL :
3188                                            PERF_RECORD_MISC_USER;
3189                 break;
3190         case PERF_RECORD_MISC_GUEST_KERNEL:
3191         case PERF_RECORD_MISC_GUEST_USER:
3192                 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_GUEST_KERNEL :
3193                                            PERF_RECORD_MISC_GUEST_USER;
3194                 break;
3195         default:
3196                 break;
3197         }
3198 out:
3199         return addr_cpumode;
3200 }
3201
3202 struct dso *machine__findnew_dso_id(struct machine *machine, const char *filename, struct dso_id *id)
3203 {
3204         return dsos__findnew_id(&machine->dsos, filename, id);
3205 }
3206
3207 struct dso *machine__findnew_dso(struct machine *machine, const char *filename)
3208 {
3209         return machine__findnew_dso_id(machine, filename, NULL);
3210 }
3211
3212 char *machine__resolve_kernel_addr(void *vmachine, unsigned long long *addrp, char **modp)
3213 {
3214         struct machine *machine = vmachine;
3215         struct map *map;
3216         struct symbol *sym = machine__find_kernel_symbol(machine, *addrp, &map);
3217
3218         if (sym == NULL)
3219                 return NULL;
3220
3221         *modp = __map__is_kmodule(map) ? (char *)map->dso->short_name : NULL;
3222         *addrp = map->unmap_ip(map, sym->start);
3223         return sym->name;
3224 }
3225
3226 int machine__for_each_dso(struct machine *machine, machine__dso_t fn, void *priv)
3227 {
3228         struct dso *pos;
3229         int err = 0;
3230
3231         list_for_each_entry(pos, &machine->dsos.head, node) {
3232                 if (fn(pos, machine, priv))
3233                         err = -1;
3234         }
3235         return err;
3236 }