tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <inttypes.h>
  17 #include <stdio.h>
  18 #include <stdbool.h>
  19 #include <errno.h>
  20 #include <linux/kernel.h>
  21 #include <linux/types.h>
  22
  23 #include "../perf.h"
  24 #include "session.h"
  25 #include "machine.h"
  26 #include "memswap.h"
  27 #include "sort.h"
  28 #include "tool.h"
  29 #include "event.h"
  30 #include "evlist.h"
  31 #include "evsel.h"
  32 #include "map.h"
  33 #include "color.h"
  34 #include "util.h"
  35 #include "thread.h"
  36 #include "thread-stack.h"
  37 #include "symbol.h"
  38 #include "callchain.h"
  39 #include "dso.h"
  40 #include "debug.h"
  41 #include "auxtrace.h"
  42 #include "tsc.h"
  43 #include "intel-pt.h"
  44 #include "config.h"
  45
  46 #include "intel-pt-decoder/intel-pt-log.h"
  47 #include "intel-pt-decoder/intel-pt-decoder.h"
  48 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  49 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  50
  51 #define MAX_TIMESTAMP (~0ULL)
  52
  53 struct intel_pt {
  54         struct auxtrace auxtrace;
  55         struct auxtrace_queues queues;
  56         struct auxtrace_heap heap;
  57         u32 auxtrace_type;
  58         struct perf_session *session;
  59         struct machine *machine;
  60         struct perf_evsel *switch_evsel;
  61         struct thread *unknown_thread;
  62         bool timeless_decoding;
  63         bool sampling_mode;
  64         bool snapshot_mode;
  65         bool per_cpu_mmaps;
  66         bool have_tsc;
  67         bool data_queued;
  68         bool est_tsc;
  69         bool sync_switch;
  70         bool mispred_all;
  71         int have_sched_switch;
  72         u32 pmu_type;
  73         u64 kernel_start;
  74         u64 switch_ip;
  75         u64 ptss_ip;
  76
  77         struct perf_tsc_conversion tc;
  78         bool cap_user_time_zero;
  79
  80         struct itrace_synth_opts synth_opts;
  81
  82         bool sample_instructions;
  83         u64 instructions_sample_type;
  84         u64 instructions_id;
  85
  86         bool sample_branches;
  87         u32 branches_filter;
  88         u64 branches_sample_type;
  89         u64 branches_id;
  90
  91         bool sample_transactions;
  92         u64 transactions_sample_type;
  93         u64 transactions_id;
  94
  95         bool synth_needs_swap;
  96
  97         u64 tsc_bit;
  98         u64 mtc_bit;
  99         u64 mtc_freq_bits;
 100         u32 tsc_ctc_ratio_n;
 101         u32 tsc_ctc_ratio_d;
 102         u64 cyc_bit;
 103         u64 noretcomp_bit;
 104         unsigned max_non_turbo_ratio;
 105
 106         unsigned long num_events;
 107
 108         char *filter;
 109         struct addr_filters filts;
 110 };
 111
 112 enum switch_state {
 113         INTEL_PT_SS_NOT_TRACING,
 114         INTEL_PT_SS_UNKNOWN,
 115         INTEL_PT_SS_TRACING,
 116         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 117         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 118 };
 119
 120 struct intel_pt_queue {
 121         struct intel_pt *pt;
 122         unsigned int queue_nr;
 123         struct auxtrace_buffer *buffer;
 124         void *decoder;
 125         const struct intel_pt_state *state;
 126         struct ip_callchain *chain;
 127         struct branch_stack *last_branch;
 128         struct branch_stack *last_branch_rb;
 129         size_t last_branch_pos;
 130         union perf_event *event_buf;
 131         bool on_heap;
 132         bool stop;
 133         bool step_through_buffers;
 134         bool use_buffer_pid_tid;
 135         pid_t pid, tid;
 136         int cpu;
 137         int switch_state;
 138         pid_t next_tid;
 139         struct thread *thread;
 140         bool exclude_kernel;
 141         bool have_sample;
 142         u64 time;
 143         u64 timestamp;
 144         u32 flags;
 145         u16 insn_len;
 146         u64 last_insn_cnt;
 147         char insn[INTEL_PT_INSN_BUF_SZ];
 148 };
 149
 150 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 151                           unsigned char *buf, size_t len)
 152 {
 153         struct intel_pt_pkt packet;
 154         size_t pos = 0;
 155         int ret, pkt_len, i;
 156         char desc[INTEL_PT_PKT_DESC_MAX];
 157         const char *color = PERF_COLOR_BLUE;
 158
 159         color_fprintf(stdout, color,
 160                       ". ... Intel Processor Trace data: size %zu bytes\n",
 161                       len);
 162
 163         while (len) {
 164                 ret = intel_pt_get_packet(buf, len, &packet);
 165                 if (ret > 0)
 166                         pkt_len = ret;
 167                 else
 168                         pkt_len = 1;
 169                 printf(".");
 170                 color_fprintf(stdout, color, "  %08x: ", pos);
 171                 for (i = 0; i < pkt_len; i++)
 172                         color_fprintf(stdout, color, " %02x", buf[i]);
 173                 for (; i < 16; i++)
 174                         color_fprintf(stdout, color, "   ");
 175                 if (ret > 0) {
 176                         ret = intel_pt_pkt_desc(&packet, desc,
 177                                                 INTEL_PT_PKT_DESC_MAX);
 178                         if (ret > 0)
 179                                 color_fprintf(stdout, color, " %s\n", desc);
 180                 } else {
 181                         color_fprintf(stdout, color, " Bad packet!\n");
 182                 }
 183                 pos += pkt_len;
 184                 buf += pkt_len;
 185                 len -= pkt_len;
 186         }
 187 }
 188
 189 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 190                                 size_t len)
 191 {
 192         printf(".\n");
 193         intel_pt_dump(pt, buf, len);
 194 }
 195
 196 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 197                                    struct auxtrace_buffer *b)
 198 {
 199         void *start;
 200
 201         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 202                                       pt->have_tsc);
 203         if (!start)
 204                 return -EINVAL;
 205         b->use_size = b->data + b->size - start;
 206         b->use_data = start;
 207         return 0;
 208 }
 209
 210 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 211                                         struct auxtrace_queue *queue,
 212                                         struct auxtrace_buffer *buffer)
 213 {
 214         if (queue->cpu == -1 && buffer->cpu != -1)
 215                 ptq->cpu = buffer->cpu;
 216
 217         ptq->pid = buffer->pid;
 218         ptq->tid = buffer->tid;
 219
 220         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 221                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 222
 223         thread__zput(ptq->thread);
 224
 225         if (ptq->tid != -1) {
 226                 if (ptq->pid != -1)
 227                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 228                                                               ptq->pid,
 229                                                               ptq->tid);
 230                 else
 231                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 232                                                            ptq->tid);
 233         }
 234 }
 235
 236 /* This function assumes data is processed sequentially only */
 237 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 238 {
 239         struct intel_pt_queue *ptq = data;
 240         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 241         struct auxtrace_queue *queue;
 242
 243         if (ptq->stop) {
 244                 b->len = 0;
 245                 return 0;
 246         }
 247
 248         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 249 next:
 250         buffer = auxtrace_buffer__next(queue, buffer);
 251         if (!buffer) {
 252                 if (old_buffer)
 253                         auxtrace_buffer__drop_data(old_buffer);
 254                 b->len = 0;
 255                 return 0;
 256         }
 257
 258         ptq->buffer = buffer;
 259
 260         if (!buffer->data) {
 261                 int fd = perf_data_file__fd(ptq->pt->session->file);
 262
 263                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 264                 if (!buffer->data)
 265                         return -ENOMEM;
 266         }
 267
 268         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 269             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 270                 return -ENOMEM;
 271
 272         if (buffer->use_data) {
 273                 b->len = buffer->use_size;
 274                 b->buf = buffer->use_data;
 275         } else {
 276                 b->len = buffer->size;
 277                 b->buf = buffer->data;
 278         }
 279         b->ref_timestamp = buffer->reference;
 280
 281         /*
 282          * If in snapshot mode and the buffer has no usable data, get next
 283          * buffer and again check overlap against old_buffer.
 284          */
 285         if (ptq->pt->snapshot_mode && !b->len)
 286                 goto next;
 287
 288         if (old_buffer)
 289                 auxtrace_buffer__drop_data(old_buffer);
 290
 291         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 292                                                       !buffer->consecutive)) {
 293                 b->consecutive = false;
 294                 b->trace_nr = buffer->buffer_nr + 1;
 295         } else {
 296                 b->consecutive = true;
 297         }
 298
 299         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 300                                         ptq->tid != buffer->tid))
 301                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 302
 303         if (ptq->step_through_buffers)
 304                 ptq->stop = true;
 305
 306         if (!b->len)
 307                 return intel_pt_get_trace(b, data);
 308
 309         return 0;
 310 }
 311
 312 struct intel_pt_cache_entry {
 313         struct auxtrace_cache_entry     entry;
 314         u64                             insn_cnt;
 315         u64                             byte_cnt;
 316         enum intel_pt_insn_op           op;
 317         enum intel_pt_insn_branch       branch;
 318         int                             length;
 319         int32_t                         rel;
 320         char                            insn[INTEL_PT_INSN_BUF_SZ];
 321 };
 322
 323 static int intel_pt_config_div(const char *var, const char *value, void *data)
 324 {
 325         int *d = data;
 326         long val;
 327
 328         if (!strcmp(var, "intel-pt.cache-divisor")) {
 329                 val = strtol(value, NULL, 0);
 330                 if (val > 0 && val <= INT_MAX)
 331                         *d = val;
 332         }
 333
 334         return 0;
 335 }
 336
 337 static int intel_pt_cache_divisor(void)
 338 {
 339         static int d;
 340
 341         if (d)
 342                 return d;
 343
 344         perf_config(intel_pt_config_div, &d);
 345
 346         if (!d)
 347                 d = 64;
 348
 349         return d;
 350 }
 351
 352 static unsigned int intel_pt_cache_size(struct dso *dso,
 353                                         struct machine *machine)
 354 {
 355         off_t size;
 356
 357         size = dso__data_size(dso, machine);
 358         size /= intel_pt_cache_divisor();
 359         if (size < 1000)
 360                 return 10;
 361         if (size > (1 << 21))
 362                 return 21;
 363         return 32 - __builtin_clz(size);
 364 }
 365
 366 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 367                                              struct machine *machine)
 368 {
 369         struct auxtrace_cache *c;
 370         unsigned int bits;
 371
 372         if (dso->auxtrace_cache)
 373                 return dso->auxtrace_cache;
 374
 375         bits = intel_pt_cache_size(dso, machine);
 376
 377         /* Ignoring cache creation failure */
 378         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 379
 380         dso->auxtrace_cache = c;
 381
 382         return c;
 383 }
 384
 385 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 386                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 387                               struct intel_pt_insn *intel_pt_insn)
 388 {
 389         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 390         struct intel_pt_cache_entry *e;
 391         int err;
 392
 393         if (!c)
 394                 return -ENOMEM;
 395
 396         e = auxtrace_cache__alloc_entry(c);
 397         if (!e)
 398                 return -ENOMEM;
 399
 400         e->insn_cnt = insn_cnt;
 401         e->byte_cnt = byte_cnt;
 402         e->op = intel_pt_insn->op;
 403         e->branch = intel_pt_insn->branch;
 404         e->length = intel_pt_insn->length;
 405         e->rel = intel_pt_insn->rel;
 406         memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
 407
 408         err = auxtrace_cache__add(c, offset, &e->entry);
 409         if (err)
 410                 auxtrace_cache__free_entry(c, e);
 411
 412         return err;
 413 }
 414
 415 static struct intel_pt_cache_entry *
 416 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 417 {
 418         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 419
 420         if (!c)
 421                 return NULL;
 422
 423         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 424 }
 425
 426 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 427                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 428                                    uint64_t to_ip, uint64_t max_insn_cnt,
 429                                    void *data)
 430 {
 431         struct intel_pt_queue *ptq = data;
 432         struct machine *machine = ptq->pt->machine;
 433         struct thread *thread;
 434         struct addr_location al;
 435         unsigned char buf[INTEL_PT_INSN_BUF_SZ];
 436         ssize_t len;
 437         int x86_64;
 438         u8 cpumode;
 439         u64 offset, start_offset, start_ip;
 440         u64 insn_cnt = 0;
 441         bool one_map = true;
 442
 443         intel_pt_insn->length = 0;
 444
 445         if (to_ip && *ip == to_ip)
 446                 goto out_no_cache;
 447
 448         if (*ip >= ptq->pt->kernel_start)
 449                 cpumode = PERF_RECORD_MISC_KERNEL;
 450         else
 451                 cpumode = PERF_RECORD_MISC_USER;
 452
 453         thread = ptq->thread;
 454         if (!thread) {
 455                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 456                         return -EINVAL;
 457                 thread = ptq->pt->unknown_thread;
 458         }
 459
 460         while (1) {
 461                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 462                 if (!al.map || !al.map->dso)
 463                         return -EINVAL;
 464
 465                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 466                     dso__data_status_seen(al.map->dso,
 467                                           DSO_DATA_STATUS_SEEN_ITRACE))
 468                         return -ENOENT;
 469
 470                 offset = al.map->map_ip(al.map, *ip);
 471
 472                 if (!to_ip && one_map) {
 473                         struct intel_pt_cache_entry *e;
 474
 475                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 476                         if (e &&
 477                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 478                                 *insn_cnt_ptr = e->insn_cnt;
 479                                 *ip += e->byte_cnt;
 480                                 intel_pt_insn->op = e->op;
 481                                 intel_pt_insn->branch = e->branch;
 482                                 intel_pt_insn->length = e->length;
 483                                 intel_pt_insn->rel = e->rel;
 484                                 memcpy(intel_pt_insn->buf, e->insn,
 485                                        INTEL_PT_INSN_BUF_SZ);
 486                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 487                                 return 0;
 488                         }
 489                 }
 490
 491                 start_offset = offset;
 492                 start_ip = *ip;
 493
 494                 /* Load maps to ensure dso->is_64_bit has been updated */
 495                 map__load(al.map);
 496
 497                 x86_64 = al.map->dso->is_64_bit;
 498
 499                 while (1) {
 500                         len = dso__data_read_offset(al.map->dso, machine,
 501                                                     offset, buf,
 502                                                     INTEL_PT_INSN_BUF_SZ);
 503                         if (len <= 0)
 504                                 return -EINVAL;
 505
 506                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 507                                 return -EINVAL;
 508
 509                         intel_pt_log_insn(intel_pt_insn, *ip);
 510
 511                         insn_cnt += 1;
 512
 513                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 514                                 goto out;
 515
 516                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 517                                 goto out_no_cache;
 518
 519                         *ip += intel_pt_insn->length;
 520
 521                         if (to_ip && *ip == to_ip)
 522                                 goto out_no_cache;
 523
 524                         if (*ip >= al.map->end)
 525                                 break;
 526
 527                         offset += intel_pt_insn->length;
 528                 }
 529                 one_map = false;
 530         }
 531 out:
 532         *insn_cnt_ptr = insn_cnt;
 533
 534         if (!one_map)
 535                 goto out_no_cache;
 536
 537         /*
 538          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 539          * entries.
 540          */
 541         if (to_ip) {
 542                 struct intel_pt_cache_entry *e;
 543
 544                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 545                 if (e)
 546                         return 0;
 547         }
 548
 549         /* Ignore cache errors */
 550         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 551                            *ip - start_ip, intel_pt_insn);
 552
 553         return 0;
 554
 555 out_no_cache:
 556         *insn_cnt_ptr = insn_cnt;
 557         return 0;
 558 }
 559
 560 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
 561                                   uint64_t offset, const char *filename)
 562 {
 563         struct addr_filter *filt;
 564         bool have_filter   = false;
 565         bool hit_tracestop = false;
 566         bool hit_filter    = false;
 567
 568         list_for_each_entry(filt, &pt->filts.head, list) {
 569                 if (filt->start)
 570                         have_filter = true;
 571
 572                 if ((filename && !filt->filename) ||
 573                     (!filename && filt->filename) ||
 574                     (filename && strcmp(filename, filt->filename)))
 575                         continue;
 576
 577                 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
 578                         continue;
 579
 580                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
 581                              ip, offset, filename ? filename : "[kernel]",
 582                              filt->start ? "filter" : "stop",
 583                              filt->addr, filt->size);
 584
 585                 if (filt->start)
 586                         hit_filter = true;
 587                 else
 588                         hit_tracestop = true;
 589         }
 590
 591         if (!hit_tracestop && !hit_filter)
 592                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
 593                              ip, offset, filename ? filename : "[kernel]");
 594
 595         return hit_tracestop || (have_filter && !hit_filter);
 596 }
 597
 598 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
 599 {
 600         struct intel_pt_queue *ptq = data;
 601         struct thread *thread;
 602         struct addr_location al;
 603         u8 cpumode;
 604         u64 offset;
 605
 606         if (ip >= ptq->pt->kernel_start)
 607                 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
 608
 609         cpumode = PERF_RECORD_MISC_USER;
 610
 611         thread = ptq->thread;
 612         if (!thread)
 613                 return -EINVAL;
 614
 615         thread__find_addr_map(thread, cpumode, MAP__FUNCTION, ip, &al);
 616         if (!al.map || !al.map->dso)
 617                 return -EINVAL;
 618
 619         offset = al.map->map_ip(al.map, ip);
 620
 621         return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
 622                                      al.map->dso->long_name);
 623 }
 624
 625 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
 626 {
 627         return __intel_pt_pgd_ip(ip, data) > 0;
 628 }
 629
 630 static bool intel_pt_get_config(struct intel_pt *pt,
 631                                 struct perf_event_attr *attr, u64 *config)
 632 {
 633         if (attr->type == pt->pmu_type) {
 634                 if (config)
 635                         *config = attr->config;
 636                 return true;
 637         }
 638
 639         return false;
 640 }
 641
 642 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 643 {
 644         struct perf_evsel *evsel;
 645
 646         evlist__for_each_entry(pt->session->evlist, evsel) {
 647                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 648                     !evsel->attr.exclude_kernel)
 649                         return false;
 650         }
 651         return true;
 652 }
 653
 654 static bool intel_pt_return_compression(struct intel_pt *pt)
 655 {
 656         struct perf_evsel *evsel;
 657         u64 config;
 658
 659         if (!pt->noretcomp_bit)
 660                 return true;
 661
 662         evlist__for_each_entry(pt->session->evlist, evsel) {
 663                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 664                     (config & pt->noretcomp_bit))
 665                         return false;
 666         }
 667         return true;
 668 }
 669
 670 static bool intel_pt_branch_enable(struct intel_pt *pt)
 671 {
 672         struct perf_evsel *evsel;
 673         u64 config;
 674
 675         evlist__for_each_entry(pt->session->evlist, evsel) {
 676                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 677                     (config & 1) && !(config & 0x2000))
 678                         return false;
 679         }
 680         return true;
 681 }
 682
 683 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 684 {
 685         struct perf_evsel *evsel;
 686         unsigned int shift;
 687         u64 config;
 688
 689         if (!pt->mtc_freq_bits)
 690                 return 0;
 691
 692         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 693                 config >>= 1;
 694
 695         evlist__for_each_entry(pt->session->evlist, evsel) {
 696                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 697                         return (config & pt->mtc_freq_bits) >> shift;
 698         }
 699         return 0;
 700 }
 701
 702 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 703 {
 704         struct perf_evsel *evsel;
 705         bool timeless_decoding = true;
 706         u64 config;
 707
 708         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 709                 return true;
 710
 711         evlist__for_each_entry(pt->session->evlist, evsel) {
 712                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 713                         return true;
 714                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 715                         if (config & pt->tsc_bit)
 716                                 timeless_decoding = false;
 717                         else
 718                                 return true;
 719                 }
 720         }
 721         return timeless_decoding;
 722 }
 723
 724 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 725 {
 726         struct perf_evsel *evsel;
 727
 728         evlist__for_each_entry(pt->session->evlist, evsel) {
 729                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 730                     !evsel->attr.exclude_kernel)
 731                         return true;
 732         }
 733         return false;
 734 }
 735
 736 static bool intel_pt_have_tsc(struct intel_pt *pt)
 737 {
 738         struct perf_evsel *evsel;
 739         bool have_tsc = false;
 740         u64 config;
 741
 742         if (!pt->tsc_bit)
 743                 return false;
 744
 745         evlist__for_each_entry(pt->session->evlist, evsel) {
 746                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 747                         if (config & pt->tsc_bit)
 748                                 have_tsc = true;
 749                         else
 750                                 return false;
 751                 }
 752         }
 753         return have_tsc;
 754 }
 755
 756 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 757 {
 758         u64 quot, rem;
 759
 760         quot = ns / pt->tc.time_mult;
 761         rem  = ns % pt->tc.time_mult;
 762         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 763                 pt->tc.time_mult;
 764 }
 765
 766 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 767                                                    unsigned int queue_nr)
 768 {
 769         struct intel_pt_params params = { .get_trace = 0, };
 770         struct intel_pt_queue *ptq;
 771
 772         ptq = zalloc(sizeof(struct intel_pt_queue));
 773         if (!ptq)
 774                 return NULL;
 775
 776         if (pt->synth_opts.callchain) {
 777                 size_t sz = sizeof(struct ip_callchain);
 778
 779                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 780                 ptq->chain = zalloc(sz);
 781                 if (!ptq->chain)
 782                         goto out_free;
 783         }
 784
 785         if (pt->synth_opts.last_branch) {
 786                 size_t sz = sizeof(struct branch_stack);
 787
 788                 sz += pt->synth_opts.last_branch_sz *
 789                       sizeof(struct branch_entry);
 790                 ptq->last_branch = zalloc(sz);
 791                 if (!ptq->last_branch)
 792                         goto out_free;
 793                 ptq->last_branch_rb = zalloc(sz);
 794                 if (!ptq->last_branch_rb)
 795                         goto out_free;
 796         }
 797
 798         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 799         if (!ptq->event_buf)
 800                 goto out_free;
 801
 802         ptq->pt = pt;
 803         ptq->queue_nr = queue_nr;
 804         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 805         ptq->pid = -1;
 806         ptq->tid = -1;
 807         ptq->cpu = -1;
 808         ptq->next_tid = -1;
 809
 810         params.get_trace = intel_pt_get_trace;
 811         params.walk_insn = intel_pt_walk_next_insn;
 812         params.data = ptq;
 813         params.return_compression = intel_pt_return_compression(pt);
 814         params.branch_enable = intel_pt_branch_enable(pt);
 815         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 816         params.mtc_period = intel_pt_mtc_period(pt);
 817         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 818         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 819
 820         if (pt->filts.cnt > 0)
 821                 params.pgd_ip = intel_pt_pgd_ip;
 822
 823         if (pt->synth_opts.instructions) {
 824                 if (pt->synth_opts.period) {
 825                         switch (pt->synth_opts.period_type) {
 826                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 827                                 params.period_type =
 828                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 829                                 params.period = pt->synth_opts.period;
 830                                 break;
 831                         case PERF_ITRACE_PERIOD_TICKS:
 832                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 833                                 params.period = pt->synth_opts.period;
 834                                 break;
 835                         case PERF_ITRACE_PERIOD_NANOSECS:
 836                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 837                                 params.period = intel_pt_ns_to_ticks(pt,
 838                                                         pt->synth_opts.period);
 839                                 break;
 840                         default:
 841                                 break;
 842                         }
 843                 }
 844
 845                 if (!params.period) {
 846                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 847                         params.period = 1;
 848                 }
 849         }
 850
 851         ptq->decoder = intel_pt_decoder_new(&params);
 852         if (!ptq->decoder)
 853                 goto out_free;
 854
 855         return ptq;
 856
 857 out_free:
 858         zfree(&ptq->event_buf);
 859         zfree(&ptq->last_branch);
 860         zfree(&ptq->last_branch_rb);
 861         zfree(&ptq->chain);
 862         free(ptq);
 863         return NULL;
 864 }
 865
 866 static void intel_pt_free_queue(void *priv)
 867 {
 868         struct intel_pt_queue *ptq = priv;
 869
 870         if (!ptq)
 871                 return;
 872         thread__zput(ptq->thread);
 873         intel_pt_decoder_free(ptq->decoder);
 874         zfree(&ptq->event_buf);
 875         zfree(&ptq->last_branch);
 876         zfree(&ptq->last_branch_rb);
 877         zfree(&ptq->chain);
 878         free(ptq);
 879 }
 880
 881 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 882                                      struct auxtrace_queue *queue)
 883 {
 884         struct intel_pt_queue *ptq = queue->priv;
 885
 886         if (queue->tid == -1 || pt->have_sched_switch) {
 887                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 888                 thread__zput(ptq->thread);
 889         }
 890
 891         if (!ptq->thread && ptq->tid != -1)
 892                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 893
 894         if (ptq->thread) {
 895                 ptq->pid = ptq->thread->pid_;
 896                 if (queue->cpu == -1)
 897                         ptq->cpu = ptq->thread->cpu;
 898         }
 899 }
 900
 901 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 902 {
 903         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 904                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 905         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 906                 if (ptq->state->to_ip)
 907                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 908                                      PERF_IP_FLAG_ASYNC |
 909                                      PERF_IP_FLAG_INTERRUPT;
 910                 else
 911                         ptq->flags = PERF_IP_FLAG_BRANCH |
 912                                      PERF_IP_FLAG_TRACE_END;
 913                 ptq->insn_len = 0;
 914         } else {
 915                 if (ptq->state->from_ip)
 916                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 917                 else
 918                         ptq->flags = PERF_IP_FLAG_BRANCH |
 919                                      PERF_IP_FLAG_TRACE_BEGIN;
 920                 if (ptq->state->flags & INTEL_PT_IN_TX)
 921                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 922                 ptq->insn_len = ptq->state->insn_len;
 923                 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
 924         }
 925 }
 926
 927 static int intel_pt_setup_queue(struct intel_pt *pt,
 928                                 struct auxtrace_queue *queue,
 929                                 unsigned int queue_nr)
 930 {
 931         struct intel_pt_queue *ptq = queue->priv;
 932
 933         if (list_empty(&queue->head))
 934                 return 0;
 935
 936         if (!ptq) {
 937                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 938                 if (!ptq)
 939                         return -ENOMEM;
 940                 queue->priv = ptq;
 941
 942                 if (queue->cpu != -1)
 943                         ptq->cpu = queue->cpu;
 944                 ptq->tid = queue->tid;
 945
 946                 if (pt->sampling_mode) {
 947                         if (pt->timeless_decoding)
 948                                 ptq->step_through_buffers = true;
 949                         if (pt->timeless_decoding || !pt->have_sched_switch)
 950                                 ptq->use_buffer_pid_tid = true;
 951                 }
 952         }
 953
 954         if (!ptq->on_heap &&
 955             (!pt->sync_switch ||
 956              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 957                 const struct intel_pt_state *state;
 958                 int ret;
 959
 960                 if (pt->timeless_decoding)
 961                         return 0;
 962
 963                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 964                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 965                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 966                 while (1) {
 967                         state = intel_pt_decode(ptq->decoder);
 968                         if (state->err) {
 969                                 if (state->err == INTEL_PT_ERR_NODATA) {
 970                                         intel_pt_log("queue %u has no timestamp\n",
 971                                                      queue_nr);
 972                                         return 0;
 973                                 }
 974                                 continue;
 975                         }
 976                         if (state->timestamp)
 977                                 break;
 978                 }
 979
 980                 ptq->timestamp = state->timestamp;
 981                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 982                              queue_nr, ptq->timestamp);
 983                 ptq->state = state;
 984                 ptq->have_sample = true;
 985                 intel_pt_sample_flags(ptq);
 986                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 987                 if (ret)
 988                         return ret;
 989                 ptq->on_heap = true;
 990         }
 991
 992         return 0;
 993 }
 994
 995 static int intel_pt_setup_queues(struct intel_pt *pt)
 996 {
 997         unsigned int i;
 998         int ret;
 999
1000         for (i = 0; i < pt->queues.nr_queues; i++) {
1001                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1002                 if (ret)
1003                         return ret;
1004         }
1005         return 0;
1006 }
1007
1008 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
1009 {
1010         struct branch_stack *bs_src = ptq->last_branch_rb;
1011         struct branch_stack *bs_dst = ptq->last_branch;
1012         size_t nr = 0;
1013
1014         bs_dst->nr = bs_src->nr;
1015
1016         if (!bs_src->nr)
1017                 return;
1018
1019         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1020         memcpy(&bs_dst->entries[0],
1021                &bs_src->entries[ptq->last_branch_pos],
1022                sizeof(struct branch_entry) * nr);
1023
1024         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1025                 memcpy(&bs_dst->entries[nr],
1026                        &bs_src->entries[0],
1027                        sizeof(struct branch_entry) * ptq->last_branch_pos);
1028         }
1029 }
1030
1031 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1032 {
1033         ptq->last_branch_pos = 0;
1034         ptq->last_branch_rb->nr = 0;
1035 }
1036
1037 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1038 {
1039         const struct intel_pt_state *state = ptq->state;
1040         struct branch_stack *bs = ptq->last_branch_rb;
1041         struct branch_entry *be;
1042
1043         if (!ptq->last_branch_pos)
1044                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1045
1046         ptq->last_branch_pos -= 1;
1047
1048         be              = &bs->entries[ptq->last_branch_pos];
1049         be->from        = state->from_ip;
1050         be->to          = state->to_ip;
1051         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1052         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1053         /* No support for mispredict */
1054         be->flags.mispred = ptq->pt->mispred_all;
1055
1056         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1057                 bs->nr += 1;
1058 }
1059
1060 static inline bool intel_pt_skip_event(struct intel_pt *pt)
1061 {
1062         return pt->synth_opts.initial_skip &&
1063                pt->num_events++ < pt->synth_opts.initial_skip;
1064 }
1065
1066 static void intel_pt_prep_b_sample(struct intel_pt *pt,
1067                                    struct intel_pt_queue *ptq,
1068                                    union perf_event *event,
1069                                    struct perf_sample *sample)
1070 {
1071         event->sample.header.type = PERF_RECORD_SAMPLE;
1072         event->sample.header.misc = PERF_RECORD_MISC_USER;
1073         event->sample.header.size = sizeof(struct perf_event_header);
1074
1075         if (!pt->timeless_decoding)
1076                 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1077
1078         sample->cpumode = PERF_RECORD_MISC_USER;
1079         sample->ip = ptq->state->from_ip;
1080         sample->pid = ptq->pid;
1081         sample->tid = ptq->tid;
1082         sample->addr = ptq->state->to_ip;
1083         sample->period = 1;
1084         sample->cpu = ptq->cpu;
1085         sample->flags = ptq->flags;
1086         sample->insn_len = ptq->insn_len;
1087         memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1088 }
1089
1090 static int intel_pt_inject_event(union perf_event *event,
1091                                  struct perf_sample *sample, u64 type,
1092                                  bool swapped)
1093 {
1094         event->header.size = perf_event__sample_event_size(sample, type, 0);
1095         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
1096 }
1097
1098 static inline int intel_pt_opt_inject(struct intel_pt *pt,
1099                                       union perf_event *event,
1100                                       struct perf_sample *sample, u64 type)
1101 {
1102         if (!pt->synth_opts.inject)
1103                 return 0;
1104
1105         return intel_pt_inject_event(event, sample, type, pt->synth_needs_swap);
1106 }
1107
1108 static int intel_pt_deliver_synth_b_event(struct intel_pt *pt,
1109                                           union perf_event *event,
1110                                           struct perf_sample *sample, u64 type)
1111 {
1112         int ret;
1113
1114         ret = intel_pt_opt_inject(pt, event, sample, type);
1115         if (ret)
1116                 return ret;
1117
1118         ret = perf_session__deliver_synth_event(pt->session, event, sample);
1119         if (ret)
1120                 pr_err("Intel PT: failed to deliver event, error %d\n", ret);
1121
1122         return ret;
1123 }
1124
1125 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1126 {
1127         struct intel_pt *pt = ptq->pt;
1128         union perf_event *event = ptq->event_buf;
1129         struct perf_sample sample = { .ip = 0, };
1130         struct dummy_branch_stack {
1131                 u64                     nr;
1132                 struct branch_entry     entries;
1133         } dummy_bs;
1134
1135         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1136                 return 0;
1137
1138         if (intel_pt_skip_event(pt))
1139                 return 0;
1140
1141         intel_pt_prep_b_sample(pt, ptq, event, &sample);
1142
1143         sample.id = ptq->pt->branches_id;
1144         sample.stream_id = ptq->pt->branches_id;
1145
1146         /*
1147          * perf report cannot handle events without a branch stack when using
1148          * SORT_MODE__BRANCH so make a dummy one.
1149          */
1150         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1151                 dummy_bs = (struct dummy_branch_stack){
1152                         .nr = 1,
1153                         .entries = {
1154                                 .from = sample.ip,
1155                                 .to = sample.addr,
1156                         },
1157                 };
1158                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1159         }
1160
1161         return intel_pt_deliver_synth_b_event(pt, event, &sample,
1162                                               pt->branches_sample_type);
1163 }
1164
1165 static void intel_pt_prep_sample(struct intel_pt *pt,
1166                                  struct intel_pt_queue *ptq,
1167                                  union perf_event *event,
1168                                  struct perf_sample *sample)
1169 {
1170         intel_pt_prep_b_sample(pt, ptq, event, sample);
1171
1172         if (pt->synth_opts.callchain) {
1173                 thread_stack__sample(ptq->thread, ptq->chain,
1174                                      pt->synth_opts.callchain_sz, sample->ip);
1175                 sample->callchain = ptq->chain;
1176         }
1177
1178         if (pt->synth_opts.last_branch) {
1179                 intel_pt_copy_last_branch_rb(ptq);
1180                 sample->branch_stack = ptq->last_branch;
1181         }
1182 }
1183
1184 static inline int intel_pt_deliver_synth_event(struct intel_pt *pt,
1185                                                struct intel_pt_queue *ptq,
1186                                                union perf_event *event,
1187                                                struct perf_sample *sample,
1188                                                u64 type)
1189 {
1190         int ret;
1191
1192         ret = intel_pt_deliver_synth_b_event(pt, event, sample, type);
1193
1194         if (pt->synth_opts.last_branch)
1195                 intel_pt_reset_last_branch_rb(ptq);
1196
1197         return ret;
1198 }
1199
1200 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1201 {
1202         struct intel_pt *pt = ptq->pt;
1203         union perf_event *event = ptq->event_buf;
1204         struct perf_sample sample = { .ip = 0, };
1205
1206         if (intel_pt_skip_event(pt))
1207                 return 0;
1208
1209         intel_pt_prep_sample(pt, ptq, event, &sample);
1210
1211         sample.id = ptq->pt->instructions_id;
1212         sample.stream_id = ptq->pt->instructions_id;
1213         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1214
1215         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1216
1217         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1218                                             pt->instructions_sample_type);
1219 }
1220
1221 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1222 {
1223         struct intel_pt *pt = ptq->pt;
1224         union perf_event *event = ptq->event_buf;
1225         struct perf_sample sample = { .ip = 0, };
1226
1227         if (intel_pt_skip_event(pt))
1228                 return 0;
1229
1230         intel_pt_prep_sample(pt, ptq, event, &sample);
1231
1232         sample.id = ptq->pt->transactions_id;
1233         sample.stream_id = ptq->pt->transactions_id;
1234
1235         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1236                                             pt->transactions_sample_type);
1237 }
1238
1239 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1240                                 pid_t pid, pid_t tid, u64 ip)
1241 {
1242         union perf_event event;
1243         char msg[MAX_AUXTRACE_ERROR_MSG];
1244         int err;
1245
1246         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1247
1248         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1249                              code, cpu, pid, tid, ip, msg);
1250
1251         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1252         if (err)
1253                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1254                        err);
1255
1256         return err;
1257 }
1258
1259 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1260 {
1261         struct auxtrace_queue *queue;
1262         pid_t tid = ptq->next_tid;
1263         int err;
1264
1265         if (tid == -1)
1266                 return 0;
1267
1268         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1269
1270         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1271
1272         queue = &pt->queues.queue_array[ptq->queue_nr];
1273         intel_pt_set_pid_tid_cpu(pt, queue);
1274
1275         ptq->next_tid = -1;
1276
1277         return err;
1278 }
1279
1280 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1281 {
1282         struct intel_pt *pt = ptq->pt;
1283
1284         return ip == pt->switch_ip &&
1285                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1286                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1287                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1288 }
1289
1290 static int intel_pt_sample(struct intel_pt_queue *ptq)
1291 {
1292         const struct intel_pt_state *state = ptq->state;
1293         struct intel_pt *pt = ptq->pt;
1294         int err;
1295
1296         if (!ptq->have_sample)
1297                 return 0;
1298
1299         ptq->have_sample = false;
1300
1301         if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {
1302                 err = intel_pt_synth_instruction_sample(ptq);
1303                 if (err)
1304                         return err;
1305         }
1306
1307         if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
1308                 err = intel_pt_synth_transaction_sample(ptq);
1309                 if (err)
1310                         return err;
1311         }
1312
1313         if (!(state->type & INTEL_PT_BRANCH))
1314                 return 0;
1315
1316         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1317                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1318                                     state->to_ip, ptq->insn_len,
1319                                     state->trace_nr);
1320         else
1321                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1322
1323         if (pt->sample_branches) {
1324                 err = intel_pt_synth_branch_sample(ptq);
1325                 if (err)
1326                         return err;
1327         }
1328
1329         if (pt->synth_opts.last_branch)
1330                 intel_pt_update_last_branch_rb(ptq);
1331
1332         if (!pt->sync_switch)
1333                 return 0;
1334
1335         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1336                 switch (ptq->switch_state) {
1337                 case INTEL_PT_SS_UNKNOWN:
1338                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1339                         err = intel_pt_next_tid(pt, ptq);
1340                         if (err)
1341                                 return err;
1342                         ptq->switch_state = INTEL_PT_SS_TRACING;
1343                         break;
1344                 default:
1345                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1346                         return 1;
1347                 }
1348         } else if (!state->to_ip) {
1349                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1350         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1351                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1352         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1353                    state->to_ip == pt->ptss_ip &&
1354                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1355                 ptq->switch_state = INTEL_PT_SS_TRACING;
1356         }
1357
1358         return 0;
1359 }
1360
1361 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1362 {
1363         struct machine *machine = pt->machine;
1364         struct map *map;
1365         struct symbol *sym, *start;
1366         u64 ip, switch_ip = 0;
1367         const char *ptss;
1368
1369         if (ptss_ip)
1370                 *ptss_ip = 0;
1371
1372         map = machine__kernel_map(machine);
1373         if (!map)
1374                 return 0;
1375
1376         if (map__load(map))
1377                 return 0;
1378
1379         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1380
1381         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1382                 if (sym->binding == STB_GLOBAL &&
1383                     !strcmp(sym->name, "__switch_to")) {
1384                         ip = map->unmap_ip(map, sym->start);
1385                         if (ip >= map->start && ip < map->end) {
1386                                 switch_ip = ip;
1387                                 break;
1388                         }
1389                 }
1390         }
1391
1392         if (!switch_ip || !ptss_ip)
1393                 return 0;
1394
1395         if (pt->have_sched_switch == 1)
1396                 ptss = "perf_trace_sched_switch";
1397         else
1398                 ptss = "__perf_event_task_sched_out";
1399
1400         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1401                 if (!strcmp(sym->name, ptss)) {
1402                         ip = map->unmap_ip(map, sym->start);
1403                         if (ip >= map->start && ip < map->end) {
1404                                 *ptss_ip = ip;
1405                                 break;
1406                         }
1407                 }
1408         }
1409
1410         return switch_ip;
1411 }
1412
1413 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1414 {
1415         const struct intel_pt_state *state = ptq->state;
1416         struct intel_pt *pt = ptq->pt;
1417         int err;
1418
1419         if (!pt->kernel_start) {
1420                 pt->kernel_start = machine__kernel_start(pt->machine);
1421                 if (pt->per_cpu_mmaps &&
1422                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1423                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1424                     !pt->sampling_mode) {
1425                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1426                         if (pt->switch_ip) {
1427                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1428                                              pt->switch_ip, pt->ptss_ip);
1429                                 pt->sync_switch = true;
1430                         }
1431                 }
1432         }
1433
1434         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1435                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1436         while (1) {
1437                 err = intel_pt_sample(ptq);
1438                 if (err)
1439                         return err;
1440
1441                 state = intel_pt_decode(ptq->decoder);
1442                 if (state->err) {
1443                         if (state->err == INTEL_PT_ERR_NODATA)
1444                                 return 1;
1445                         if (pt->sync_switch &&
1446                             state->from_ip >= pt->kernel_start) {
1447                                 pt->sync_switch = false;
1448                                 intel_pt_next_tid(pt, ptq);
1449                         }
1450                         if (pt->synth_opts.errors) {
1451                                 err = intel_pt_synth_error(pt, state->err,
1452                                                            ptq->cpu, ptq->pid,
1453                                                            ptq->tid,
1454                                                            state->from_ip);
1455                                 if (err)
1456                                         return err;
1457                         }
1458                         continue;
1459                 }
1460
1461                 ptq->state = state;
1462                 ptq->have_sample = true;
1463                 intel_pt_sample_flags(ptq);
1464
1465                 /* Use estimated TSC upon return to user space */
1466                 if (pt->est_tsc &&
1467                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1468                     state->to_ip && state->to_ip < pt->kernel_start) {
1469                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1470                                      state->timestamp, state->est_timestamp);
1471                         ptq->timestamp = state->est_timestamp;
1472                 /* Use estimated TSC in unknown switch state */
1473                 } else if (pt->sync_switch &&
1474                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1475                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1476                            ptq->next_tid == -1) {
1477                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1478                                      state->timestamp, state->est_timestamp);
1479                         ptq->timestamp = state->est_timestamp;
1480                 } else if (state->timestamp > ptq->timestamp) {
1481                         ptq->timestamp = state->timestamp;
1482                 }
1483
1484                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1485                         *timestamp = ptq->timestamp;
1486                         return 0;
1487                 }
1488         }
1489         return 0;
1490 }
1491
1492 static inline int intel_pt_update_queues(struct intel_pt *pt)
1493 {
1494         if (pt->queues.new_data) {
1495                 pt->queues.new_data = false;
1496                 return intel_pt_setup_queues(pt);
1497         }
1498         return 0;
1499 }
1500
1501 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1502 {
1503         unsigned int queue_nr;
1504         u64 ts;
1505         int ret;
1506
1507         while (1) {
1508                 struct auxtrace_queue *queue;
1509                 struct intel_pt_queue *ptq;
1510
1511                 if (!pt->heap.heap_cnt)
1512                         return 0;
1513
1514                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1515                         return 0;
1516
1517                 queue_nr = pt->heap.heap_array[0].queue_nr;
1518                 queue = &pt->queues.queue_array[queue_nr];
1519                 ptq = queue->priv;
1520
1521                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1522                              queue_nr, pt->heap.heap_array[0].ordinal,
1523                              timestamp);
1524
1525                 auxtrace_heap__pop(&pt->heap);
1526
1527                 if (pt->heap.heap_cnt) {
1528                         ts = pt->heap.heap_array[0].ordinal + 1;
1529                         if (ts > timestamp)
1530                                 ts = timestamp;
1531                 } else {
1532                         ts = timestamp;
1533                 }
1534
1535                 intel_pt_set_pid_tid_cpu(pt, queue);
1536
1537                 ret = intel_pt_run_decoder(ptq, &ts);
1538
1539                 if (ret < 0) {
1540                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1541                         return ret;
1542                 }
1543
1544                 if (!ret) {
1545                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1546                         if (ret < 0)
1547                                 return ret;
1548                 } else {
1549                         ptq->on_heap = false;
1550                 }
1551         }
1552
1553         return 0;
1554 }
1555
1556 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1557                                             u64 time_)
1558 {
1559         struct auxtrace_queues *queues = &pt->queues;
1560         unsigned int i;
1561         u64 ts = 0;
1562
1563         for (i = 0; i < queues->nr_queues; i++) {
1564                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1565                 struct intel_pt_queue *ptq = queue->priv;
1566
1567                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1568                         ptq->time = time_;
1569                         intel_pt_set_pid_tid_cpu(pt, queue);
1570                         intel_pt_run_decoder(ptq, &ts);
1571                 }
1572         }
1573         return 0;
1574 }
1575
1576 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1577 {
1578         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1579                                     sample->pid, sample->tid, 0);
1580 }
1581
1582 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1583 {
1584         unsigned i, j;
1585
1586         if (cpu < 0 || !pt->queues.nr_queues)
1587                 return NULL;
1588
1589         if ((unsigned)cpu >= pt->queues.nr_queues)
1590                 i = pt->queues.nr_queues - 1;
1591         else
1592                 i = cpu;
1593
1594         if (pt->queues.queue_array[i].cpu == cpu)
1595                 return pt->queues.queue_array[i].priv;
1596
1597         for (j = 0; i > 0; j++) {
1598                 if (pt->queues.queue_array[--i].cpu == cpu)
1599                         return pt->queues.queue_array[i].priv;
1600         }
1601
1602         for (; j < pt->queues.nr_queues; j++) {
1603                 if (pt->queues.queue_array[j].cpu == cpu)
1604                         return pt->queues.queue_array[j].priv;
1605         }
1606
1607         return NULL;
1608 }
1609
1610 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1611                                 u64 timestamp)
1612 {
1613         struct intel_pt_queue *ptq;
1614         int err;
1615
1616         if (!pt->sync_switch)
1617                 return 1;
1618
1619         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1620         if (!ptq)
1621                 return 1;
1622
1623         switch (ptq->switch_state) {
1624         case INTEL_PT_SS_NOT_TRACING:
1625                 ptq->next_tid = -1;
1626                 break;
1627         case INTEL_PT_SS_UNKNOWN:
1628         case INTEL_PT_SS_TRACING:
1629                 ptq->next_tid = tid;
1630                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1631                 return 0;
1632         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1633                 if (!ptq->on_heap) {
1634                         ptq->timestamp = perf_time_to_tsc(timestamp,
1635                                                           &pt->tc);
1636                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1637                                                  ptq->timestamp);
1638                         if (err)
1639                                 return err;
1640                         ptq->on_heap = true;
1641                 }
1642                 ptq->switch_state = INTEL_PT_SS_TRACING;
1643                 break;
1644         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1645                 ptq->next_tid = tid;
1646                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1647                 break;
1648         default:
1649                 break;
1650         }
1651
1652         return 1;
1653 }
1654
1655 static int intel_pt_process_switch(struct intel_pt *pt,
1656                                    struct perf_sample *sample)
1657 {
1658         struct perf_evsel *evsel;
1659         pid_t tid;
1660         int cpu, ret;
1661
1662         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1663         if (evsel != pt->switch_evsel)
1664                 return 0;
1665
1666         tid = perf_evsel__intval(evsel, sample, "next_pid");
1667         cpu = sample->cpu;
1668
1669         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1670                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1671                      &pt->tc));
1672
1673         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1674         if (ret <= 0)
1675                 return ret;
1676
1677         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1678 }
1679
1680 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1681                                    struct perf_sample *sample)
1682 {
1683         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1684         pid_t pid, tid;
1685         int cpu, ret;
1686
1687         cpu = sample->cpu;
1688
1689         if (pt->have_sched_switch == 3) {
1690                 if (!out)
1691                         return 0;
1692                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1693                         pr_err("Expecting CPU-wide context switch event\n");
1694                         return -EINVAL;
1695                 }
1696                 pid = event->context_switch.next_prev_pid;
1697                 tid = event->context_switch.next_prev_tid;
1698         } else {
1699                 if (out)
1700                         return 0;
1701                 pid = sample->pid;
1702                 tid = sample->tid;
1703         }
1704
1705         if (tid == -1) {
1706                 pr_err("context_switch event has no tid\n");
1707                 return -EINVAL;
1708         }
1709
1710         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1711                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1712                      &pt->tc));
1713
1714         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1715         if (ret <= 0)
1716                 return ret;
1717
1718         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1719 }
1720
1721 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1722                                          union perf_event *event,
1723                                          struct perf_sample *sample)
1724 {
1725         if (!pt->per_cpu_mmaps)
1726                 return 0;
1727
1728         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1729                      sample->cpu, event->itrace_start.pid,
1730                      event->itrace_start.tid, sample->time,
1731                      perf_time_to_tsc(sample->time, &pt->tc));
1732
1733         return machine__set_current_tid(pt->machine, sample->cpu,
1734                                         event->itrace_start.pid,
1735                                         event->itrace_start.tid);
1736 }
1737
1738 static int intel_pt_process_event(struct perf_session *session,
1739                                   union perf_event *event,
1740                                   struct perf_sample *sample,
1741                                   struct perf_tool *tool)
1742 {
1743         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1744                                            auxtrace);
1745         u64 timestamp;
1746         int err = 0;
1747
1748         if (dump_trace)
1749                 return 0;
1750
1751         if (!tool->ordered_events) {
1752                 pr_err("Intel Processor Trace requires ordered events\n");
1753                 return -EINVAL;
1754         }
1755
1756         if (sample->time && sample->time != (u64)-1)
1757                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1758         else
1759                 timestamp = 0;
1760
1761         if (timestamp || pt->timeless_decoding) {
1762                 err = intel_pt_update_queues(pt);
1763                 if (err)
1764                         return err;
1765         }
1766
1767         if (pt->timeless_decoding) {
1768                 if (event->header.type == PERF_RECORD_EXIT) {
1769                         err = intel_pt_process_timeless_queues(pt,
1770                                                                event->fork.tid,
1771                                                                sample->time);
1772                 }
1773         } else if (timestamp) {
1774                 err = intel_pt_process_queues(pt, timestamp);
1775         }
1776         if (err)
1777                 return err;
1778
1779         if (event->header.type == PERF_RECORD_AUX &&
1780             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1781             pt->synth_opts.errors) {
1782                 err = intel_pt_lost(pt, sample);
1783                 if (err)
1784                         return err;
1785         }
1786
1787         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1788                 err = intel_pt_process_switch(pt, sample);
1789         else if (event->header.type == PERF_RECORD_ITRACE_START)
1790                 err = intel_pt_process_itrace_start(pt, event, sample);
1791         else if (event->header.type == PERF_RECORD_SWITCH ||
1792                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1793                 err = intel_pt_context_switch(pt, event, sample);
1794
1795         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1796                      perf_event__name(event->header.type), event->header.type,
1797                      sample->cpu, sample->time, timestamp);
1798
1799         return err;
1800 }
1801
1802 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1803 {
1804         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1805                                            auxtrace);
1806         int ret;
1807
1808         if (dump_trace)
1809                 return 0;
1810
1811         if (!tool->ordered_events)
1812                 return -EINVAL;
1813
1814         ret = intel_pt_update_queues(pt);
1815         if (ret < 0)
1816                 return ret;
1817
1818         if (pt->timeless_decoding)
1819                 return intel_pt_process_timeless_queues(pt, -1,
1820                                                         MAX_TIMESTAMP - 1);
1821
1822         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1823 }
1824
1825 static void intel_pt_free_events(struct perf_session *session)
1826 {
1827         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1828                                            auxtrace);
1829         struct auxtrace_queues *queues = &pt->queues;
1830         unsigned int i;
1831
1832         for (i = 0; i < queues->nr_queues; i++) {
1833                 intel_pt_free_queue(queues->queue_array[i].priv);
1834                 queues->queue_array[i].priv = NULL;
1835         }
1836         intel_pt_log_disable();
1837         auxtrace_queues__free(queues);
1838 }
1839
1840 static void intel_pt_free(struct perf_session *session)
1841 {
1842         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1843                                            auxtrace);
1844
1845         auxtrace_heap__free(&pt->heap);
1846         intel_pt_free_events(session);
1847         session->auxtrace = NULL;
1848         thread__put(pt->unknown_thread);
1849         addr_filters__exit(&pt->filts);
1850         zfree(&pt->filter);
1851         free(pt);
1852 }
1853
1854 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1855                                            union perf_event *event,
1856                                            struct perf_tool *tool __maybe_unused)
1857 {
1858         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1859                                            auxtrace);
1860
1861         if (pt->sampling_mode)
1862                 return 0;
1863
1864         if (!pt->data_queued) {
1865                 struct auxtrace_buffer *buffer;
1866                 off_t data_offset;
1867                 int fd = perf_data_file__fd(session->file);
1868                 int err;
1869
1870                 if (perf_data_file__is_pipe(session->file)) {
1871                         data_offset = 0;
1872                 } else {
1873                         data_offset = lseek(fd, 0, SEEK_CUR);
1874                         if (data_offset == -1)
1875                                 return -errno;
1876                 }
1877
1878                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1879                                                  data_offset, &buffer);
1880                 if (err)
1881                         return err;
1882
1883                 /* Dump here now we have copied a piped trace out of the pipe */
1884                 if (dump_trace) {
1885                         if (auxtrace_buffer__get_data(buffer, fd)) {
1886                                 intel_pt_dump_event(pt, buffer->data,
1887                                                     buffer->size);
1888                                 auxtrace_buffer__put_data(buffer);
1889                         }
1890                 }
1891         }
1892
1893         return 0;
1894 }
1895
1896 struct intel_pt_synth {
1897         struct perf_tool dummy_tool;
1898         struct perf_session *session;
1899 };
1900
1901 static int intel_pt_event_synth(struct perf_tool *tool,
1902                                 union perf_event *event,
1903                                 struct perf_sample *sample __maybe_unused,
1904                                 struct machine *machine __maybe_unused)
1905 {
1906         struct intel_pt_synth *intel_pt_synth =
1907                         container_of(tool, struct intel_pt_synth, dummy_tool);
1908
1909         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1910                                                  NULL);
1911 }
1912
1913 static int intel_pt_synth_event(struct perf_session *session, const char *name,
1914                                 struct perf_event_attr *attr, u64 id)
1915 {
1916         struct intel_pt_synth intel_pt_synth;
1917         int err;
1918
1919         pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1920                  name, id, (u64)attr->sample_type);
1921
1922         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1923         intel_pt_synth.session = session;
1924
1925         err = perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1926                                           &id, intel_pt_event_synth);
1927         if (err)
1928                 pr_err("%s: failed to synthesize '%s' event type\n",
1929                        __func__, name);
1930
1931         return err;
1932 }
1933
1934 static void intel_pt_set_event_name(struct perf_evlist *evlist, u64 id,
1935                                     const char *name)
1936 {
1937         struct perf_evsel *evsel;
1938
1939         evlist__for_each_entry(evlist, evsel) {
1940                 if (evsel->id && evsel->id[0] == id) {
1941                         if (evsel->name)
1942                                 zfree(&evsel->name);
1943                         evsel->name = strdup(name);
1944                         break;
1945                 }
1946         }
1947 }
1948
1949 static struct perf_evsel *intel_pt_evsel(struct intel_pt *pt,
1950                                          struct perf_evlist *evlist)
1951 {
1952         struct perf_evsel *evsel;
1953
1954         evlist__for_each_entry(evlist, evsel) {
1955                 if (evsel->attr.type == pt->pmu_type && evsel->ids)
1956                         return evsel;
1957         }
1958
1959         return NULL;
1960 }
1961
1962 static int intel_pt_synth_events(struct intel_pt *pt,
1963                                  struct perf_session *session)
1964 {
1965         struct perf_evlist *evlist = session->evlist;
1966         struct perf_evsel *evsel = intel_pt_evsel(pt, evlist);
1967         struct perf_event_attr attr;
1968         u64 id;
1969         int err;
1970
1971         if (!evsel) {
1972                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1973                 return 0;
1974         }
1975
1976         memset(&attr, 0, sizeof(struct perf_event_attr));
1977         attr.size = sizeof(struct perf_event_attr);
1978         attr.type = PERF_TYPE_HARDWARE;
1979         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1980         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1981                             PERF_SAMPLE_PERIOD;
1982         if (pt->timeless_decoding)
1983                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1984         else
1985                 attr.sample_type |= PERF_SAMPLE_TIME;
1986         if (!pt->per_cpu_mmaps)
1987                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1988         attr.exclude_user = evsel->attr.exclude_user;
1989         attr.exclude_kernel = evsel->attr.exclude_kernel;
1990         attr.exclude_hv = evsel->attr.exclude_hv;
1991         attr.exclude_host = evsel->attr.exclude_host;
1992         attr.exclude_guest = evsel->attr.exclude_guest;
1993         attr.sample_id_all = evsel->attr.sample_id_all;
1994         attr.read_format = evsel->attr.read_format;
1995
1996         id = evsel->id[0] + 1000000000;
1997         if (!id)
1998                 id = 1;
1999
2000         if (pt->synth_opts.instructions) {
2001                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2002                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
2003                         attr.sample_period =
2004                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
2005                 else
2006                         attr.sample_period = pt->synth_opts.period;
2007                 if (pt->synth_opts.callchain)
2008                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2009                 if (pt->synth_opts.last_branch)
2010                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2011                 err = intel_pt_synth_event(session, "instructions", &attr, id);
2012                 if (err)
2013                         return err;
2014                 pt->sample_instructions = true;
2015                 pt->instructions_sample_type = attr.sample_type;
2016                 pt->instructions_id = id;
2017                 id += 1;
2018         }
2019
2020         if (pt->synth_opts.transactions) {
2021                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2022                 attr.sample_period = 1;
2023                 if (pt->synth_opts.callchain)
2024                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2025                 if (pt->synth_opts.last_branch)
2026                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2027                 err = intel_pt_synth_event(session, "transactions", &attr, id);
2028                 if (err)
2029                         return err;
2030                 pt->sample_transactions = true;
2031                 pt->transactions_sample_type = attr.sample_type;
2032                 pt->transactions_id = id;
2033                 intel_pt_set_event_name(evlist, id, "transactions");
2034                 id += 1;
2035         }
2036
2037         if (pt->synth_opts.branches) {
2038                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2039                 attr.sample_period = 1;
2040                 attr.sample_type |= PERF_SAMPLE_ADDR;
2041                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
2042                 attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
2043                 err = intel_pt_synth_event(session, "branches", &attr, id);
2044                 if (err)
2045                         return err;
2046                 pt->sample_branches = true;
2047                 pt->branches_sample_type = attr.sample_type;
2048                 pt->branches_id = id;
2049         }
2050
2051         pt->synth_needs_swap = evsel->needs_swap;
2052
2053         return 0;
2054 }
2055
2056 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
2057 {
2058         struct perf_evsel *evsel;
2059
2060         evlist__for_each_entry_reverse(evlist, evsel) {
2061                 const char *name = perf_evsel__name(evsel);
2062
2063                 if (!strcmp(name, "sched:sched_switch"))
2064                         return evsel;
2065         }
2066
2067         return NULL;
2068 }
2069
2070 static bool intel_pt_find_switch(struct perf_evlist *evlist)
2071 {
2072         struct perf_evsel *evsel;
2073
2074         evlist__for_each_entry(evlist, evsel) {
2075                 if (evsel->attr.context_switch)
2076                         return true;
2077         }
2078
2079         return false;
2080 }
2081
2082 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2083 {
2084         struct intel_pt *pt = data;
2085
2086         if (!strcmp(var, "intel-pt.mispred-all"))
2087                 pt->mispred_all = perf_config_bool(var, value);
2088
2089         return 0;
2090 }
2091
2092 static const char * const intel_pt_info_fmts[] = {
2093         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
2094         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
2095         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
2096         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
2097         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
2098         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
2099         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
2100         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
2101         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
2102         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
2103         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
2104         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
2105         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
2106         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2107         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
2108         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
2109 };
2110
2111 static void intel_pt_print_info(u64 *arr, int start, int finish)
2112 {
2113         int i;
2114
2115         if (!dump_trace)
2116                 return;
2117
2118         for (i = start; i <= finish; i++)
2119                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2120 }
2121
2122 static void intel_pt_print_info_str(const char *name, const char *str)
2123 {
2124         if (!dump_trace)
2125                 return;
2126
2127         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
2128 }
2129
2130 static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2131 {
2132         return auxtrace_info->header.size >=
2133                 sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2134 }
2135
2136 int intel_pt_process_auxtrace_info(union perf_event *event,
2137                                    struct perf_session *session)
2138 {
2139         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2140         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2141         struct intel_pt *pt;
2142         void *info_end;
2143         u64 *info;
2144         int err;
2145
2146         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2147                                         min_sz)
2148                 return -EINVAL;
2149
2150         pt = zalloc(sizeof(struct intel_pt));
2151         if (!pt)
2152                 return -ENOMEM;
2153
2154         addr_filters__init(&pt->filts);
2155
2156         err = perf_config(intel_pt_perf_config, pt);
2157         if (err)
2158                 goto err_free;
2159
2160         err = auxtrace_queues__init(&pt->queues);
2161         if (err)
2162                 goto err_free;
2163
2164         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2165
2166         pt->session = session;
2167         pt->machine = &session->machines.host; /* No kvm support */
2168         pt->auxtrace_type = auxtrace_info->type;
2169         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2170         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2171         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2172         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2173         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2174         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2175         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2176         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2177         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2178         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2179         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2180                             INTEL_PT_PER_CPU_MMAPS);
2181
2182         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
2183                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2184                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2185                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2186                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2187                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2188                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2189                                     INTEL_PT_CYC_BIT);
2190         }
2191
2192         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2193                 pt->max_non_turbo_ratio =
2194                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2195                 intel_pt_print_info(&auxtrace_info->priv[0],
2196                                     INTEL_PT_MAX_NONTURBO_RATIO,
2197                                     INTEL_PT_MAX_NONTURBO_RATIO);
2198         }
2199
2200         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2201         info_end = (void *)info + auxtrace_info->header.size;
2202
2203         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2204                 size_t len;
2205
2206                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2207                 intel_pt_print_info(&auxtrace_info->priv[0],
2208                                     INTEL_PT_FILTER_STR_LEN,
2209                                     INTEL_PT_FILTER_STR_LEN);
2210                 if (len) {
2211                         const char *filter = (const char *)info;
2212
2213                         len = roundup(len + 1, 8);
2214                         info += len >> 3;
2215                         if ((void *)info > info_end) {
2216                                 pr_err("%s: bad filter string length\n", __func__);
2217                                 err = -EINVAL;
2218                                 goto err_free_queues;
2219                         }
2220                         pt->filter = memdup(filter, len);
2221                         if (!pt->filter) {
2222                                 err = -ENOMEM;
2223                                 goto err_free_queues;
2224                         }
2225                         if (session->header.needs_swap)
2226                                 mem_bswap_64(pt->filter, len);
2227                         if (pt->filter[len - 1]) {
2228                                 pr_err("%s: filter string not null terminated\n", __func__);
2229                                 err = -EINVAL;
2230                                 goto err_free_queues;
2231                         }
2232                         err = addr_filters__parse_bare_filter(&pt->filts,
2233                                                               filter);
2234                         if (err)
2235                                 goto err_free_queues;
2236                 }
2237                 intel_pt_print_info_str("Filter string", pt->filter);
2238         }
2239
2240         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2241         pt->have_tsc = intel_pt_have_tsc(pt);
2242         pt->sampling_mode = false;
2243         pt->est_tsc = !pt->timeless_decoding;
2244
2245         pt->unknown_thread = thread__new(999999999, 999999999);
2246         if (!pt->unknown_thread) {
2247                 err = -ENOMEM;
2248                 goto err_free_queues;
2249         }
2250
2251         /*
2252          * Since this thread will not be kept in any rbtree not in a
2253          * list, initialize its list node so that at thread__put() the
2254          * current thread lifetime assuption is kept and we don't segfault
2255          * at list_del_init().
2256          */
2257         INIT_LIST_HEAD(&pt->unknown_thread->node);
2258
2259         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2260         if (err)
2261                 goto err_delete_thread;
2262         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2263                 err = -ENOMEM;
2264                 goto err_delete_thread;
2265         }
2266
2267         pt->auxtrace.process_event = intel_pt_process_event;
2268         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2269         pt->auxtrace.flush_events = intel_pt_flush;
2270         pt->auxtrace.free_events = intel_pt_free_events;
2271         pt->auxtrace.free = intel_pt_free;
2272         session->auxtrace = &pt->auxtrace;
2273
2274         if (dump_trace)
2275                 return 0;
2276
2277         if (pt->have_sched_switch == 1) {
2278                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2279                 if (!pt->switch_evsel) {
2280                         pr_err("%s: missing sched_switch event\n", __func__);
2281                         err = -EINVAL;
2282                         goto err_delete_thread;
2283                 }
2284         } else if (pt->have_sched_switch == 2 &&
2285                    !intel_pt_find_switch(session->evlist)) {
2286                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2287                 err = -EINVAL;
2288                 goto err_delete_thread;
2289         }
2290
2291         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2292                 pt->synth_opts = *session->itrace_synth_opts;
2293         } else {
2294                 itrace_synth_opts__set_default(&pt->synth_opts);
2295                 if (use_browser != -1) {
2296                         pt->synth_opts.branches = false;
2297                         pt->synth_opts.callchain = true;
2298                 }
2299                 if (session->itrace_synth_opts)
2300                         pt->synth_opts.thread_stack =
2301                                 session->itrace_synth_opts->thread_stack;
2302         }
2303
2304         if (pt->synth_opts.log)
2305                 intel_pt_log_enable();
2306
2307         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2308         if (pt->tc.time_mult) {
2309                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2310
2311                 if (!pt->max_non_turbo_ratio)
2312                         pt->max_non_turbo_ratio =
2313                                         (tsc_freq + 50000000) / 100000000;
2314                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2315                 intel_pt_log("Maximum non-turbo ratio %u\n",
2316                              pt->max_non_turbo_ratio);
2317         }
2318
2319         if (pt->synth_opts.calls)
2320                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2321                                        PERF_IP_FLAG_TRACE_END;
2322         if (pt->synth_opts.returns)
2323                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2324                                        PERF_IP_FLAG_TRACE_BEGIN;
2325
2326         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2327                 symbol_conf.use_callchain = true;
2328                 if (callchain_register_param(&callchain_param) < 0) {
2329                         symbol_conf.use_callchain = false;
2330                         pt->synth_opts.callchain = false;
2331                 }
2332         }
2333
2334         err = intel_pt_synth_events(pt, session);
2335         if (err)
2336                 goto err_delete_thread;
2337
2338         err = auxtrace_queues__process_index(&pt->queues, session);
2339         if (err)
2340                 goto err_delete_thread;
2341
2342         if (pt->queues.populated)
2343                 pt->data_queued = true;
2344
2345         if (pt->timeless_decoding)
2346                 pr_debug2("Intel PT decoding without timestamps\n");
2347
2348         return 0;
2349
2350 err_delete_thread:
2351         thread__zput(pt->unknown_thread);
2352 err_free_queues:
2353         intel_pt_log_disable();
2354         auxtrace_queues__free(&pt->queues);
2355         session->auxtrace = NULL;
2356 err_free:
2357         addr_filters__exit(&pt->filts);
2358         zfree(&pt->filter);
2359         free(pt);
2360         return err;
2361 }