tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_pt_decoder.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2014, Intel Corporation.
   5  */
   6
   7 #ifndef _GNU_SOURCE
   8 #define _GNU_SOURCE
   9 #endif
  10 #include <stdlib.h>
  11 #include <stdbool.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <linux/compiler.h>
  17 #include <linux/string.h>
  18 #include <linux/zalloc.h>
  19
  20 #include "../auxtrace.h"
  21
  22 #include "intel-pt-insn-decoder.h"
  23 #include "intel-pt-pkt-decoder.h"
  24 #include "intel-pt-decoder.h"
  25 #include "intel-pt-log.h"
  26
  27 #define BITULL(x) (1ULL << (x))
  28
  29 /* IA32_RTIT_CTL MSR bits */
  30 #define INTEL_PT_CYC_ENABLE             BITULL(1)
  31 #define INTEL_PT_CYC_THRESHOLD          (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
  32 #define INTEL_PT_CYC_THRESHOLD_SHIFT    19
  33
  34 #define INTEL_PT_BLK_SIZE 1024
  35
  36 #define BIT63 (((uint64_t)1 << 63))
  37
  38 #define SEVEN_BYTES 0xffffffffffffffULL
  39
  40 #define NO_VMCS 0xffffffffffULL
  41
  42 #define INTEL_PT_RETURN 1
  43
  44 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  45 #define INTEL_PT_MAX_LOOPS 10000
  46
  47 struct intel_pt_blk {
  48         struct intel_pt_blk *prev;
  49         uint64_t ip[INTEL_PT_BLK_SIZE];
  50 };
  51
  52 struct intel_pt_stack {
  53         struct intel_pt_blk *blk;
  54         struct intel_pt_blk *spare;
  55         int pos;
  56 };
  57
  58 enum intel_pt_p_once {
  59         INTEL_PT_PRT_ONCE_UNK_VMCS,
  60         INTEL_PT_PRT_ONCE_ERANGE,
  61 };
  62
  63 enum intel_pt_pkt_state {
  64         INTEL_PT_STATE_NO_PSB,
  65         INTEL_PT_STATE_NO_IP,
  66         INTEL_PT_STATE_ERR_RESYNC,
  67         INTEL_PT_STATE_IN_SYNC,
  68         INTEL_PT_STATE_TNT_CONT,
  69         INTEL_PT_STATE_TNT,
  70         INTEL_PT_STATE_TIP,
  71         INTEL_PT_STATE_TIP_PGD,
  72         INTEL_PT_STATE_FUP,
  73         INTEL_PT_STATE_FUP_NO_TIP,
  74         INTEL_PT_STATE_FUP_IN_PSB,
  75         INTEL_PT_STATE_RESAMPLE,
  76         INTEL_PT_STATE_VM_TIME_CORRELATION,
  77 };
  78
  79 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  80 {
  81         switch (pkt_state) {
  82         case INTEL_PT_STATE_NO_PSB:
  83         case INTEL_PT_STATE_NO_IP:
  84         case INTEL_PT_STATE_ERR_RESYNC:
  85         case INTEL_PT_STATE_IN_SYNC:
  86         case INTEL_PT_STATE_TNT_CONT:
  87         case INTEL_PT_STATE_RESAMPLE:
  88         case INTEL_PT_STATE_VM_TIME_CORRELATION:
  89                 return true;
  90         case INTEL_PT_STATE_TNT:
  91         case INTEL_PT_STATE_TIP:
  92         case INTEL_PT_STATE_TIP_PGD:
  93         case INTEL_PT_STATE_FUP:
  94         case INTEL_PT_STATE_FUP_NO_TIP:
  95         case INTEL_PT_STATE_FUP_IN_PSB:
  96                 return false;
  97         default:
  98                 return true;
  99         };
 100 }
 101
 102 #ifdef INTEL_PT_STRICT
 103 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
 104 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
 105 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
 106 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
 107 #else
 108 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
 109 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
 110 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
 111 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
 112 #endif
 113
 114 struct intel_pt_decoder {
 115         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 116         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 117                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 118                          uint64_t max_insn_cnt, void *data);
 119         bool (*pgd_ip)(uint64_t ip, void *data);
 120         int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
 121         struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);
 122         void *data;
 123         struct intel_pt_state state;
 124         const unsigned char *buf;
 125         size_t len;
 126         bool return_compression;
 127         bool branch_enable;
 128         bool mtc_insn;
 129         bool pge;
 130         bool have_tma;
 131         bool have_cyc;
 132         bool fixup_last_mtc;
 133         bool have_last_ip;
 134         bool in_psb;
 135         bool hop;
 136         bool leap;
 137         bool vm_time_correlation;
 138         bool vm_tm_corr_dry_run;
 139         bool vm_tm_corr_reliable;
 140         bool vm_tm_corr_same_buf;
 141         bool vm_tm_corr_continuous;
 142         bool nr;
 143         bool next_nr;
 144         enum intel_pt_param_flags flags;
 145         uint64_t pos;
 146         uint64_t last_ip;
 147         uint64_t ip;
 148         uint64_t pip_payload;
 149         uint64_t timestamp;
 150         uint64_t tsc_timestamp;
 151         uint64_t ref_timestamp;
 152         uint64_t buf_timestamp;
 153         uint64_t sample_timestamp;
 154         uint64_t ret_addr;
 155         uint64_t ctc_timestamp;
 156         uint64_t ctc_delta;
 157         uint64_t cycle_cnt;
 158         uint64_t cyc_ref_timestamp;
 159         uint64_t first_timestamp;
 160         uint64_t last_reliable_timestamp;
 161         uint64_t vmcs;
 162         uint64_t print_once;
 163         uint64_t last_ctc;
 164         uint32_t last_mtc;
 165         uint32_t tsc_ctc_ratio_n;
 166         uint32_t tsc_ctc_ratio_d;
 167         uint32_t tsc_ctc_mult;
 168         uint32_t tsc_slip;
 169         uint32_t ctc_rem_mask;
 170         int mtc_shift;
 171         struct intel_pt_stack stack;
 172         enum intel_pt_pkt_state pkt_state;
 173         enum intel_pt_pkt_ctx pkt_ctx;
 174         enum intel_pt_pkt_ctx prev_pkt_ctx;
 175         enum intel_pt_blk_type blk_type;
 176         int blk_type_pos;
 177         struct intel_pt_pkt packet;
 178         struct intel_pt_pkt tnt;
 179         int pkt_step;
 180         int pkt_len;
 181         int last_packet_type;
 182         unsigned int cbr;
 183         unsigned int cbr_seen;
 184         unsigned int max_non_turbo_ratio;
 185         double max_non_turbo_ratio_fp;
 186         double cbr_cyc_to_tsc;
 187         double calc_cyc_to_tsc;
 188         bool have_calc_cyc_to_tsc;
 189         int exec_mode;
 190         unsigned int insn_bytes;
 191         uint64_t period;
 192         enum intel_pt_period_type period_type;
 193         uint64_t tot_insn_cnt;
 194         uint64_t period_insn_cnt;
 195         uint64_t period_mask;
 196         uint64_t period_ticks;
 197         uint64_t last_masked_timestamp;
 198         uint64_t tot_cyc_cnt;
 199         uint64_t sample_tot_cyc_cnt;
 200         uint64_t base_cyc_cnt;
 201         uint64_t cyc_cnt_timestamp;
 202         uint64_t ctl;
 203         uint64_t cyc_threshold;
 204         double tsc_to_cyc;
 205         bool continuous_period;
 206         bool overflow;
 207         bool set_fup_tx_flags;
 208         bool set_fup_ptw;
 209         bool set_fup_mwait;
 210         bool set_fup_pwre;
 211         bool set_fup_exstop;
 212         bool set_fup_bep;
 213         bool sample_cyc;
 214         unsigned int fup_tx_flags;
 215         unsigned int tx_flags;
 216         uint64_t fup_ptw_payload;
 217         uint64_t fup_mwait_payload;
 218         uint64_t fup_pwre_payload;
 219         uint64_t cbr_payload;
 220         uint64_t timestamp_insn_cnt;
 221         uint64_t sample_insn_cnt;
 222         uint64_t stuck_ip;
 223         int no_progress;
 224         int stuck_ip_prd;
 225         int stuck_ip_cnt;
 226         uint64_t psb_ip;
 227         const unsigned char *next_buf;
 228         size_t next_len;
 229         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 230 };
 231
 232 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 233 {
 234         int i;
 235
 236         for (i = 0; x != 1; i++)
 237                 x >>= 1;
 238
 239         return x << i;
 240 }
 241
 242 __printf(1, 2)
 243 static void p_log(const char *fmt, ...)
 244 {
 245         char buf[512];
 246         va_list args;
 247
 248         va_start(args, fmt);
 249         vsnprintf(buf, sizeof(buf), fmt, args);
 250         va_end(args);
 251
 252         fprintf(stderr, "%s\n", buf);
 253         intel_pt_log("%s\n", buf);
 254 }
 255
 256 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
 257                                 enum intel_pt_p_once id)
 258 {
 259         uint64_t bit = 1ULL << id;
 260
 261         if (decoder->print_once & bit)
 262                 return false;
 263         decoder->print_once |= bit;
 264         return true;
 265 }
 266
 267 static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
 268 {
 269         if (!(ctl & INTEL_PT_CYC_ENABLE))
 270                 return 0;
 271
 272         return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
 273 }
 274
 275 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 276 {
 277         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 278                 uint64_t period;
 279
 280                 period = intel_pt_lower_power_of_2(decoder->period);
 281                 decoder->period_mask  = ~(period - 1);
 282                 decoder->period_ticks = period;
 283         }
 284 }
 285
 286 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 287 {
 288         if (!d)
 289                 return 0;
 290         return (t / d) * n + ((t % d) * n) / d;
 291 }
 292
 293 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 294 {
 295         struct intel_pt_decoder *decoder;
 296
 297         if (!params->get_trace || !params->walk_insn)
 298                 return NULL;
 299
 300         decoder = zalloc(sizeof(struct intel_pt_decoder));
 301         if (!decoder)
 302                 return NULL;
 303
 304         decoder->get_trace          = params->get_trace;
 305         decoder->walk_insn          = params->walk_insn;
 306         decoder->pgd_ip             = params->pgd_ip;
 307         decoder->lookahead          = params->lookahead;
 308         decoder->findnew_vmcs_info  = params->findnew_vmcs_info;
 309         decoder->data               = params->data;
 310         decoder->return_compression = params->return_compression;
 311         decoder->branch_enable      = params->branch_enable;
 312         decoder->hop                = params->quick >= 1;
 313         decoder->leap               = params->quick >= 2;
 314         decoder->vm_time_correlation = params->vm_time_correlation;
 315         decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
 316         decoder->first_timestamp    = params->first_timestamp;
 317         decoder->last_reliable_timestamp = params->first_timestamp;
 318
 319         decoder->flags              = params->flags;
 320
 321         decoder->ctl                = params->ctl;
 322         decoder->period             = params->period;
 323         decoder->period_type        = params->period_type;
 324
 325         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 326         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 327
 328         decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
 329
 330         intel_pt_setup_period(decoder);
 331
 332         decoder->mtc_shift = params->mtc_period;
 333         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 334
 335         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 336         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 337
 338         if (!decoder->tsc_ctc_ratio_n)
 339                 decoder->tsc_ctc_ratio_d = 0;
 340
 341         if (decoder->tsc_ctc_ratio_d) {
 342                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 343                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 344                                                 decoder->tsc_ctc_ratio_d;
 345         }
 346
 347         /*
 348          * A TSC packet can slip past MTC packets so that the timestamp appears
 349          * to go backwards. One estimate is that can be up to about 40 CPU
 350          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 351          * slippage an order of magnitude more to be on the safe side.
 352          */
 353         decoder->tsc_slip = 0x10000;
 354
 355         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 356         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 357         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 358         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 359         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 360
 361         if (decoder->hop)
 362                 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
 363
 364         return decoder;
 365 }
 366
 367 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
 368                                   uint64_t first_timestamp)
 369 {
 370         decoder->first_timestamp = first_timestamp;
 371 }
 372
 373 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 374 {
 375         struct intel_pt_blk *blk = stack->blk;
 376
 377         stack->blk = blk->prev;
 378         if (!stack->spare)
 379                 stack->spare = blk;
 380         else
 381                 free(blk);
 382 }
 383
 384 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 385 {
 386         if (!stack->pos) {
 387                 if (!stack->blk)
 388                         return 0;
 389                 intel_pt_pop_blk(stack);
 390                 if (!stack->blk)
 391                         return 0;
 392                 stack->pos = INTEL_PT_BLK_SIZE;
 393         }
 394         return stack->blk->ip[--stack->pos];
 395 }
 396
 397 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 398 {
 399         struct intel_pt_blk *blk;
 400
 401         if (stack->spare) {
 402                 blk = stack->spare;
 403                 stack->spare = NULL;
 404         } else {
 405                 blk = malloc(sizeof(struct intel_pt_blk));
 406                 if (!blk)
 407                         return -ENOMEM;
 408         }
 409
 410         blk->prev = stack->blk;
 411         stack->blk = blk;
 412         stack->pos = 0;
 413         return 0;
 414 }
 415
 416 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 417 {
 418         int err;
 419
 420         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 421                 err = intel_pt_alloc_blk(stack);
 422                 if (err)
 423                         return err;
 424         }
 425
 426         stack->blk->ip[stack->pos++] = ip;
 427         return 0;
 428 }
 429
 430 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 431 {
 432         while (stack->blk)
 433                 intel_pt_pop_blk(stack);
 434         stack->pos = 0;
 435 }
 436
 437 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 438 {
 439         intel_pt_clear_stack(stack);
 440         zfree(&stack->blk);
 441         zfree(&stack->spare);
 442 }
 443
 444 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 445 {
 446         intel_pt_free_stack(&decoder->stack);
 447         free(decoder);
 448 }
 449
 450 static int intel_pt_ext_err(int code)
 451 {
 452         switch (code) {
 453         case -ENOMEM:
 454                 return INTEL_PT_ERR_NOMEM;
 455         case -ENOSYS:
 456                 return INTEL_PT_ERR_INTERN;
 457         case -EBADMSG:
 458                 return INTEL_PT_ERR_BADPKT;
 459         case -ENODATA:
 460                 return INTEL_PT_ERR_NODATA;
 461         case -EILSEQ:
 462                 return INTEL_PT_ERR_NOINSN;
 463         case -ENOENT:
 464                 return INTEL_PT_ERR_MISMAT;
 465         case -EOVERFLOW:
 466                 return INTEL_PT_ERR_OVR;
 467         case -ENOSPC:
 468                 return INTEL_PT_ERR_LOST;
 469         case -ELOOP:
 470                 return INTEL_PT_ERR_NELOOP;
 471         default:
 472                 return INTEL_PT_ERR_UNK;
 473         }
 474 }
 475
 476 static const char *intel_pt_err_msgs[] = {
 477         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 478         [INTEL_PT_ERR_INTERN] = "Internal error",
 479         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 480         [INTEL_PT_ERR_NODATA] = "No more data",
 481         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 482         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 483         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 484         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 485         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 486         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 487 };
 488
 489 int intel_pt__strerror(int code, char *buf, size_t buflen)
 490 {
 491         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 492                 code = INTEL_PT_ERR_UNK;
 493         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 494         return 0;
 495 }
 496
 497 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 498                                  uint64_t last_ip)
 499 {
 500         uint64_t ip;
 501
 502         switch (packet->count) {
 503         case 1:
 504                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 505                      packet->payload;
 506                 break;
 507         case 2:
 508                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 509                      packet->payload;
 510                 break;
 511         case 3:
 512                 ip = packet->payload;
 513                 /* Sign-extend 6-byte ip */
 514                 if (ip & (uint64_t)0x800000000000ULL)
 515                         ip |= (uint64_t)0xffff000000000000ULL;
 516                 break;
 517         case 4:
 518                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 519                      packet->payload;
 520                 break;
 521         case 6:
 522                 ip = packet->payload;
 523                 break;
 524         default:
 525                 return 0;
 526         }
 527
 528         return ip;
 529 }
 530
 531 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 532 {
 533         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 534         decoder->have_last_ip = true;
 535 }
 536
 537 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 538 {
 539         intel_pt_set_last_ip(decoder);
 540         decoder->ip = decoder->last_ip;
 541 }
 542
 543 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 544 {
 545         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 546                             decoder->buf);
 547 }
 548
 549 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 550 {
 551         intel_pt_log("ERROR: Internal error\n");
 552         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 553         return -ENOSYS;
 554 }
 555
 556 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 557 {
 558         decoder->tx_flags = 0;
 559 }
 560
 561 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 562 {
 563         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 564 }
 565
 566 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
 567 {
 568         decoder->pip_payload = decoder->packet.payload;
 569 }
 570
 571 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
 572 {
 573         decoder->next_nr = decoder->pip_payload & 1;
 574 }
 575
 576 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
 577 {
 578         decoder->nr = decoder->pip_payload & 1;
 579         decoder->next_nr = decoder->nr;
 580 }
 581
 582 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
 583 {
 584         intel_pt_update_pip(decoder);
 585         intel_pt_set_nr(decoder);
 586 }
 587
 588 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 589 {
 590         intel_pt_clear_tx_flags(decoder);
 591         decoder->have_tma = false;
 592         decoder->pkt_len = 1;
 593         decoder->pkt_step = 1;
 594         intel_pt_decoder_log_packet(decoder);
 595         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 596                 intel_pt_log("ERROR: Bad packet\n");
 597                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 598         }
 599         return -EBADMSG;
 600 }
 601
 602 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
 603 {
 604         decoder->sample_timestamp = decoder->timestamp;
 605         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
 606 }
 607
 608 static void intel_pt_reposition(struct intel_pt_decoder *decoder)
 609 {
 610         decoder->ip = 0;
 611         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 612         decoder->timestamp = 0;
 613         decoder->have_tma = false;
 614 }
 615
 616 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
 617 {
 618         struct intel_pt_buffer buffer = { .buf = 0, };
 619         int ret;
 620
 621         decoder->pkt_step = 0;
 622
 623         intel_pt_log("Getting more data\n");
 624         ret = decoder->get_trace(&buffer, decoder->data);
 625         if (ret)
 626                 return ret;
 627         decoder->buf = buffer.buf;
 628         decoder->len = buffer.len;
 629         if (!decoder->len) {
 630                 intel_pt_log("No more data\n");
 631                 return -ENODATA;
 632         }
 633         decoder->buf_timestamp = buffer.ref_timestamp;
 634         if (!buffer.consecutive || reposition) {
 635                 intel_pt_reposition(decoder);
 636                 decoder->ref_timestamp = buffer.ref_timestamp;
 637                 decoder->state.trace_nr = buffer.trace_nr;
 638                 decoder->vm_tm_corr_same_buf = false;
 639                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 640                              decoder->ref_timestamp);
 641                 return -ENOLINK;
 642         }
 643
 644         return 0;
 645 }
 646
 647 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
 648                                   bool reposition)
 649 {
 650         if (!decoder->next_buf)
 651                 return intel_pt_get_data(decoder, reposition);
 652
 653         decoder->buf = decoder->next_buf;
 654         decoder->len = decoder->next_len;
 655         decoder->next_buf = 0;
 656         decoder->next_len = 0;
 657         return 0;
 658 }
 659
 660 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 661 {
 662         unsigned char *buf = decoder->temp_buf;
 663         size_t old_len, len, n;
 664         int ret;
 665
 666         old_len = decoder->len;
 667         len = decoder->len;
 668         memcpy(buf, decoder->buf, len);
 669
 670         ret = intel_pt_get_data(decoder, false);
 671         if (ret) {
 672                 decoder->pos += old_len;
 673                 return ret < 0 ? ret : -EINVAL;
 674         }
 675
 676         n = INTEL_PT_PKT_MAX_SZ - len;
 677         if (n > decoder->len)
 678                 n = decoder->len;
 679         memcpy(buf + len, decoder->buf, n);
 680         len += n;
 681
 682         decoder->prev_pkt_ctx = decoder->pkt_ctx;
 683         ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
 684         if (ret < (int)old_len) {
 685                 decoder->next_buf = decoder->buf;
 686                 decoder->next_len = decoder->len;
 687                 decoder->buf = buf;
 688                 decoder->len = old_len;
 689                 return intel_pt_bad_packet(decoder);
 690         }
 691
 692         decoder->next_buf = decoder->buf + (ret - old_len);
 693         decoder->next_len = decoder->len - (ret - old_len);
 694
 695         decoder->buf = buf;
 696         decoder->len = ret;
 697
 698         return ret;
 699 }
 700
 701 struct intel_pt_pkt_info {
 702         struct intel_pt_decoder   *decoder;
 703         struct intel_pt_pkt       packet;
 704         uint64_t                  pos;
 705         int                       pkt_len;
 706         int                       last_packet_type;
 707         void                      *data;
 708 };
 709
 710 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 711
 712 /* Lookahead packets in current buffer */
 713 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 714                                   intel_pt_pkt_cb_t cb, void *data)
 715 {
 716         struct intel_pt_pkt_info pkt_info;
 717         const unsigned char *buf = decoder->buf;
 718         enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
 719         size_t len = decoder->len;
 720         int ret;
 721
 722         pkt_info.decoder          = decoder;
 723         pkt_info.pos              = decoder->pos;
 724         pkt_info.pkt_len          = decoder->pkt_step;
 725         pkt_info.last_packet_type = decoder->last_packet_type;
 726         pkt_info.data             = data;
 727
 728         while (1) {
 729                 do {
 730                         pkt_info.pos += pkt_info.pkt_len;
 731                         buf          += pkt_info.pkt_len;
 732                         len          -= pkt_info.pkt_len;
 733
 734                         if (!len)
 735                                 return INTEL_PT_NEED_MORE_BYTES;
 736
 737                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
 738                                                   &pkt_ctx);
 739                         if (!ret)
 740                                 return INTEL_PT_NEED_MORE_BYTES;
 741                         if (ret < 0)
 742                                 return ret;
 743
 744                         pkt_info.pkt_len = ret;
 745                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 746
 747                 ret = cb(&pkt_info);
 748                 if (ret)
 749                         return 0;
 750
 751                 pkt_info.last_packet_type = pkt_info.packet.type;
 752         }
 753 }
 754
 755 struct intel_pt_calc_cyc_to_tsc_info {
 756         uint64_t        cycle_cnt;
 757         unsigned int    cbr;
 758         uint32_t        last_mtc;
 759         uint64_t        ctc_timestamp;
 760         uint64_t        ctc_delta;
 761         uint64_t        tsc_timestamp;
 762         uint64_t        timestamp;
 763         bool            have_tma;
 764         bool            fixup_last_mtc;
 765         bool            from_mtc;
 766         double          cbr_cyc_to_tsc;
 767 };
 768
 769 /*
 770  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 771  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 772  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 773  * packet by copying the missing bits from the current MTC assuming the least
 774  * difference between the two, and that the current MTC comes after last_mtc.
 775  */
 776 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 777                                     uint32_t *last_mtc)
 778 {
 779         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 780         uint32_t mask = ~(first_missing_bit - 1);
 781
 782         *last_mtc |= mtc & mask;
 783         if (*last_mtc >= mtc) {
 784                 *last_mtc -= first_missing_bit;
 785                 *last_mtc &= 0xff;
 786         }
 787 }
 788
 789 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 790 {
 791         struct intel_pt_decoder *decoder = pkt_info->decoder;
 792         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 793         uint64_t timestamp;
 794         double cyc_to_tsc;
 795         unsigned int cbr;
 796         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 797
 798         switch (pkt_info->packet.type) {
 799         case INTEL_PT_TNT:
 800         case INTEL_PT_TIP_PGE:
 801         case INTEL_PT_TIP:
 802         case INTEL_PT_FUP:
 803         case INTEL_PT_PSB:
 804         case INTEL_PT_PIP:
 805         case INTEL_PT_MODE_EXEC:
 806         case INTEL_PT_MODE_TSX:
 807         case INTEL_PT_PSBEND:
 808         case INTEL_PT_PAD:
 809         case INTEL_PT_VMCS:
 810         case INTEL_PT_MNT:
 811         case INTEL_PT_PTWRITE:
 812         case INTEL_PT_PTWRITE_IP:
 813         case INTEL_PT_BBP:
 814         case INTEL_PT_BIP:
 815         case INTEL_PT_BEP:
 816         case INTEL_PT_BEP_IP:
 817                 return 0;
 818
 819         case INTEL_PT_MTC:
 820                 if (!data->have_tma)
 821                         return 0;
 822
 823                 mtc = pkt_info->packet.payload;
 824                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 825                         data->fixup_last_mtc = false;
 826                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 827                                                 &data->last_mtc);
 828                 }
 829                 if (mtc > data->last_mtc)
 830                         mtc_delta = mtc - data->last_mtc;
 831                 else
 832                         mtc_delta = mtc + 256 - data->last_mtc;
 833                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 834                 data->last_mtc = mtc;
 835
 836                 if (decoder->tsc_ctc_mult) {
 837                         timestamp = data->ctc_timestamp +
 838                                 data->ctc_delta * decoder->tsc_ctc_mult;
 839                 } else {
 840                         timestamp = data->ctc_timestamp +
 841                                 multdiv(data->ctc_delta,
 842                                         decoder->tsc_ctc_ratio_n,
 843                                         decoder->tsc_ctc_ratio_d);
 844                 }
 845
 846                 if (timestamp < data->timestamp)
 847                         return 1;
 848
 849                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 850                         data->timestamp = timestamp;
 851                         return 0;
 852                 }
 853
 854                 break;
 855
 856         case INTEL_PT_TSC:
 857                 /*
 858                  * For now, do not support using TSC packets - refer
 859                  * intel_pt_calc_cyc_to_tsc().
 860                  */
 861                 if (data->from_mtc)
 862                         return 1;
 863                 timestamp = pkt_info->packet.payload |
 864                             (data->timestamp & (0xffULL << 56));
 865                 if (data->from_mtc && timestamp < data->timestamp &&
 866                     data->timestamp - timestamp < decoder->tsc_slip)
 867                         return 1;
 868                 if (timestamp < data->timestamp)
 869                         timestamp += (1ULL << 56);
 870                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 871                         if (data->from_mtc)
 872                                 return 1;
 873                         data->tsc_timestamp = timestamp;
 874                         data->timestamp = timestamp;
 875                         return 0;
 876                 }
 877                 break;
 878
 879         case INTEL_PT_TMA:
 880                 if (data->from_mtc)
 881                         return 1;
 882
 883                 if (!decoder->tsc_ctc_ratio_d)
 884                         return 0;
 885
 886                 ctc = pkt_info->packet.payload;
 887                 fc = pkt_info->packet.count;
 888                 ctc_rem = ctc & decoder->ctc_rem_mask;
 889
 890                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 891
 892                 data->ctc_timestamp = data->tsc_timestamp - fc;
 893                 if (decoder->tsc_ctc_mult) {
 894                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 895                 } else {
 896                         data->ctc_timestamp -=
 897                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 898                                         decoder->tsc_ctc_ratio_d);
 899                 }
 900
 901                 data->ctc_delta = 0;
 902                 data->have_tma = true;
 903                 data->fixup_last_mtc = true;
 904
 905                 return 0;
 906
 907         case INTEL_PT_CYC:
 908                 data->cycle_cnt += pkt_info->packet.payload;
 909                 return 0;
 910
 911         case INTEL_PT_CBR:
 912                 cbr = pkt_info->packet.payload;
 913                 if (data->cbr && data->cbr != cbr)
 914                         return 1;
 915                 data->cbr = cbr;
 916                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 917                 return 0;
 918
 919         case INTEL_PT_TIP_PGD:
 920         case INTEL_PT_TRACESTOP:
 921         case INTEL_PT_EXSTOP:
 922         case INTEL_PT_EXSTOP_IP:
 923         case INTEL_PT_MWAIT:
 924         case INTEL_PT_PWRE:
 925         case INTEL_PT_PWRX:
 926         case INTEL_PT_OVF:
 927         case INTEL_PT_BAD: /* Does not happen */
 928         default:
 929                 return 1;
 930         }
 931
 932         if (!data->cbr && decoder->cbr) {
 933                 data->cbr = decoder->cbr;
 934                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 935         }
 936
 937         if (!data->cycle_cnt)
 938                 return 1;
 939
 940         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 941
 942         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 943             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 944                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 945                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 946                 return 1;
 947         }
 948
 949         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 950         decoder->have_calc_cyc_to_tsc = true;
 951
 952         if (data->cbr) {
 953                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 954                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 955         } else {
 956                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 957                              cyc_to_tsc, pkt_info->pos);
 958         }
 959
 960         return 1;
 961 }
 962
 963 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 964                                      bool from_mtc)
 965 {
 966         struct intel_pt_calc_cyc_to_tsc_info data = {
 967                 .cycle_cnt      = 0,
 968                 .cbr            = 0,
 969                 .last_mtc       = decoder->last_mtc,
 970                 .ctc_timestamp  = decoder->ctc_timestamp,
 971                 .ctc_delta      = decoder->ctc_delta,
 972                 .tsc_timestamp  = decoder->tsc_timestamp,
 973                 .timestamp      = decoder->timestamp,
 974                 .have_tma       = decoder->have_tma,
 975                 .fixup_last_mtc = decoder->fixup_last_mtc,
 976                 .from_mtc       = from_mtc,
 977                 .cbr_cyc_to_tsc = 0,
 978         };
 979
 980         /*
 981          * For now, do not support using TSC packets for at least the reasons:
 982          * 1) timing might have stopped
 983          * 2) TSC packets within PSB+ can slip against CYC packets
 984          */
 985         if (!from_mtc)
 986                 return;
 987
 988         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 989 }
 990
 991 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 992 {
 993         int ret;
 994
 995         decoder->last_packet_type = decoder->packet.type;
 996
 997         do {
 998                 decoder->pos += decoder->pkt_step;
 999                 decoder->buf += decoder->pkt_step;
1000                 decoder->len -= decoder->pkt_step;
1001
1002                 if (!decoder->len) {
1003                         ret = intel_pt_get_next_data(decoder, false);
1004                         if (ret)
1005                                 return ret;
1006                 }
1007
1008                 decoder->prev_pkt_ctx = decoder->pkt_ctx;
1009                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
1010                                           &decoder->packet, &decoder->pkt_ctx);
1011                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1012                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1013                         ret = intel_pt_get_split_packet(decoder);
1014                         if (ret < 0)
1015                                 return ret;
1016                 }
1017                 if (ret <= 0)
1018                         return intel_pt_bad_packet(decoder);
1019
1020                 decoder->pkt_len = ret;
1021                 decoder->pkt_step = ret;
1022                 intel_pt_decoder_log_packet(decoder);
1023         } while (decoder->packet.type == INTEL_PT_PAD);
1024
1025         return 0;
1026 }
1027
1028 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1029 {
1030         uint64_t timestamp, masked_timestamp;
1031
1032         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1033         masked_timestamp = timestamp & decoder->period_mask;
1034         if (decoder->continuous_period) {
1035                 if (masked_timestamp > decoder->last_masked_timestamp)
1036                         return 1;
1037         } else {
1038                 timestamp += 1;
1039                 masked_timestamp = timestamp & decoder->period_mask;
1040                 if (masked_timestamp > decoder->last_masked_timestamp) {
1041                         decoder->last_masked_timestamp = masked_timestamp;
1042                         decoder->continuous_period = true;
1043                 }
1044         }
1045
1046         if (masked_timestamp < decoder->last_masked_timestamp)
1047                 return decoder->period_ticks;
1048
1049         return decoder->period_ticks - (timestamp - masked_timestamp);
1050 }
1051
1052 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1053 {
1054         switch (decoder->period_type) {
1055         case INTEL_PT_PERIOD_INSTRUCTIONS:
1056                 return decoder->period - decoder->period_insn_cnt;
1057         case INTEL_PT_PERIOD_TICKS:
1058                 return intel_pt_next_period(decoder);
1059         case INTEL_PT_PERIOD_NONE:
1060         case INTEL_PT_PERIOD_MTC:
1061         default:
1062                 return 0;
1063         }
1064 }
1065
1066 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1067 {
1068         uint64_t timestamp, masked_timestamp;
1069
1070         switch (decoder->period_type) {
1071         case INTEL_PT_PERIOD_INSTRUCTIONS:
1072                 decoder->period_insn_cnt = 0;
1073                 break;
1074         case INTEL_PT_PERIOD_TICKS:
1075                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1076                 masked_timestamp = timestamp & decoder->period_mask;
1077                 if (masked_timestamp > decoder->last_masked_timestamp)
1078                         decoder->last_masked_timestamp = masked_timestamp;
1079                 else
1080                         decoder->last_masked_timestamp += decoder->period_ticks;
1081                 break;
1082         case INTEL_PT_PERIOD_NONE:
1083         case INTEL_PT_PERIOD_MTC:
1084         default:
1085                 break;
1086         }
1087
1088         decoder->state.type |= INTEL_PT_INSTRUCTION;
1089 }
1090
1091 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1092                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1093 {
1094         uint64_t max_insn_cnt, insn_cnt = 0;
1095         int err;
1096
1097         if (!decoder->mtc_insn)
1098                 decoder->mtc_insn = true;
1099
1100         max_insn_cnt = intel_pt_next_sample(decoder);
1101
1102         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1103                                  max_insn_cnt, decoder->data);
1104
1105         decoder->tot_insn_cnt += insn_cnt;
1106         decoder->timestamp_insn_cnt += insn_cnt;
1107         decoder->sample_insn_cnt += insn_cnt;
1108         decoder->period_insn_cnt += insn_cnt;
1109
1110         if (err) {
1111                 decoder->no_progress = 0;
1112                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
1113                 intel_pt_log_at("ERROR: Failed to get instruction",
1114                                 decoder->ip);
1115                 if (err == -ENOENT)
1116                         return -ENOLINK;
1117                 return -EILSEQ;
1118         }
1119
1120         if (ip && decoder->ip == ip) {
1121                 err = -EAGAIN;
1122                 goto out;
1123         }
1124
1125         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1126                 intel_pt_sample_insn(decoder);
1127
1128         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1129                 decoder->state.type = INTEL_PT_INSTRUCTION;
1130                 decoder->state.from_ip = decoder->ip;
1131                 decoder->state.to_ip = 0;
1132                 decoder->ip += intel_pt_insn->length;
1133                 err = INTEL_PT_RETURN;
1134                 goto out;
1135         }
1136
1137         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1138                 /* Zero-length calls are excluded */
1139                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1140                     intel_pt_insn->rel) {
1141                         err = intel_pt_push(&decoder->stack, decoder->ip +
1142                                             intel_pt_insn->length);
1143                         if (err)
1144                                 goto out;
1145                 }
1146         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1147                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1148         }
1149
1150         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1151                 int cnt = decoder->no_progress++;
1152
1153                 decoder->state.from_ip = decoder->ip;
1154                 decoder->ip += intel_pt_insn->length +
1155                                 intel_pt_insn->rel;
1156                 decoder->state.to_ip = decoder->ip;
1157                 err = INTEL_PT_RETURN;
1158
1159                 /*
1160                  * Check for being stuck in a loop.  This can happen if a
1161                  * decoder error results in the decoder erroneously setting the
1162                  * ip to an address that is itself in an infinite loop that
1163                  * consumes no packets.  When that happens, there must be an
1164                  * unconditional branch.
1165                  */
1166                 if (cnt) {
1167                         if (cnt == 1) {
1168                                 decoder->stuck_ip = decoder->state.to_ip;
1169                                 decoder->stuck_ip_prd = 1;
1170                                 decoder->stuck_ip_cnt = 1;
1171                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
1172                                    decoder->state.to_ip == decoder->stuck_ip) {
1173                                 intel_pt_log_at("ERROR: Never-ending loop",
1174                                                 decoder->state.to_ip);
1175                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1176                                 err = -ELOOP;
1177                                 goto out;
1178                         } else if (!--decoder->stuck_ip_cnt) {
1179                                 decoder->stuck_ip_prd += 1;
1180                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1181                                 decoder->stuck_ip = decoder->state.to_ip;
1182                         }
1183                 }
1184                 goto out_no_progress;
1185         }
1186 out:
1187         decoder->no_progress = 0;
1188 out_no_progress:
1189         decoder->state.insn_op = intel_pt_insn->op;
1190         decoder->state.insn_len = intel_pt_insn->length;
1191         memcpy(decoder->state.insn, intel_pt_insn->buf,
1192                INTEL_PT_INSN_BUF_SZ);
1193
1194         if (decoder->tx_flags & INTEL_PT_IN_TX)
1195                 decoder->state.flags |= INTEL_PT_IN_TX;
1196
1197         return err;
1198 }
1199
1200 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1201 {
1202         bool ret = false;
1203
1204         if (decoder->set_fup_tx_flags) {
1205                 decoder->set_fup_tx_flags = false;
1206                 decoder->tx_flags = decoder->fup_tx_flags;
1207                 decoder->state.type = INTEL_PT_TRANSACTION;
1208                 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1209                         decoder->state.type |= INTEL_PT_BRANCH;
1210                 decoder->state.from_ip = decoder->ip;
1211                 decoder->state.to_ip = 0;
1212                 decoder->state.flags = decoder->fup_tx_flags;
1213                 return true;
1214         }
1215         if (decoder->set_fup_ptw) {
1216                 decoder->set_fup_ptw = false;
1217                 decoder->state.type = INTEL_PT_PTW;
1218                 decoder->state.flags |= INTEL_PT_FUP_IP;
1219                 decoder->state.from_ip = decoder->ip;
1220                 decoder->state.to_ip = 0;
1221                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1222                 return true;
1223         }
1224         if (decoder->set_fup_mwait) {
1225                 decoder->set_fup_mwait = false;
1226                 decoder->state.type = INTEL_PT_MWAIT_OP;
1227                 decoder->state.from_ip = decoder->ip;
1228                 decoder->state.to_ip = 0;
1229                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1230                 ret = true;
1231         }
1232         if (decoder->set_fup_pwre) {
1233                 decoder->set_fup_pwre = false;
1234                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1235                 decoder->state.type &= ~INTEL_PT_BRANCH;
1236                 decoder->state.from_ip = decoder->ip;
1237                 decoder->state.to_ip = 0;
1238                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1239                 ret = true;
1240         }
1241         if (decoder->set_fup_exstop) {
1242                 decoder->set_fup_exstop = false;
1243                 decoder->state.type |= INTEL_PT_EX_STOP;
1244                 decoder->state.type &= ~INTEL_PT_BRANCH;
1245                 decoder->state.flags |= INTEL_PT_FUP_IP;
1246                 decoder->state.from_ip = decoder->ip;
1247                 decoder->state.to_ip = 0;
1248                 ret = true;
1249         }
1250         if (decoder->set_fup_bep) {
1251                 decoder->set_fup_bep = false;
1252                 decoder->state.type |= INTEL_PT_BLK_ITEMS;
1253                 decoder->state.type &= ~INTEL_PT_BRANCH;
1254                 decoder->state.from_ip = decoder->ip;
1255                 decoder->state.to_ip = 0;
1256                 ret = true;
1257         }
1258         return ret;
1259 }
1260
1261 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1262                                           struct intel_pt_insn *intel_pt_insn,
1263                                           uint64_t ip, int err)
1264 {
1265         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1266                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1267                ip == decoder->ip + intel_pt_insn->length;
1268 }
1269
1270 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1271 {
1272         struct intel_pt_insn intel_pt_insn;
1273         uint64_t ip;
1274         int err;
1275
1276         ip = decoder->last_ip;
1277
1278         while (1) {
1279                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1280                 if (err == INTEL_PT_RETURN)
1281                         return 0;
1282                 if (err == -EAGAIN ||
1283                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1284                         bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1285
1286                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1287                         if (intel_pt_fup_event(decoder) && no_tip)
1288                                 return 0;
1289                         return -EAGAIN;
1290                 }
1291                 decoder->set_fup_tx_flags = false;
1292                 if (err)
1293                         return err;
1294
1295                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1296                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1297                                         decoder->ip);
1298                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1299                         return -ENOENT;
1300                 }
1301
1302                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1303                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1304                                         decoder->ip);
1305                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1306                         return -ENOENT;
1307                 }
1308
1309                 intel_pt_bug(decoder);
1310         }
1311 }
1312
1313 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1314 {
1315         struct intel_pt_insn intel_pt_insn;
1316         int err;
1317
1318         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1319         if (err == INTEL_PT_RETURN &&
1320             decoder->pgd_ip &&
1321             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1322             (decoder->state.type & INTEL_PT_BRANCH) &&
1323             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1324                 /* Unconditional branch leaving filter region */
1325                 decoder->no_progress = 0;
1326                 decoder->pge = false;
1327                 decoder->continuous_period = false;
1328                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1329                 decoder->state.type |= INTEL_PT_TRACE_END;
1330                 intel_pt_update_nr(decoder);
1331                 return 0;
1332         }
1333         if (err == INTEL_PT_RETURN)
1334                 return 0;
1335         if (err)
1336                 return err;
1337
1338         intel_pt_update_nr(decoder);
1339
1340         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1341                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1342                         decoder->pge = false;
1343                         decoder->continuous_period = false;
1344                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1345                         decoder->state.from_ip = decoder->ip;
1346                         if (decoder->packet.count == 0) {
1347                                 decoder->state.to_ip = 0;
1348                         } else {
1349                                 decoder->state.to_ip = decoder->last_ip;
1350                                 decoder->ip = decoder->last_ip;
1351                         }
1352                         decoder->state.type |= INTEL_PT_TRACE_END;
1353                 } else {
1354                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1355                         decoder->state.from_ip = decoder->ip;
1356                         if (decoder->packet.count == 0) {
1357                                 decoder->state.to_ip = 0;
1358                         } else {
1359                                 decoder->state.to_ip = decoder->last_ip;
1360                                 decoder->ip = decoder->last_ip;
1361                         }
1362                 }
1363                 return 0;
1364         }
1365
1366         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1367                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1368                                  intel_pt_insn.rel;
1369
1370                 if (decoder->pgd_ip &&
1371                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1372                     decoder->pgd_ip(to_ip, decoder->data)) {
1373                         /* Conditional branch leaving filter region */
1374                         decoder->pge = false;
1375                         decoder->continuous_period = false;
1376                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1377                         decoder->ip = to_ip;
1378                         decoder->state.from_ip = decoder->ip;
1379                         decoder->state.to_ip = to_ip;
1380                         decoder->state.type |= INTEL_PT_TRACE_END;
1381                         return 0;
1382                 }
1383                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1384                                 decoder->ip);
1385                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1386                 return -ENOENT;
1387         }
1388
1389         return intel_pt_bug(decoder);
1390 }
1391
1392 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1393 {
1394         struct intel_pt_insn intel_pt_insn;
1395         int err;
1396
1397         while (1) {
1398                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1399                 if (err == INTEL_PT_RETURN)
1400                         return 0;
1401                 if (err)
1402                         return err;
1403
1404                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1405                         if (!decoder->return_compression) {
1406                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1407                                                 decoder->ip);
1408                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1409                                 return -ENOENT;
1410                         }
1411                         if (!decoder->ret_addr) {
1412                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1413                                                 decoder->ip);
1414                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1415                                 return -ENOENT;
1416                         }
1417                         if (!(decoder->tnt.payload & BIT63)) {
1418                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1419                                                 decoder->ip);
1420                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1421                                 return -ENOENT;
1422                         }
1423                         decoder->tnt.count -= 1;
1424                         if (decoder->tnt.count)
1425                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1426                         else
1427                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1428                         decoder->tnt.payload <<= 1;
1429                         decoder->state.from_ip = decoder->ip;
1430                         decoder->ip = decoder->ret_addr;
1431                         decoder->state.to_ip = decoder->ip;
1432                         return 0;
1433                 }
1434
1435                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1436                         /* Handle deferred TIPs */
1437                         err = intel_pt_get_next_packet(decoder);
1438                         if (err)
1439                                 return err;
1440                         if (decoder->packet.type != INTEL_PT_TIP ||
1441                             decoder->packet.count == 0) {
1442                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1443                                                 decoder->ip);
1444                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1445                                 decoder->pkt_step = 0;
1446                                 return -ENOENT;
1447                         }
1448                         intel_pt_set_last_ip(decoder);
1449                         decoder->state.from_ip = decoder->ip;
1450                         decoder->state.to_ip = decoder->last_ip;
1451                         decoder->ip = decoder->last_ip;
1452                         intel_pt_update_nr(decoder);
1453                         return 0;
1454                 }
1455
1456                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1457                         decoder->tnt.count -= 1;
1458                         if (decoder->tnt.count)
1459                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1460                         else
1461                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1462                         if (decoder->tnt.payload & BIT63) {
1463                                 decoder->tnt.payload <<= 1;
1464                                 decoder->state.from_ip = decoder->ip;
1465                                 decoder->ip += intel_pt_insn.length +
1466                                                intel_pt_insn.rel;
1467                                 decoder->state.to_ip = decoder->ip;
1468                                 return 0;
1469                         }
1470                         /* Instruction sample for a non-taken branch */
1471                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1472                                 decoder->tnt.payload <<= 1;
1473                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1474                                 decoder->state.from_ip = decoder->ip;
1475                                 decoder->state.to_ip = 0;
1476                                 decoder->ip += intel_pt_insn.length;
1477                                 return 0;
1478                         }
1479                         decoder->sample_cyc = false;
1480                         decoder->ip += intel_pt_insn.length;
1481                         if (!decoder->tnt.count) {
1482                                 intel_pt_update_sample_time(decoder);
1483                                 return -EAGAIN;
1484                         }
1485                         decoder->tnt.payload <<= 1;
1486                         continue;
1487                 }
1488
1489                 return intel_pt_bug(decoder);
1490         }
1491 }
1492
1493 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1494 {
1495         unsigned int fup_tx_flags;
1496         int err;
1497
1498         fup_tx_flags = decoder->packet.payload &
1499                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1500         err = intel_pt_get_next_packet(decoder);
1501         if (err)
1502                 return err;
1503         if (decoder->packet.type == INTEL_PT_FUP) {
1504                 decoder->fup_tx_flags = fup_tx_flags;
1505                 decoder->set_fup_tx_flags = true;
1506                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1507                         *no_tip = true;
1508         } else {
1509                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1510                                 decoder->pos);
1511                 intel_pt_update_in_tx(decoder);
1512         }
1513         return 0;
1514 }
1515
1516 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1517 {
1518         timestamp |= (ref_timestamp & (0xffULL << 56));
1519
1520         if (timestamp < ref_timestamp) {
1521                 if (ref_timestamp - timestamp > (1ULL << 55))
1522                         timestamp += (1ULL << 56);
1523         } else {
1524                 if (timestamp - ref_timestamp > (1ULL << 55))
1525                         timestamp -= (1ULL << 56);
1526         }
1527
1528         return timestamp;
1529 }
1530
1531 /* For use only when decoder->vm_time_correlation is true */
1532 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1533                                    uint64_t timestamp)
1534 {
1535         uint64_t max_timestamp = decoder->buf_timestamp;
1536
1537         if (!max_timestamp) {
1538                 max_timestamp = decoder->last_reliable_timestamp +
1539                                 0x400000000ULL;
1540         }
1541         return timestamp >= decoder->last_reliable_timestamp &&
1542                timestamp < decoder->buf_timestamp;
1543 }
1544
1545 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1546 {
1547         uint64_t timestamp;
1548         bool bad = false;
1549
1550         decoder->have_tma = false;
1551
1552         if (decoder->ref_timestamp) {
1553                 timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1554                                             decoder->ref_timestamp);
1555                 decoder->tsc_timestamp = timestamp;
1556                 decoder->timestamp = timestamp;
1557                 decoder->ref_timestamp = 0;
1558                 decoder->timestamp_insn_cnt = 0;
1559         } else if (decoder->timestamp) {
1560                 timestamp = decoder->packet.payload |
1561                             (decoder->timestamp & (0xffULL << 56));
1562                 decoder->tsc_timestamp = timestamp;
1563                 if (timestamp < decoder->timestamp &&
1564                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1565                         intel_pt_log_to("Suppressing backwards timestamp",
1566                                         timestamp);
1567                         timestamp = decoder->timestamp;
1568                 }
1569                 if (timestamp < decoder->timestamp) {
1570                         if (!decoder->buf_timestamp ||
1571                             (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1572                                 intel_pt_log_to("Wraparound timestamp", timestamp);
1573                                 timestamp += (1ULL << 56);
1574                                 decoder->tsc_timestamp = timestamp;
1575                         } else {
1576                                 intel_pt_log_to("Suppressing bad timestamp", timestamp);
1577                                 timestamp = decoder->timestamp;
1578                                 bad = true;
1579                         }
1580                 }
1581                 if (decoder->vm_time_correlation &&
1582                     (bad || !intel_pt_time_in_range(decoder, timestamp)) &&
1583                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1584                         p_log("Timestamp out of range");
1585                 decoder->timestamp = timestamp;
1586                 decoder->timestamp_insn_cnt = 0;
1587         }
1588
1589         if (decoder->last_packet_type == INTEL_PT_CYC) {
1590                 decoder->cyc_ref_timestamp = decoder->timestamp;
1591                 decoder->cycle_cnt = 0;
1592                 decoder->have_calc_cyc_to_tsc = false;
1593                 intel_pt_calc_cyc_to_tsc(decoder, false);
1594         }
1595
1596         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1597 }
1598
1599 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1600 {
1601         intel_pt_log("ERROR: Buffer overflow\n");
1602         intel_pt_clear_tx_flags(decoder);
1603         intel_pt_set_nr(decoder);
1604         decoder->timestamp_insn_cnt = 0;
1605         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1606         decoder->overflow = true;
1607         return -EOVERFLOW;
1608 }
1609
1610 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1611 {
1612         if (decoder->have_cyc)
1613                 return;
1614
1615         decoder->cyc_cnt_timestamp = decoder->timestamp;
1616         decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1617 }
1618
1619 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1620 {
1621         decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1622
1623         if (decoder->pge)
1624                 intel_pt_mtc_cyc_cnt_pge(decoder);
1625 }
1626
1627 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1628 {
1629         uint64_t tot_cyc_cnt, tsc_delta;
1630
1631         if (decoder->have_cyc)
1632                 return;
1633
1634         decoder->sample_cyc = true;
1635
1636         if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1637                 return;
1638
1639         tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1640         tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1641
1642         if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1643                 decoder->tot_cyc_cnt = tot_cyc_cnt;
1644 }
1645
1646 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1647 {
1648         uint32_t ctc = decoder->packet.payload;
1649         uint32_t fc = decoder->packet.count;
1650         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1651
1652         if (!decoder->tsc_ctc_ratio_d)
1653                 return;
1654
1655         if (decoder->pge && !decoder->in_psb)
1656                 intel_pt_mtc_cyc_cnt_pge(decoder);
1657         else
1658                 intel_pt_mtc_cyc_cnt_upd(decoder);
1659
1660         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1661         decoder->last_ctc = ctc - ctc_rem;
1662         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1663         if (decoder->tsc_ctc_mult) {
1664                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1665         } else {
1666                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1667                                                   decoder->tsc_ctc_ratio_n,
1668                                                   decoder->tsc_ctc_ratio_d);
1669         }
1670         decoder->ctc_delta = 0;
1671         decoder->have_tma = true;
1672         decoder->fixup_last_mtc = true;
1673         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1674                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1675 }
1676
1677 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1678 {
1679         uint64_t timestamp;
1680         uint32_t mtc, mtc_delta;
1681
1682         if (!decoder->have_tma)
1683                 return;
1684
1685         mtc = decoder->packet.payload;
1686
1687         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1688                 decoder->fixup_last_mtc = false;
1689                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1690                                         &decoder->last_mtc);
1691         }
1692
1693         if (mtc > decoder->last_mtc)
1694                 mtc_delta = mtc - decoder->last_mtc;
1695         else
1696                 mtc_delta = mtc + 256 - decoder->last_mtc;
1697
1698         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1699
1700         if (decoder->tsc_ctc_mult) {
1701                 timestamp = decoder->ctc_timestamp +
1702                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1703         } else {
1704                 timestamp = decoder->ctc_timestamp +
1705                             multdiv(decoder->ctc_delta,
1706                                     decoder->tsc_ctc_ratio_n,
1707                                     decoder->tsc_ctc_ratio_d);
1708         }
1709
1710         if (timestamp < decoder->timestamp)
1711                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1712                              timestamp, decoder->timestamp);
1713         else
1714                 decoder->timestamp = timestamp;
1715
1716         intel_pt_mtc_cyc_cnt_upd(decoder);
1717
1718         decoder->timestamp_insn_cnt = 0;
1719         decoder->last_mtc = mtc;
1720
1721         if (decoder->last_packet_type == INTEL_PT_CYC) {
1722                 decoder->cyc_ref_timestamp = decoder->timestamp;
1723                 decoder->cycle_cnt = 0;
1724                 decoder->have_calc_cyc_to_tsc = false;
1725                 intel_pt_calc_cyc_to_tsc(decoder, true);
1726         }
1727
1728         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1729 }
1730
1731 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1732 {
1733         unsigned int cbr = decoder->packet.payload & 0xff;
1734
1735         decoder->cbr_payload = decoder->packet.payload;
1736
1737         if (decoder->cbr == cbr)
1738                 return;
1739
1740         decoder->cbr = cbr;
1741         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1742
1743         intel_pt_mtc_cyc_cnt_cbr(decoder);
1744 }
1745
1746 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1747 {
1748         uint64_t timestamp = decoder->cyc_ref_timestamp;
1749
1750         decoder->have_cyc = true;
1751
1752         decoder->cycle_cnt += decoder->packet.payload;
1753         if (decoder->pge)
1754                 decoder->tot_cyc_cnt += decoder->packet.payload;
1755         decoder->sample_cyc = true;
1756
1757         if (!decoder->cyc_ref_timestamp)
1758                 return;
1759
1760         if (decoder->have_calc_cyc_to_tsc)
1761                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1762         else if (decoder->cbr)
1763                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1764         else
1765                 return;
1766
1767         if (timestamp < decoder->timestamp)
1768                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1769                              timestamp, decoder->timestamp);
1770         else
1771                 decoder->timestamp = timestamp;
1772
1773         decoder->timestamp_insn_cnt = 0;
1774
1775         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1776 }
1777
1778 static void intel_pt_bbp(struct intel_pt_decoder *decoder)
1779 {
1780         if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
1781                 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
1782                 decoder->state.items.is_32_bit = false;
1783         }
1784         decoder->blk_type = decoder->packet.payload;
1785         decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
1786         if (decoder->blk_type == INTEL_PT_GP_REGS)
1787                 decoder->state.items.is_32_bit = decoder->packet.count;
1788         if (decoder->blk_type_pos < 0) {
1789                 intel_pt_log("WARNING: Unknown block type %u\n",
1790                              decoder->blk_type);
1791         } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
1792                 intel_pt_log("WARNING: Duplicate block type %u\n",
1793                              decoder->blk_type);
1794         }
1795 }
1796
1797 static void intel_pt_bip(struct intel_pt_decoder *decoder)
1798 {
1799         uint32_t id = decoder->packet.count;
1800         uint32_t bit = 1 << id;
1801         int pos = decoder->blk_type_pos;
1802
1803         if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
1804                 intel_pt_log("WARNING: Unknown block item %u type %d\n",
1805                              id, decoder->blk_type);
1806                 return;
1807         }
1808
1809         if (decoder->state.items.mask[pos] & bit) {
1810                 intel_pt_log("WARNING: Duplicate block item %u type %d\n",
1811                              id, decoder->blk_type);
1812         }
1813
1814         decoder->state.items.mask[pos] |= bit;
1815         decoder->state.items.val[pos][id] = decoder->packet.payload;
1816 }
1817
1818 /* Walk PSB+ packets when already in sync. */
1819 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1820 {
1821         int err;
1822
1823         decoder->in_psb = true;
1824
1825         while (1) {
1826                 err = intel_pt_get_next_packet(decoder);
1827                 if (err)
1828                         goto out;
1829
1830                 switch (decoder->packet.type) {
1831                 case INTEL_PT_PSBEND:
1832                         err = 0;
1833                         goto out;
1834
1835                 case INTEL_PT_TIP_PGD:
1836                 case INTEL_PT_TIP_PGE:
1837                 case INTEL_PT_TIP:
1838                 case INTEL_PT_TNT:
1839                 case INTEL_PT_TRACESTOP:
1840                 case INTEL_PT_BAD:
1841                 case INTEL_PT_PSB:
1842                 case INTEL_PT_PTWRITE:
1843                 case INTEL_PT_PTWRITE_IP:
1844                 case INTEL_PT_EXSTOP:
1845                 case INTEL_PT_EXSTOP_IP:
1846                 case INTEL_PT_MWAIT:
1847                 case INTEL_PT_PWRE:
1848                 case INTEL_PT_PWRX:
1849                 case INTEL_PT_BBP:
1850                 case INTEL_PT_BIP:
1851                 case INTEL_PT_BEP:
1852                 case INTEL_PT_BEP_IP:
1853                         decoder->have_tma = false;
1854                         intel_pt_log("ERROR: Unexpected packet\n");
1855                         err = -EAGAIN;
1856                         goto out;
1857
1858                 case INTEL_PT_OVF:
1859                         err = intel_pt_overflow(decoder);
1860                         goto out;
1861
1862                 case INTEL_PT_TSC:
1863                         intel_pt_calc_tsc_timestamp(decoder);
1864                         break;
1865
1866                 case INTEL_PT_TMA:
1867                         intel_pt_calc_tma(decoder);
1868                         break;
1869
1870                 case INTEL_PT_CBR:
1871                         intel_pt_calc_cbr(decoder);
1872                         break;
1873
1874                 case INTEL_PT_MODE_EXEC:
1875                         decoder->exec_mode = decoder->packet.payload;
1876                         break;
1877
1878                 case INTEL_PT_PIP:
1879                         intel_pt_set_pip(decoder);
1880                         break;
1881
1882                 case INTEL_PT_FUP:
1883                         decoder->pge = true;
1884                         if (decoder->packet.count) {
1885                                 intel_pt_set_last_ip(decoder);
1886                                 decoder->psb_ip = decoder->last_ip;
1887                         }
1888                         break;
1889
1890                 case INTEL_PT_MODE_TSX:
1891                         intel_pt_update_in_tx(decoder);
1892                         break;
1893
1894                 case INTEL_PT_MTC:
1895                         intel_pt_calc_mtc_timestamp(decoder);
1896                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1897                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1898                         break;
1899
1900                 case INTEL_PT_CYC:
1901                         intel_pt_calc_cyc_timestamp(decoder);
1902                         break;
1903
1904                 case INTEL_PT_VMCS:
1905                 case INTEL_PT_MNT:
1906                 case INTEL_PT_PAD:
1907                 default:
1908                         break;
1909                 }
1910         }
1911 out:
1912         decoder->in_psb = false;
1913
1914         return err;
1915 }
1916
1917 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1918 {
1919         int err;
1920
1921         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1922                 decoder->tx_flags = 0;
1923                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1924                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1925         } else {
1926                 decoder->state.flags |= INTEL_PT_ASYNC;
1927         }
1928
1929         while (1) {
1930                 err = intel_pt_get_next_packet(decoder);
1931                 if (err)
1932                         return err;
1933
1934                 switch (decoder->packet.type) {
1935                 case INTEL_PT_TNT:
1936                 case INTEL_PT_FUP:
1937                 case INTEL_PT_TRACESTOP:
1938                 case INTEL_PT_PSB:
1939                 case INTEL_PT_TSC:
1940                 case INTEL_PT_TMA:
1941                 case INTEL_PT_MODE_TSX:
1942                 case INTEL_PT_BAD:
1943                 case INTEL_PT_PSBEND:
1944                 case INTEL_PT_PTWRITE:
1945                 case INTEL_PT_PTWRITE_IP:
1946                 case INTEL_PT_EXSTOP:
1947                 case INTEL_PT_EXSTOP_IP:
1948                 case INTEL_PT_MWAIT:
1949                 case INTEL_PT_PWRE:
1950                 case INTEL_PT_PWRX:
1951                 case INTEL_PT_BBP:
1952                 case INTEL_PT_BIP:
1953                 case INTEL_PT_BEP:
1954                 case INTEL_PT_BEP_IP:
1955                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1956                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1957                         decoder->pkt_step = 0;
1958                         return -ENOENT;
1959
1960                 case INTEL_PT_CBR:
1961                         intel_pt_calc_cbr(decoder);
1962                         break;
1963
1964                 case INTEL_PT_OVF:
1965                         return intel_pt_overflow(decoder);
1966
1967                 case INTEL_PT_TIP_PGD:
1968                         decoder->state.from_ip = decoder->ip;
1969                         if (decoder->packet.count == 0) {
1970                                 decoder->state.to_ip = 0;
1971                         } else {
1972                                 intel_pt_set_ip(decoder);
1973                                 decoder->state.to_ip = decoder->ip;
1974                         }
1975                         decoder->pge = false;
1976                         decoder->continuous_period = false;
1977                         decoder->state.type |= INTEL_PT_TRACE_END;
1978                         intel_pt_update_nr(decoder);
1979                         return 0;
1980
1981                 case INTEL_PT_TIP_PGE:
1982                         decoder->pge = true;
1983                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1984                                      decoder->ip);
1985                         decoder->state.from_ip = 0;
1986                         if (decoder->packet.count == 0) {
1987                                 decoder->state.to_ip = 0;
1988                         } else {
1989                                 intel_pt_set_ip(decoder);
1990                                 decoder->state.to_ip = decoder->ip;
1991                         }
1992                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1993                         intel_pt_mtc_cyc_cnt_pge(decoder);
1994                         intel_pt_set_nr(decoder);
1995                         return 0;
1996
1997                 case INTEL_PT_TIP:
1998                         decoder->state.from_ip = decoder->ip;
1999                         if (decoder->packet.count == 0) {
2000                                 decoder->state.to_ip = 0;
2001                         } else {
2002                                 intel_pt_set_ip(decoder);
2003                                 decoder->state.to_ip = decoder->ip;
2004                         }
2005                         intel_pt_update_nr(decoder);
2006                         return 0;
2007
2008                 case INTEL_PT_PIP:
2009                         intel_pt_update_pip(decoder);
2010                         break;
2011
2012                 case INTEL_PT_MTC:
2013                         intel_pt_calc_mtc_timestamp(decoder);
2014                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2015                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2016                         break;
2017
2018                 case INTEL_PT_CYC:
2019                         intel_pt_calc_cyc_timestamp(decoder);
2020                         break;
2021
2022                 case INTEL_PT_MODE_EXEC:
2023                         decoder->exec_mode = decoder->packet.payload;
2024                         break;
2025
2026                 case INTEL_PT_VMCS:
2027                 case INTEL_PT_MNT:
2028                 case INTEL_PT_PAD:
2029                         break;
2030
2031                 default:
2032                         return intel_pt_bug(decoder);
2033                 }
2034         }
2035 }
2036
2037 static int intel_pt_resample(struct intel_pt_decoder *decoder)
2038 {
2039         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2040         decoder->state.type = INTEL_PT_INSTRUCTION;
2041         decoder->state.from_ip = decoder->ip;
2042         decoder->state.to_ip = 0;
2043         return 0;
2044 }
2045
2046 struct intel_pt_vm_tsc_info {
2047         struct intel_pt_pkt pip_packet;
2048         struct intel_pt_pkt vmcs_packet;
2049         struct intel_pt_pkt tma_packet;
2050         bool tsc, pip, vmcs, tma, psbend;
2051         uint64_t ctc_delta;
2052         uint64_t last_ctc;
2053         int max_lookahead;
2054 };
2055
2056 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2057 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2058 {
2059         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2060
2061         switch (pkt_info->packet.type) {
2062         case INTEL_PT_PAD:
2063         case INTEL_PT_MNT:
2064         case INTEL_PT_MODE_EXEC:
2065         case INTEL_PT_MODE_TSX:
2066         case INTEL_PT_MTC:
2067         case INTEL_PT_FUP:
2068         case INTEL_PT_CYC:
2069         case INTEL_PT_CBR:
2070                 break;
2071
2072         case INTEL_PT_TSC:
2073                 data->tsc = true;
2074                 break;
2075
2076         case INTEL_PT_TMA:
2077                 data->tma_packet = pkt_info->packet;
2078                 data->tma = true;
2079                 break;
2080
2081         case INTEL_PT_PIP:
2082                 data->pip_packet = pkt_info->packet;
2083                 data->pip = true;
2084                 break;
2085
2086         case INTEL_PT_VMCS:
2087                 data->vmcs_packet = pkt_info->packet;
2088                 data->vmcs = true;
2089                 break;
2090
2091         case INTEL_PT_PSBEND:
2092                 data->psbend = true;
2093                 return 1;
2094
2095         case INTEL_PT_TIP_PGE:
2096         case INTEL_PT_PTWRITE:
2097         case INTEL_PT_PTWRITE_IP:
2098         case INTEL_PT_EXSTOP:
2099         case INTEL_PT_EXSTOP_IP:
2100         case INTEL_PT_MWAIT:
2101         case INTEL_PT_PWRE:
2102         case INTEL_PT_PWRX:
2103         case INTEL_PT_BBP:
2104         case INTEL_PT_BIP:
2105         case INTEL_PT_BEP:
2106         case INTEL_PT_BEP_IP:
2107         case INTEL_PT_OVF:
2108         case INTEL_PT_BAD:
2109         case INTEL_PT_TNT:
2110         case INTEL_PT_TIP_PGD:
2111         case INTEL_PT_TIP:
2112         case INTEL_PT_PSB:
2113         case INTEL_PT_TRACESTOP:
2114         default:
2115                 return 1;
2116         }
2117
2118         return 0;
2119 }
2120
2121 struct intel_pt_ovf_fup_info {
2122         int max_lookahead;
2123         bool found;
2124 };
2125
2126 /* Lookahead to detect a FUP packet after OVF */
2127 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2128 {
2129         struct intel_pt_ovf_fup_info *data = pkt_info->data;
2130
2131         if (pkt_info->packet.type == INTEL_PT_CYC ||
2132             pkt_info->packet.type == INTEL_PT_MTC ||
2133             pkt_info->packet.type == INTEL_PT_TSC)
2134                 return !--(data->max_lookahead);
2135         data->found = pkt_info->packet.type == INTEL_PT_FUP;
2136         return 1;
2137 }
2138
2139 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2140 {
2141         struct intel_pt_ovf_fup_info data = {
2142                 .max_lookahead = 16,
2143                 .found = false,
2144         };
2145
2146         intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2147         return data.found;
2148 }
2149
2150 /* Lookahead and get the TMA packet after TSC */
2151 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2152 {
2153         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2154
2155         if (pkt_info->packet.type == INTEL_PT_CYC ||
2156             pkt_info->packet.type == INTEL_PT_MTC)
2157                 return !--(data->max_lookahead);
2158
2159         if (pkt_info->packet.type == INTEL_PT_TMA) {
2160                 data->tma_packet = pkt_info->packet;
2161                 data->tma = true;
2162         }
2163         return 1;
2164 }
2165
2166 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2167 {
2168         if (decoder->tsc_ctc_mult)
2169                 return ctc * decoder->tsc_ctc_mult;
2170         else
2171                 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2172 }
2173
2174 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2175                                            uint32_t ctc,
2176                                            uint32_t fc,
2177                                            uint64_t last_ctc_timestamp,
2178                                            uint64_t ctc_delta,
2179                                            uint32_t last_ctc)
2180 {
2181         /* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2182         uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2183         /*
2184          * Number of CTC ticks from there until current TMA packet. We would
2185          * expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2186          * past an MTC, so a sign-extended value is used.
2187          */
2188         uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2189         /* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2190         uint64_t new_ctc_delta = ctc_delta + delta;
2191         uint64_t expected_tsc;
2192
2193         /*
2194          * Convert CTC ticks to TSC ticks, add the starting point
2195          * (last_ctc_timestamp) and the fast counter from the TMA packet.
2196          */
2197         expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2198
2199         if (intel_pt_enable_logging) {
2200                 intel_pt_log_x64(last_mtc_ctc);
2201                 intel_pt_log_x32(last_ctc);
2202                 intel_pt_log_x64(ctc_delta);
2203                 intel_pt_log_x64(delta);
2204                 intel_pt_log_x32(ctc);
2205                 intel_pt_log_x64(new_ctc_delta);
2206                 intel_pt_log_x64(last_ctc_timestamp);
2207                 intel_pt_log_x32(fc);
2208                 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2209                 intel_pt_log_x64(expected_tsc);
2210         }
2211
2212         return expected_tsc;
2213 }
2214
2215 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2216                                       struct intel_pt_vm_tsc_info *data)
2217 {
2218         uint32_t ctc = data->tma_packet.payload;
2219         uint32_t fc = data->tma_packet.count;
2220
2221         return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2222                                           decoder->ctc_timestamp,
2223                                           data->ctc_delta, data->last_ctc);
2224 }
2225
2226 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2227                                       struct intel_pt_vmcs_info *vmcs_info)
2228 {
2229         uint64_t payload = decoder->packet.payload;
2230
2231         /* VMX adds the TSC Offset, so subtract to get host TSC */
2232         decoder->packet.payload -= vmcs_info->tsc_offset;
2233         /* TSC packet has only 7 bytes */
2234         decoder->packet.payload &= SEVEN_BYTES;
2235
2236         /*
2237          * The buffer is mmapped from the data file, so this also updates the
2238          * data file.
2239          */
2240         if (!decoder->vm_tm_corr_dry_run)
2241                 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2242
2243         intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2244                      "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2245                      payload, decoder->packet.payload, vmcs_info->vmcs,
2246                      vmcs_info->tsc_offset);
2247 }
2248
2249 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2250                                              uint64_t tsc_offset)
2251 {
2252         struct intel_pt_vmcs_info vmcs_info = {
2253                 .vmcs = NO_VMCS,
2254                 .tsc_offset = tsc_offset
2255         };
2256
2257         intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2258 }
2259
2260 static inline bool in_vm(uint64_t pip_payload)
2261 {
2262         return pip_payload & 1;
2263 }
2264
2265 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2266 {
2267         return pip_packet->payload & 1;
2268 }
2269
2270 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2271 {
2272         p_log("VMCS: %#" PRIx64 "  TSC Offset %#" PRIx64,
2273               vmcs_info->vmcs, vmcs_info->tsc_offset);
2274 }
2275
2276 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2277                                     struct intel_pt_vm_tsc_info *data)
2278 {
2279         memset(data, 0, sizeof(*data));
2280         data->ctc_delta = decoder->ctc_delta;
2281         data->last_ctc = decoder->last_ctc;
2282         intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2283         if (data->tsc && !data->psbend)
2284                 p_log("ERROR: PSB without PSBEND");
2285         decoder->in_psb = data->psbend;
2286 }
2287
2288 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2289                                           struct intel_pt_vm_tsc_info *data,
2290                                           struct intel_pt_vmcs_info *vmcs_info,
2291                                           uint64_t host_tsc)
2292 {
2293         if (!decoder->in_psb) {
2294                 /* Can't happen */
2295                 p_log("ERROR: First TSC is not in PSB+");
2296         }
2297
2298         if (data->pip) {
2299                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2300                         if (vmcs_info && vmcs_info->tsc_offset) {
2301                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2302                                 decoder->vm_tm_corr_reliable = true;
2303                         } else {
2304                                 p_log("ERROR: First TSC, unknown TSC Offset");
2305                         }
2306                 } else { /* Host */
2307                         decoder->vm_tm_corr_reliable = true;
2308                 }
2309         } else { /* Host or Guest */
2310                 decoder->vm_tm_corr_reliable = false;
2311                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2312                         /* Assume Host */
2313                 } else {
2314                         /* Assume Guest */
2315                         if (vmcs_info && vmcs_info->tsc_offset)
2316                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2317                         else
2318                                 p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2319                 }
2320         }
2321 }
2322
2323 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2324                                     struct intel_pt_vm_tsc_info *data)
2325 {
2326         struct intel_pt_vmcs_info *vmcs_info;
2327         uint64_t tsc_offset = 0;
2328         uint64_t vmcs;
2329         bool reliable = true;
2330         uint64_t expected_tsc;
2331         uint64_t host_tsc;
2332         uint64_t ref_timestamp;
2333
2334         bool assign = false;
2335         bool assign_reliable = false;
2336
2337         /* Already have 'data' for the in_psb case */
2338         if (!decoder->in_psb) {
2339                 memset(data, 0, sizeof(*data));
2340                 data->ctc_delta = decoder->ctc_delta;
2341                 data->last_ctc = decoder->last_ctc;
2342                 data->max_lookahead = 16;
2343                 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2344                 if (decoder->pge) {
2345                         data->pip = true;
2346                         data->pip_packet.payload = decoder->pip_payload;
2347                 }
2348         }
2349
2350         /* Calculations depend on having TMA packets */
2351         if (!data->tma) {
2352                 p_log("ERROR: TSC without TMA");
2353                 return;
2354         }
2355
2356         vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2357         if (vmcs == NO_VMCS)
2358                 vmcs = 0;
2359
2360         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2361
2362         ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2363         host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2364
2365         if (!decoder->ctc_timestamp) {
2366                 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2367                 return;
2368         }
2369
2370         expected_tsc = intel_pt_expected_tsc(decoder, data);
2371
2372         tsc_offset = host_tsc - expected_tsc;
2373
2374         /* Determine if TSC is from Host or Guest */
2375         if (data->pip) {
2376                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2377                         if (!vmcs_info) {
2378                                 /* PIP NR=1 without VMCS cannot happen */
2379                                 p_log("ERROR: Missing VMCS");
2380                                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2381                                 decoder->vm_tm_corr_reliable = false;
2382                                 return;
2383                         }
2384                 } else { /* Host */
2385                         decoder->last_reliable_timestamp = host_tsc;
2386                         decoder->vm_tm_corr_reliable = true;
2387                         return;
2388                 }
2389         } else { /* Host or Guest */
2390                 reliable = false; /* Host/Guest is a guess, so not reliable */
2391                 if (decoder->in_psb) {
2392                         if (!tsc_offset)
2393                                 return; /* Zero TSC Offset, assume Host */
2394                         /*
2395                          * TSC packet has only 7 bytes of TSC. We have no
2396                          * information about the Guest's 8th byte, but it
2397                          * doesn't matter because we only need 7 bytes.
2398                          * Here, since the 8th byte is unreliable and
2399                          * irrelevant, compare only 7 byes.
2400                          */
2401                         if (vmcs_info &&
2402                             (tsc_offset & SEVEN_BYTES) ==
2403                             (vmcs_info->tsc_offset & SEVEN_BYTES)) {
2404                                 /* Same TSC Offset as last VMCS, assume Guest */
2405                                 goto guest;
2406                         }
2407                 }
2408                 /*
2409                  * Check if the host_tsc is within the expected range.
2410                  * Note, we could narrow the range more by looking ahead for
2411                  * the next host TSC in the same buffer, but we don't bother to
2412                  * do that because this is probably good enough.
2413                  */
2414                 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2415                         /* Within expected range for Host TSC, assume Host */
2416                         decoder->vm_tm_corr_reliable = false;
2417                         return;
2418                 }
2419         }
2420
2421 guest: /* Assuming Guest */
2422
2423         /* Determine whether to assign TSC Offset */
2424         if (vmcs_info && vmcs_info->vmcs) {
2425                 if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2426                         assign = false;
2427                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2428                            decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2429                         /* Continuous tracing, TSC in a PSB is not a time loss */
2430                         assign = true;
2431                         assign_reliable = true;
2432                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2433                         /*
2434                          * Unlikely to be a time loss TSC in a PSB which is not
2435                          * at the start of a buffer.
2436                          */
2437                         assign = true;
2438                         assign_reliable = false;
2439                 }
2440         }
2441
2442         /* Record VMCS TSC Offset */
2443         if (assign && (vmcs_info->tsc_offset != tsc_offset ||
2444                        vmcs_info->reliable != assign_reliable)) {
2445                 bool print = vmcs_info->tsc_offset != tsc_offset;
2446
2447                 vmcs_info->tsc_offset = tsc_offset;
2448                 vmcs_info->reliable = assign_reliable;
2449                 if (print)
2450                         intel_pt_print_vmcs_info(vmcs_info);
2451         }
2452
2453         /* Determine what TSC Offset to use */
2454         if (vmcs_info && vmcs_info->tsc_offset) {
2455                 if (!vmcs_info->reliable)
2456                         reliable = false;
2457                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2458         } else {
2459                 reliable = false;
2460                 if (vmcs_info) {
2461                         if (!vmcs_info->error_printed) {
2462                                 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2463                                       vmcs_info->vmcs);
2464                                 vmcs_info->error_printed = true;
2465                         }
2466                 } else {
2467                         if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2468                                 p_log("ERROR: Unknown VMCS");
2469                 }
2470                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2471         }
2472
2473         decoder->vm_tm_corr_reliable = reliable;
2474 }
2475
2476 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2477 {
2478         uint64_t host_tsc = decoder->packet.payload;
2479         uint64_t guest_tsc = decoder->packet.payload;
2480         struct intel_pt_vmcs_info *vmcs_info;
2481         uint64_t vmcs;
2482
2483         vmcs = decoder->vmcs;
2484         if (vmcs == NO_VMCS)
2485                 vmcs = 0;
2486
2487         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2488
2489         if (decoder->pge) {
2490                 if (in_vm(decoder->pip_payload)) { /* Guest */
2491                         if (!vmcs_info) {
2492                                 /* PIP NR=1 without VMCS cannot happen */
2493                                 p_log("ERROR: Missing VMCS");
2494                         }
2495                 } else { /* Host */
2496                         return;
2497                 }
2498         } else { /* Host or Guest */
2499                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2500                         /* Within expected range for Host TSC, assume Host */
2501                         return;
2502                 }
2503         }
2504
2505         if (vmcs_info) {
2506                 /* Translate Guest TSC to Host TSC */
2507                 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2508                 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2509                 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2510                              "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2511                              guest_tsc, host_tsc, vmcs_info->vmcs,
2512                              vmcs_info->tsc_offset);
2513                 if (!intel_pt_time_in_range(decoder, host_tsc) &&
2514                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2515                         p_log("Timestamp out of range");
2516         } else {
2517                 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2518                         p_log("ERROR: Unknown VMCS");
2519                 host_tsc = decoder->timestamp;
2520         }
2521
2522         decoder->packet.payload = host_tsc;
2523
2524         if (!decoder->vm_tm_corr_dry_run)
2525                 memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2526 }
2527
2528 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2529 {
2530         struct intel_pt_vm_tsc_info data = { .psbend = false };
2531         bool pge;
2532         int err;
2533
2534         if (decoder->in_psb)
2535                 intel_pt_vm_tm_corr_psb(decoder, &data);
2536
2537         while (1) {
2538                 err = intel_pt_get_next_packet(decoder);
2539                 if (err == -ENOLINK)
2540                         continue;
2541                 if (err)
2542                         break;
2543
2544                 switch (decoder->packet.type) {
2545                 case INTEL_PT_TIP_PGD:
2546                         decoder->pge = false;
2547                         decoder->vm_tm_corr_continuous = false;
2548                         break;
2549
2550                 case INTEL_PT_TNT:
2551                 case INTEL_PT_TIP:
2552                 case INTEL_PT_TIP_PGE:
2553                         decoder->pge = true;
2554                         break;
2555
2556                 case INTEL_PT_OVF:
2557                         decoder->in_psb = false;
2558                         pge = decoder->pge;
2559                         decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2560                         if (pge != decoder->pge)
2561                                 intel_pt_log("Surprising PGE change in OVF!");
2562                         if (!decoder->pge)
2563                                 decoder->vm_tm_corr_continuous = false;
2564                         break;
2565
2566                 case INTEL_PT_FUP:
2567                         if (decoder->in_psb)
2568                                 decoder->pge = true;
2569                         break;
2570
2571                 case INTEL_PT_TRACESTOP:
2572                         decoder->pge = false;
2573                         decoder->vm_tm_corr_continuous = false;
2574                         decoder->have_tma = false;
2575                         break;
2576
2577                 case INTEL_PT_PSB:
2578                         intel_pt_vm_tm_corr_psb(decoder, &data);
2579                         break;
2580
2581                 case INTEL_PT_PIP:
2582                         decoder->pip_payload = decoder->packet.payload;
2583                         break;
2584
2585                 case INTEL_PT_MTC:
2586                         intel_pt_calc_mtc_timestamp(decoder);
2587                         break;
2588
2589                 case INTEL_PT_TSC:
2590                         intel_pt_vm_tm_corr_tsc(decoder, &data);
2591                         intel_pt_calc_tsc_timestamp(decoder);
2592                         decoder->vm_tm_corr_same_buf = true;
2593                         decoder->vm_tm_corr_continuous = decoder->pge;
2594                         break;
2595
2596                 case INTEL_PT_TMA:
2597                         intel_pt_calc_tma(decoder);
2598                         break;
2599
2600                 case INTEL_PT_CYC:
2601                         intel_pt_calc_cyc_timestamp(decoder);
2602                         break;
2603
2604                 case INTEL_PT_CBR:
2605                         intel_pt_calc_cbr(decoder);
2606                         break;
2607
2608                 case INTEL_PT_PSBEND:
2609                         decoder->in_psb = false;
2610                         data.psbend = false;
2611                         break;
2612
2613                 case INTEL_PT_VMCS:
2614                         if (decoder->packet.payload != NO_VMCS)
2615                                 decoder->vmcs = decoder->packet.payload;
2616                         break;
2617
2618                 case INTEL_PT_BBP:
2619                         decoder->blk_type = decoder->packet.payload;
2620                         break;
2621
2622                 case INTEL_PT_BIP:
2623                         if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2624                             decoder->packet.count == 2)
2625                                 intel_pt_vm_tm_corr_pebs_tsc(decoder);
2626                         break;
2627
2628                 case INTEL_PT_BEP:
2629                 case INTEL_PT_BEP_IP:
2630                         decoder->blk_type = 0;
2631                         break;
2632
2633                 case INTEL_PT_MODE_EXEC:
2634                 case INTEL_PT_MODE_TSX:
2635                 case INTEL_PT_MNT:
2636                 case INTEL_PT_PAD:
2637                 case INTEL_PT_PTWRITE_IP:
2638                 case INTEL_PT_PTWRITE:
2639                 case INTEL_PT_MWAIT:
2640                 case INTEL_PT_PWRE:
2641                 case INTEL_PT_EXSTOP_IP:
2642                 case INTEL_PT_EXSTOP:
2643                 case INTEL_PT_PWRX:
2644                 case INTEL_PT_BAD: /* Does not happen */
2645                 default:
2646                         break;
2647                 }
2648         }
2649
2650         return err;
2651 }
2652
2653 #define HOP_PROCESS     0
2654 #define HOP_IGNORE      1
2655 #define HOP_RETURN      2
2656 #define HOP_AGAIN       3
2657
2658 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2659
2660 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2661 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
2662 {
2663         /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2664         if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2665                 *err = intel_pt_scan_for_psb(decoder);
2666                 if (*err)
2667                         return HOP_RETURN;
2668         }
2669
2670         switch (decoder->packet.type) {
2671         case INTEL_PT_TNT:
2672                 return HOP_IGNORE;
2673
2674         case INTEL_PT_TIP_PGD:
2675                 if (!decoder->packet.count) {
2676                         intel_pt_set_nr(decoder);
2677                         return HOP_IGNORE;
2678                 }
2679                 intel_pt_set_ip(decoder);
2680                 decoder->state.type |= INTEL_PT_TRACE_END;
2681                 decoder->state.from_ip = 0;
2682                 decoder->state.to_ip = decoder->ip;
2683                 intel_pt_update_nr(decoder);
2684                 return HOP_RETURN;
2685
2686         case INTEL_PT_TIP:
2687                 if (!decoder->packet.count) {
2688                         intel_pt_set_nr(decoder);
2689                         return HOP_IGNORE;
2690                 }
2691                 intel_pt_set_ip(decoder);
2692                 decoder->state.type = INTEL_PT_INSTRUCTION;
2693                 decoder->state.from_ip = decoder->ip;
2694                 decoder->state.to_ip = 0;
2695                 intel_pt_update_nr(decoder);
2696                 return HOP_RETURN;
2697
2698         case INTEL_PT_FUP:
2699                 if (!decoder->packet.count)
2700                         return HOP_IGNORE;
2701                 intel_pt_set_ip(decoder);
2702                 if (intel_pt_fup_event(decoder))
2703                         return HOP_RETURN;
2704                 if (!decoder->branch_enable)
2705                         *no_tip = true;
2706                 if (*no_tip) {
2707                         decoder->state.type = INTEL_PT_INSTRUCTION;
2708                         decoder->state.from_ip = decoder->ip;
2709                         decoder->state.to_ip = 0;
2710                         return HOP_RETURN;
2711                 }
2712                 *err = intel_pt_walk_fup_tip(decoder);
2713                 if (!*err)
2714                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2715                 return HOP_RETURN;
2716
2717         case INTEL_PT_PSB:
2718                 decoder->state.psb_offset = decoder->pos;
2719                 decoder->psb_ip = 0;
2720                 decoder->last_ip = 0;
2721                 decoder->have_last_ip = true;
2722                 *err = intel_pt_walk_psbend(decoder);
2723                 if (*err == -EAGAIN)
2724                         return HOP_AGAIN;
2725                 if (*err)
2726                         return HOP_RETURN;
2727                 decoder->state.type = INTEL_PT_PSB_EVT;
2728                 if (decoder->psb_ip) {
2729                         decoder->state.type |= INTEL_PT_INSTRUCTION;
2730                         decoder->ip = decoder->psb_ip;
2731                 }
2732                 decoder->state.from_ip = decoder->psb_ip;
2733                 decoder->state.to_ip = 0;
2734                 return HOP_RETURN;
2735
2736         case INTEL_PT_BAD:
2737         case INTEL_PT_PAD:
2738         case INTEL_PT_TIP_PGE:
2739         case INTEL_PT_TSC:
2740         case INTEL_PT_TMA:
2741         case INTEL_PT_MODE_EXEC:
2742         case INTEL_PT_MODE_TSX:
2743         case INTEL_PT_MTC:
2744         case INTEL_PT_CYC:
2745         case INTEL_PT_VMCS:
2746         case INTEL_PT_PSBEND:
2747         case INTEL_PT_CBR:
2748         case INTEL_PT_TRACESTOP:
2749         case INTEL_PT_PIP:
2750         case INTEL_PT_OVF:
2751         case INTEL_PT_MNT:
2752         case INTEL_PT_PTWRITE:
2753         case INTEL_PT_PTWRITE_IP:
2754         case INTEL_PT_EXSTOP:
2755         case INTEL_PT_EXSTOP_IP:
2756         case INTEL_PT_MWAIT:
2757         case INTEL_PT_PWRE:
2758         case INTEL_PT_PWRX:
2759         case INTEL_PT_BBP:
2760         case INTEL_PT_BIP:
2761         case INTEL_PT_BEP:
2762         case INTEL_PT_BEP_IP:
2763         default:
2764                 return HOP_PROCESS;
2765         }
2766 }
2767
2768 struct intel_pt_psb_info {
2769         struct intel_pt_pkt fup_packet;
2770         bool fup;
2771         int after_psbend;
2772 };
2773
2774 /* Lookahead and get the FUP packet from PSB+ */
2775 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2776 {
2777         struct intel_pt_psb_info *data = pkt_info->data;
2778
2779         switch (pkt_info->packet.type) {
2780         case INTEL_PT_PAD:
2781         case INTEL_PT_MNT:
2782         case INTEL_PT_TSC:
2783         case INTEL_PT_TMA:
2784         case INTEL_PT_MODE_EXEC:
2785         case INTEL_PT_MODE_TSX:
2786         case INTEL_PT_MTC:
2787         case INTEL_PT_CYC:
2788         case INTEL_PT_VMCS:
2789         case INTEL_PT_CBR:
2790         case INTEL_PT_PIP:
2791                 if (data->after_psbend) {
2792                         data->after_psbend -= 1;
2793                         if (!data->after_psbend)
2794                                 return 1;
2795                 }
2796                 break;
2797
2798         case INTEL_PT_FUP:
2799                 if (data->after_psbend)
2800                         return 1;
2801                 if (data->fup || pkt_info->packet.count == 0)
2802                         return 1;
2803                 data->fup_packet = pkt_info->packet;
2804                 data->fup = true;
2805                 break;
2806
2807         case INTEL_PT_PSBEND:
2808                 if (!data->fup)
2809                         return 1;
2810                 /* Keep going to check for a TIP.PGE */
2811                 data->after_psbend = 6;
2812                 break;
2813
2814         case INTEL_PT_TIP_PGE:
2815                 /* Ignore FUP in PSB+ if followed by TIP.PGE */
2816                 if (data->after_psbend)
2817                         data->fup = false;
2818                 return 1;
2819
2820         case INTEL_PT_PTWRITE:
2821         case INTEL_PT_PTWRITE_IP:
2822         case INTEL_PT_EXSTOP:
2823         case INTEL_PT_EXSTOP_IP:
2824         case INTEL_PT_MWAIT:
2825         case INTEL_PT_PWRE:
2826         case INTEL_PT_PWRX:
2827         case INTEL_PT_BBP:
2828         case INTEL_PT_BIP:
2829         case INTEL_PT_BEP:
2830         case INTEL_PT_BEP_IP:
2831                 if (data->after_psbend) {
2832                         data->after_psbend -= 1;
2833                         if (!data->after_psbend)
2834                                 return 1;
2835                         break;
2836                 }
2837                 return 1;
2838
2839         case INTEL_PT_OVF:
2840         case INTEL_PT_BAD:
2841         case INTEL_PT_TNT:
2842         case INTEL_PT_TIP_PGD:
2843         case INTEL_PT_TIP:
2844         case INTEL_PT_PSB:
2845         case INTEL_PT_TRACESTOP:
2846         default:
2847                 return 1;
2848         }
2849
2850         return 0;
2851 }
2852
2853 static int intel_pt_psb(struct intel_pt_decoder *decoder)
2854 {
2855         int err;
2856
2857         decoder->last_ip = 0;
2858         decoder->psb_ip = 0;
2859         decoder->have_last_ip = true;
2860         intel_pt_clear_stack(&decoder->stack);
2861         err = intel_pt_walk_psbend(decoder);
2862         if (err)
2863                 return err;
2864         decoder->state.type = INTEL_PT_PSB_EVT;
2865         decoder->state.from_ip = decoder->psb_ip;
2866         decoder->state.to_ip = 0;
2867         return 0;
2868 }
2869
2870 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
2871 {
2872         int err;
2873
2874         if (decoder->ip != decoder->last_ip) {
2875                 err = intel_pt_walk_fup(decoder);
2876                 if (!err || err != -EAGAIN)
2877                         return err;
2878         }
2879
2880         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2881         err = intel_pt_psb(decoder);
2882         if (err) {
2883                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2884                 return -ENOENT;
2885         }
2886
2887         return 0;
2888 }
2889
2890 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
2891 {
2892         struct intel_pt_psb_info data = { .fup = false };
2893
2894         if (!decoder->branch_enable || !decoder->pge)
2895                 return false;
2896
2897         intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
2898         if (!data.fup)
2899                 return false;
2900
2901         decoder->packet = data.fup_packet;
2902         intel_pt_set_last_ip(decoder);
2903         decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
2904
2905         *err = intel_pt_fup_in_psb(decoder);
2906
2907         return true;
2908 }
2909
2910 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
2911 {
2912         int last_packet_type = INTEL_PT_PAD;
2913         bool no_tip = false;
2914         int err;
2915
2916         while (1) {
2917                 err = intel_pt_get_next_packet(decoder);
2918                 if (err)
2919                         return err;
2920 next:
2921                 if (decoder->cyc_threshold) {
2922                         if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
2923                                 decoder->sample_cyc = false;
2924                         last_packet_type = decoder->packet.type;
2925                 }
2926
2927                 if (decoder->hop) {
2928                         switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
2929                         case HOP_IGNORE:
2930                                 continue;
2931                         case HOP_RETURN:
2932                                 return err;
2933                         case HOP_AGAIN:
2934                                 goto next;
2935                         default:
2936                                 break;
2937                         }
2938                 }
2939
2940                 switch (decoder->packet.type) {
2941                 case INTEL_PT_TNT:
2942                         if (!decoder->packet.count)
2943                                 break;
2944                         decoder->tnt = decoder->packet;
2945                         decoder->pkt_state = INTEL_PT_STATE_TNT;
2946                         err = intel_pt_walk_tnt(decoder);
2947                         if (err == -EAGAIN)
2948                                 break;
2949                         return err;
2950
2951                 case INTEL_PT_TIP_PGD:
2952                         if (decoder->packet.count != 0)
2953                                 intel_pt_set_last_ip(decoder);
2954                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
2955                         return intel_pt_walk_tip(decoder);
2956
2957                 case INTEL_PT_TIP_PGE: {
2958                         decoder->pge = true;
2959                         intel_pt_mtc_cyc_cnt_pge(decoder);
2960                         intel_pt_set_nr(decoder);
2961                         if (decoder->packet.count == 0) {
2962                                 intel_pt_log_at("Skipping zero TIP.PGE",
2963                                                 decoder->pos);
2964                                 break;
2965                         }
2966                         intel_pt_set_ip(decoder);
2967                         decoder->state.from_ip = 0;
2968                         decoder->state.to_ip = decoder->ip;
2969                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2970                         /*
2971                          * In hop mode, resample to get the to_ip as an
2972                          * "instruction" sample.
2973                          */
2974                         if (decoder->hop)
2975                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2976                         return 0;
2977                 }
2978
2979                 case INTEL_PT_OVF:
2980                         return intel_pt_overflow(decoder);
2981
2982                 case INTEL_PT_TIP:
2983                         if (decoder->packet.count != 0)
2984                                 intel_pt_set_last_ip(decoder);
2985                         decoder->pkt_state = INTEL_PT_STATE_TIP;
2986                         return intel_pt_walk_tip(decoder);
2987
2988                 case INTEL_PT_FUP:
2989                         if (decoder->packet.count == 0) {
2990                                 intel_pt_log_at("Skipping zero FUP",
2991                                                 decoder->pos);
2992                                 no_tip = false;
2993                                 break;
2994                         }
2995                         intel_pt_set_last_ip(decoder);
2996                         if (!decoder->branch_enable) {
2997                                 decoder->ip = decoder->last_ip;
2998                                 if (intel_pt_fup_event(decoder))
2999                                         return 0;
3000                                 no_tip = false;
3001                                 break;
3002                         }
3003                         if (decoder->set_fup_mwait)
3004                                 no_tip = true;
3005                         if (no_tip)
3006                                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3007                         else
3008                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
3009                         err = intel_pt_walk_fup(decoder);
3010                         if (err != -EAGAIN)
3011                                 return err;
3012                         if (no_tip) {
3013                                 no_tip = false;
3014                                 break;
3015                         }
3016                         return intel_pt_walk_fup_tip(decoder);
3017
3018                 case INTEL_PT_TRACESTOP:
3019                         decoder->pge = false;
3020                         decoder->continuous_period = false;
3021                         intel_pt_clear_tx_flags(decoder);
3022                         decoder->have_tma = false;
3023                         break;
3024
3025                 case INTEL_PT_PSB:
3026                         decoder->state.psb_offset = decoder->pos;
3027                         decoder->psb_ip = 0;
3028                         if (intel_pt_psb_with_fup(decoder, &err))
3029                                 return err;
3030                         err = intel_pt_psb(decoder);
3031                         if (err == -EAGAIN)
3032                                 goto next;
3033                         return err;
3034
3035                 case INTEL_PT_PIP:
3036                         intel_pt_update_pip(decoder);
3037                         break;
3038
3039                 case INTEL_PT_MTC:
3040                         intel_pt_calc_mtc_timestamp(decoder);
3041                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3042                                 break;
3043                         /*
3044                          * Ensure that there has been an instruction since the
3045                          * last MTC.
3046                          */
3047                         if (!decoder->mtc_insn)
3048                                 break;
3049                         decoder->mtc_insn = false;
3050                         /* Ensure that there is a timestamp */
3051                         if (!decoder->timestamp)
3052                                 break;
3053                         decoder->state.type = INTEL_PT_INSTRUCTION;
3054                         decoder->state.from_ip = decoder->ip;
3055                         decoder->state.to_ip = 0;
3056                         decoder->mtc_insn = false;
3057                         return 0;
3058
3059                 case INTEL_PT_TSC:
3060                         intel_pt_calc_tsc_timestamp(decoder);
3061                         break;
3062
3063                 case INTEL_PT_TMA:
3064                         intel_pt_calc_tma(decoder);
3065                         break;
3066
3067                 case INTEL_PT_CYC:
3068                         intel_pt_calc_cyc_timestamp(decoder);
3069                         break;
3070
3071                 case INTEL_PT_CBR:
3072                         intel_pt_calc_cbr(decoder);
3073                         if (decoder->cbr != decoder->cbr_seen) {
3074                                 decoder->state.type = 0;
3075                                 return 0;
3076                         }
3077                         break;
3078
3079                 case INTEL_PT_MODE_EXEC:
3080                         decoder->exec_mode = decoder->packet.payload;
3081                         break;
3082
3083                 case INTEL_PT_MODE_TSX:
3084                         /* MODE_TSX need not be followed by FUP */
3085                         if (!decoder->pge || decoder->in_psb) {
3086                                 intel_pt_update_in_tx(decoder);
3087                                 break;
3088                         }
3089                         err = intel_pt_mode_tsx(decoder, &no_tip);
3090                         if (err)
3091                                 return err;
3092                         goto next;
3093
3094                 case INTEL_PT_BAD: /* Does not happen */
3095                         return intel_pt_bug(decoder);
3096
3097                 case INTEL_PT_PSBEND:
3098                 case INTEL_PT_VMCS:
3099                 case INTEL_PT_MNT:
3100                 case INTEL_PT_PAD:
3101                         break;
3102
3103                 case INTEL_PT_PTWRITE_IP:
3104                         decoder->fup_ptw_payload = decoder->packet.payload;
3105                         err = intel_pt_get_next_packet(decoder);
3106                         if (err)
3107                                 return err;
3108                         if (decoder->packet.type == INTEL_PT_FUP) {
3109                                 decoder->set_fup_ptw = true;
3110                                 no_tip = true;
3111                         } else {
3112                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3113                                                 decoder->pos);
3114                         }
3115                         goto next;
3116
3117                 case INTEL_PT_PTWRITE:
3118                         decoder->state.type = INTEL_PT_PTW;
3119                         decoder->state.from_ip = decoder->ip;
3120                         decoder->state.to_ip = 0;
3121                         decoder->state.ptw_payload = decoder->packet.payload;
3122                         return 0;
3123
3124                 case INTEL_PT_MWAIT:
3125                         decoder->fup_mwait_payload = decoder->packet.payload;
3126                         decoder->set_fup_mwait = true;
3127                         break;
3128
3129                 case INTEL_PT_PWRE:
3130                         if (decoder->set_fup_mwait) {
3131                                 decoder->fup_pwre_payload =
3132                                                         decoder->packet.payload;
3133                                 decoder->set_fup_pwre = true;
3134                                 break;
3135                         }
3136                         decoder->state.type = INTEL_PT_PWR_ENTRY;
3137                         decoder->state.from_ip = decoder->ip;
3138                         decoder->state.to_ip = 0;
3139                         decoder->state.pwrx_payload = decoder->packet.payload;
3140                         return 0;
3141
3142                 case INTEL_PT_EXSTOP_IP:
3143                         err = intel_pt_get_next_packet(decoder);
3144                         if (err)
3145                                 return err;
3146                         if (decoder->packet.type == INTEL_PT_FUP) {
3147                                 decoder->set_fup_exstop = true;
3148                                 no_tip = true;
3149                         } else {
3150                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3151                                                 decoder->pos);
3152                         }
3153                         goto next;
3154
3155                 case INTEL_PT_EXSTOP:
3156                         decoder->state.type = INTEL_PT_EX_STOP;
3157                         decoder->state.from_ip = decoder->ip;
3158                         decoder->state.to_ip = 0;
3159                         return 0;
3160
3161                 case INTEL_PT_PWRX:
3162                         decoder->state.type = INTEL_PT_PWR_EXIT;
3163                         decoder->state.from_ip = decoder->ip;
3164                         decoder->state.to_ip = 0;
3165                         decoder->state.pwrx_payload = decoder->packet.payload;
3166                         return 0;
3167
3168                 case INTEL_PT_BBP:
3169                         intel_pt_bbp(decoder);
3170                         break;
3171
3172                 case INTEL_PT_BIP:
3173                         intel_pt_bip(decoder);
3174                         break;
3175
3176                 case INTEL_PT_BEP:
3177                         decoder->state.type = INTEL_PT_BLK_ITEMS;
3178                         decoder->state.from_ip = decoder->ip;
3179                         decoder->state.to_ip = 0;
3180                         return 0;
3181
3182                 case INTEL_PT_BEP_IP:
3183                         err = intel_pt_get_next_packet(decoder);
3184                         if (err)
3185                                 return err;
3186                         if (decoder->packet.type == INTEL_PT_FUP) {
3187                                 decoder->set_fup_bep = true;
3188                                 no_tip = true;
3189                         } else {
3190                                 intel_pt_log_at("ERROR: Missing FUP after BEP",
3191                                                 decoder->pos);
3192                         }
3193                         goto next;
3194
3195                 default:
3196                         return intel_pt_bug(decoder);
3197                 }
3198         }
3199 }
3200
3201 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3202 {
3203         return decoder->packet.count &&
3204                (decoder->have_last_ip || decoder->packet.count == 3 ||
3205                 decoder->packet.count == 6);
3206 }
3207
3208 /* Walk PSB+ packets to get in sync. */
3209 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3210 {
3211         int err;
3212
3213         decoder->in_psb = true;
3214
3215         while (1) {
3216                 err = intel_pt_get_next_packet(decoder);
3217                 if (err)
3218                         goto out;
3219
3220                 switch (decoder->packet.type) {
3221                 case INTEL_PT_TIP_PGD:
3222                         decoder->continuous_period = false;
3223                         __fallthrough;
3224                 case INTEL_PT_TIP_PGE:
3225                 case INTEL_PT_TIP:
3226                 case INTEL_PT_PTWRITE:
3227                 case INTEL_PT_PTWRITE_IP:
3228                 case INTEL_PT_EXSTOP:
3229                 case INTEL_PT_EXSTOP_IP:
3230                 case INTEL_PT_MWAIT:
3231                 case INTEL_PT_PWRE:
3232                 case INTEL_PT_PWRX:
3233                 case INTEL_PT_BBP:
3234                 case INTEL_PT_BIP:
3235                 case INTEL_PT_BEP:
3236                 case INTEL_PT_BEP_IP:
3237                         intel_pt_log("ERROR: Unexpected packet\n");
3238                         err = -ENOENT;
3239                         goto out;
3240
3241                 case INTEL_PT_FUP:
3242                         decoder->pge = true;
3243                         if (intel_pt_have_ip(decoder)) {
3244                                 uint64_t current_ip = decoder->ip;
3245
3246                                 intel_pt_set_ip(decoder);
3247                                 decoder->psb_ip = decoder->ip;
3248                                 if (current_ip)
3249                                         intel_pt_log_to("Setting IP",
3250                                                         decoder->ip);
3251                         }
3252                         break;
3253
3254                 case INTEL_PT_MTC:
3255                         intel_pt_calc_mtc_timestamp(decoder);
3256                         break;
3257
3258                 case INTEL_PT_TSC:
3259                         intel_pt_calc_tsc_timestamp(decoder);
3260                         break;
3261
3262                 case INTEL_PT_TMA:
3263                         intel_pt_calc_tma(decoder);
3264                         break;
3265
3266                 case INTEL_PT_CYC:
3267                         intel_pt_calc_cyc_timestamp(decoder);
3268                         break;
3269
3270                 case INTEL_PT_CBR:
3271                         intel_pt_calc_cbr(decoder);
3272                         break;
3273
3274                 case INTEL_PT_PIP:
3275                         intel_pt_set_pip(decoder);
3276                         break;
3277
3278                 case INTEL_PT_MODE_EXEC:
3279                         decoder->exec_mode = decoder->packet.payload;
3280                         break;
3281
3282                 case INTEL_PT_MODE_TSX:
3283                         intel_pt_update_in_tx(decoder);
3284                         break;
3285
3286                 case INTEL_PT_TRACESTOP:
3287                         decoder->pge = false;
3288                         decoder->continuous_period = false;
3289                         intel_pt_clear_tx_flags(decoder);
3290                         __fallthrough;
3291
3292                 case INTEL_PT_TNT:
3293                         decoder->have_tma = false;
3294                         intel_pt_log("ERROR: Unexpected packet\n");
3295                         if (decoder->ip)
3296                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
3297                         else
3298                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3299                         err = -ENOENT;
3300                         goto out;
3301
3302                 case INTEL_PT_BAD: /* Does not happen */
3303                         err = intel_pt_bug(decoder);
3304                         goto out;
3305
3306                 case INTEL_PT_OVF:
3307                         err = intel_pt_overflow(decoder);
3308                         goto out;
3309
3310                 case INTEL_PT_PSBEND:
3311                         err = 0;
3312                         goto out;
3313
3314                 case INTEL_PT_PSB:
3315                 case INTEL_PT_VMCS:
3316                 case INTEL_PT_MNT:
3317                 case INTEL_PT_PAD:
3318                 default:
3319                         break;
3320                 }
3321         }
3322 out:
3323         decoder->in_psb = false;
3324
3325         return err;
3326 }
3327
3328 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3329 {
3330         int err;
3331
3332         while (1) {
3333                 err = intel_pt_get_next_packet(decoder);
3334                 if (err)
3335                         return err;
3336
3337                 switch (decoder->packet.type) {
3338                 case INTEL_PT_TIP_PGD:
3339                         decoder->continuous_period = false;
3340                         decoder->pge = false;
3341                         if (intel_pt_have_ip(decoder))
3342                                 intel_pt_set_ip(decoder);
3343                         if (!decoder->ip)
3344                                 break;
3345                         decoder->state.type |= INTEL_PT_TRACE_END;
3346                         return 0;
3347
3348                 case INTEL_PT_TIP_PGE:
3349                         decoder->pge = true;
3350                         intel_pt_mtc_cyc_cnt_pge(decoder);
3351                         if (intel_pt_have_ip(decoder))
3352                                 intel_pt_set_ip(decoder);
3353                         if (!decoder->ip)
3354                                 break;
3355                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3356                         return 0;
3357
3358                 case INTEL_PT_TIP:
3359                         decoder->pge = true;
3360                         if (intel_pt_have_ip(decoder))
3361                                 intel_pt_set_ip(decoder);
3362                         if (!decoder->ip)
3363                                 break;
3364                         return 0;
3365
3366                 case INTEL_PT_FUP:
3367                         if (intel_pt_have_ip(decoder))
3368                                 intel_pt_set_ip(decoder);
3369                         if (decoder->ip)
3370                                 return 0;
3371                         break;
3372
3373                 case INTEL_PT_MTC:
3374                         intel_pt_calc_mtc_timestamp(decoder);
3375                         break;
3376
3377                 case INTEL_PT_TSC:
3378                         intel_pt_calc_tsc_timestamp(decoder);
3379                         break;
3380
3381                 case INTEL_PT_TMA:
3382                         intel_pt_calc_tma(decoder);
3383                         break;
3384
3385                 case INTEL_PT_CYC:
3386                         intel_pt_calc_cyc_timestamp(decoder);
3387                         break;
3388
3389                 case INTEL_PT_CBR:
3390                         intel_pt_calc_cbr(decoder);
3391                         break;
3392
3393                 case INTEL_PT_PIP:
3394                         intel_pt_set_pip(decoder);
3395                         break;
3396
3397                 case INTEL_PT_MODE_EXEC:
3398                         decoder->exec_mode = decoder->packet.payload;
3399                         break;
3400
3401                 case INTEL_PT_MODE_TSX:
3402                         intel_pt_update_in_tx(decoder);
3403                         break;
3404
3405                 case INTEL_PT_OVF:
3406                         return intel_pt_overflow(decoder);
3407
3408                 case INTEL_PT_BAD: /* Does not happen */
3409                         return intel_pt_bug(decoder);
3410
3411                 case INTEL_PT_TRACESTOP:
3412                         decoder->pge = false;
3413                         decoder->continuous_period = false;
3414                         intel_pt_clear_tx_flags(decoder);
3415                         decoder->have_tma = false;
3416                         break;
3417
3418                 case INTEL_PT_PSB:
3419                         decoder->state.psb_offset = decoder->pos;
3420                         decoder->psb_ip = 0;
3421                         decoder->last_ip = 0;
3422                         decoder->have_last_ip = true;
3423                         intel_pt_clear_stack(&decoder->stack);
3424                         err = intel_pt_walk_psb(decoder);
3425                         if (err)
3426                                 return err;
3427                         decoder->state.type = INTEL_PT_PSB_EVT;
3428                         decoder->state.from_ip = decoder->psb_ip;
3429                         decoder->state.to_ip = 0;
3430                         return 0;
3431
3432                 case INTEL_PT_TNT:
3433                 case INTEL_PT_PSBEND:
3434                 case INTEL_PT_VMCS:
3435                 case INTEL_PT_MNT:
3436                 case INTEL_PT_PAD:
3437                 case INTEL_PT_PTWRITE:
3438                 case INTEL_PT_PTWRITE_IP:
3439                 case INTEL_PT_EXSTOP:
3440                 case INTEL_PT_EXSTOP_IP:
3441                 case INTEL_PT_MWAIT:
3442                 case INTEL_PT_PWRE:
3443                 case INTEL_PT_PWRX:
3444                 case INTEL_PT_BBP:
3445                 case INTEL_PT_BIP:
3446                 case INTEL_PT_BEP:
3447                 case INTEL_PT_BEP_IP:
3448                 default:
3449                         break;
3450                 }
3451         }
3452 }
3453
3454 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3455 {
3456         int err;
3457
3458         decoder->set_fup_tx_flags = false;
3459         decoder->set_fup_ptw = false;
3460         decoder->set_fup_mwait = false;
3461         decoder->set_fup_pwre = false;
3462         decoder->set_fup_exstop = false;
3463         decoder->set_fup_bep = false;
3464
3465         if (!decoder->branch_enable) {
3466                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3467                 decoder->overflow = false;
3468                 decoder->state.type = 0; /* Do not have a sample */
3469                 return 0;
3470         }
3471
3472         intel_pt_log("Scanning for full IP\n");
3473         err = intel_pt_walk_to_ip(decoder);
3474         if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3475                 return err;
3476
3477         /* In hop mode, resample to get the to_ip as an "instruction" sample */
3478         if (decoder->hop)
3479                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3480         else
3481                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3482         decoder->overflow = false;
3483
3484         decoder->state.from_ip = 0;
3485         decoder->state.to_ip = decoder->ip;
3486         intel_pt_log_to("Setting IP", decoder->ip);
3487
3488         return 0;
3489 }
3490
3491 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3492 {
3493         const unsigned char *end = decoder->buf + decoder->len;
3494         size_t i;
3495
3496         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3497                 if (i > decoder->len)
3498                         continue;
3499                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3500                         return i;
3501         }
3502         return 0;
3503 }
3504
3505 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3506 {
3507         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3508         const char *psb = INTEL_PT_PSB_STR;
3509
3510         if (rest_psb > decoder->len ||
3511             memcmp(decoder->buf, psb + part_psb, rest_psb))
3512                 return 0;
3513
3514         return rest_psb;
3515 }
3516
3517 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3518                                   int part_psb)
3519 {
3520         int rest_psb, ret;
3521
3522         decoder->pos += decoder->len;
3523         decoder->len = 0;
3524
3525         ret = intel_pt_get_next_data(decoder, false);
3526         if (ret)
3527                 return ret;
3528
3529         rest_psb = intel_pt_rest_psb(decoder, part_psb);
3530         if (!rest_psb)
3531                 return 0;
3532
3533         decoder->pos -= part_psb;
3534         decoder->next_buf = decoder->buf + rest_psb;
3535         decoder->next_len = decoder->len - rest_psb;
3536         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3537         decoder->buf = decoder->temp_buf;
3538         decoder->len = INTEL_PT_PSB_LEN;
3539
3540         return 0;
3541 }
3542
3543 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3544 {
3545         unsigned char *next;
3546         int ret;
3547
3548         intel_pt_log("Scanning for PSB\n");
3549         while (1) {
3550                 if (!decoder->len) {
3551                         ret = intel_pt_get_next_data(decoder, false);
3552                         if (ret)
3553                                 return ret;
3554                 }
3555
3556                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3557                               INTEL_PT_PSB_LEN);
3558                 if (!next) {
3559                         int part_psb;
3560
3561                         part_psb = intel_pt_part_psb(decoder);
3562                         if (part_psb) {
3563                                 ret = intel_pt_get_split_psb(decoder, part_psb);
3564                                 if (ret)
3565                                         return ret;
3566                         } else {
3567                                 decoder->pos += decoder->len;
3568                                 decoder->len = 0;
3569                         }
3570                         continue;
3571                 }
3572
3573                 decoder->pkt_step = next - decoder->buf;
3574                 return intel_pt_get_next_packet(decoder);
3575         }
3576 }
3577
3578 static int intel_pt_sync(struct intel_pt_decoder *decoder)
3579 {
3580         int err;
3581
3582         decoder->pge = false;
3583         decoder->continuous_period = false;
3584         decoder->have_last_ip = false;
3585         decoder->last_ip = 0;
3586         decoder->psb_ip = 0;
3587         decoder->ip = 0;
3588         intel_pt_clear_stack(&decoder->stack);
3589
3590         err = intel_pt_scan_for_psb(decoder);
3591         if (err)
3592                 return err;
3593
3594         if (decoder->vm_time_correlation) {
3595                 decoder->in_psb = true;
3596                 if (!decoder->timestamp)
3597                         decoder->timestamp = 1;
3598                 decoder->state.type = 0;
3599                 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3600                 return 0;
3601         }
3602
3603         decoder->have_last_ip = true;
3604         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
3605
3606         err = intel_pt_walk_psb(decoder);
3607         if (err)
3608                 return err;
3609
3610         decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3611         decoder->state.from_ip = decoder->psb_ip;
3612         decoder->state.to_ip = 0;
3613
3614         if (decoder->ip) {
3615                 /*
3616                  * In hop mode, resample to get the PSB FUP ip as an
3617                  * "instruction" sample.
3618                  */
3619                 if (decoder->hop)
3620                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3621                 else
3622                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3623         }
3624
3625         return 0;
3626 }
3627
3628 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3629 {
3630         uint64_t est = decoder->sample_insn_cnt << 1;
3631
3632         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
3633                 goto out;
3634
3635         est *= decoder->max_non_turbo_ratio;
3636         est /= decoder->cbr;
3637 out:
3638         return decoder->sample_timestamp + est;
3639 }
3640
3641 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
3642 {
3643         int err;
3644
3645         do {
3646                 decoder->state.type = INTEL_PT_BRANCH;
3647                 decoder->state.flags = 0;
3648
3649                 switch (decoder->pkt_state) {
3650                 case INTEL_PT_STATE_NO_PSB:
3651                         err = intel_pt_sync(decoder);
3652                         break;
3653                 case INTEL_PT_STATE_NO_IP:
3654                         decoder->have_last_ip = false;
3655                         decoder->last_ip = 0;
3656                         decoder->ip = 0;
3657                         __fallthrough;
3658                 case INTEL_PT_STATE_ERR_RESYNC:
3659                         err = intel_pt_sync_ip(decoder);
3660                         break;
3661                 case INTEL_PT_STATE_IN_SYNC:
3662                         err = intel_pt_walk_trace(decoder);
3663                         break;
3664                 case INTEL_PT_STATE_TNT:
3665                 case INTEL_PT_STATE_TNT_CONT:
3666                         err = intel_pt_walk_tnt(decoder);
3667                         if (err == -EAGAIN)
3668                                 err = intel_pt_walk_trace(decoder);
3669                         break;
3670                 case INTEL_PT_STATE_TIP:
3671                 case INTEL_PT_STATE_TIP_PGD:
3672                         err = intel_pt_walk_tip(decoder);
3673                         break;
3674                 case INTEL_PT_STATE_FUP:
3675                         err = intel_pt_walk_fup(decoder);
3676                         if (err == -EAGAIN)
3677                                 err = intel_pt_walk_fup_tip(decoder);
3678                         break;
3679                 case INTEL_PT_STATE_FUP_NO_TIP:
3680                         err = intel_pt_walk_fup(decoder);
3681                         if (err == -EAGAIN)
3682                                 err = intel_pt_walk_trace(decoder);
3683                         break;
3684                 case INTEL_PT_STATE_FUP_IN_PSB:
3685                         err = intel_pt_fup_in_psb(decoder);
3686                         break;
3687                 case INTEL_PT_STATE_RESAMPLE:
3688                         err = intel_pt_resample(decoder);
3689                         break;
3690                 case INTEL_PT_STATE_VM_TIME_CORRELATION:
3691                         err = intel_pt_vm_time_correlation(decoder);
3692                         break;
3693                 default:
3694                         err = intel_pt_bug(decoder);
3695                         break;
3696                 }
3697         } while (err == -ENOLINK);
3698
3699         if (err) {
3700                 decoder->state.err = intel_pt_ext_err(err);
3701                 decoder->state.from_ip = decoder->ip;
3702                 intel_pt_update_sample_time(decoder);
3703                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3704                 intel_pt_set_nr(decoder);
3705         } else {
3706                 decoder->state.err = 0;
3707                 if (decoder->cbr != decoder->cbr_seen) {
3708                         decoder->cbr_seen = decoder->cbr;
3709                         if (!decoder->state.type) {
3710                                 decoder->state.from_ip = decoder->ip;
3711                                 decoder->state.to_ip = 0;
3712                         }
3713                         decoder->state.type |= INTEL_PT_CBR_CHG;
3714                         decoder->state.cbr_payload = decoder->cbr_payload;
3715                         decoder->state.cbr = decoder->cbr;
3716                 }
3717                 if (intel_pt_sample_time(decoder->pkt_state)) {
3718                         intel_pt_update_sample_time(decoder);
3719                         if (decoder->sample_cyc) {
3720                                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3721                                 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3722                                 decoder->sample_cyc = false;
3723                         }
3724                 }
3725                 /*
3726                  * When using only TSC/MTC to compute cycles, IPC can be
3727                  * sampled as soon as the cycle count changes.
3728                  */
3729                 if (!decoder->have_cyc)
3730                         decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3731         }
3732
3733          /* Let PSB event always have TSC timestamp */
3734         if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
3735                 decoder->sample_timestamp = decoder->tsc_timestamp;
3736
3737         decoder->state.from_nr = decoder->nr;
3738         decoder->state.to_nr = decoder->next_nr;
3739         decoder->nr = decoder->next_nr;
3740
3741         decoder->state.timestamp = decoder->sample_timestamp;
3742         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
3743         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
3744         decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
3745
3746         return &decoder->state;
3747 }
3748
3749 /**
3750  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
3751  * @buf: pointer to buffer pointer
3752  * @len: size of buffer
3753  *
3754  * Updates the buffer pointer to point to the start of the next PSB packet if
3755  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
3756  * @len is adjusted accordingly.
3757  *
3758  * Return: %true if a PSB packet is found, %false otherwise.
3759  */
3760 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
3761 {
3762         unsigned char *next;
3763
3764         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3765         if (next) {
3766                 *len -= next - *buf;
3767                 *buf = next;
3768                 return true;
3769         }
3770         return false;
3771 }
3772
3773 /**
3774  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
3775  *                     packet.
3776  * @buf: pointer to buffer pointer
3777  * @len: size of buffer
3778  *
3779  * Updates the buffer pointer to point to the start of the following PSB packet
3780  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
3781  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
3782  *
3783  * Return: %true if a PSB packet is found, %false otherwise.
3784  */
3785 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
3786 {
3787         unsigned char *next;
3788
3789         if (!*len)
3790                 return false;
3791
3792         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3793         if (next) {
3794                 *len -= next - *buf;
3795                 *buf = next;
3796                 return true;
3797         }
3798         return false;
3799 }
3800
3801 /**
3802  * intel_pt_last_psb - find the last PSB packet in a buffer.
3803  * @buf: buffer
3804  * @len: size of buffer
3805  *
3806  * This function finds the last PSB in a buffer.
3807  *
3808  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
3809  */
3810 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
3811 {
3812         const char *n = INTEL_PT_PSB_STR;
3813         unsigned char *p;
3814         size_t k;
3815
3816         if (len < INTEL_PT_PSB_LEN)
3817                 return NULL;
3818
3819         k = len - INTEL_PT_PSB_LEN + 1;
3820         while (1) {
3821                 p = memrchr(buf, n[0], k);
3822                 if (!p)
3823                         return NULL;
3824                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
3825                         return p;
3826                 k = p - buf;
3827                 if (!k)
3828                         return NULL;
3829         }
3830 }
3831
3832 /**
3833  * intel_pt_next_tsc - find and return next TSC.
3834  * @buf: buffer
3835  * @len: size of buffer
3836  * @tsc: TSC value returned
3837  * @rem: returns remaining size when TSC is found
3838  *
3839  * Find a TSC packet in @buf and return the TSC value.  This function assumes
3840  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
3841  * PSBEND packet is found.
3842  *
3843  * Return: %true if TSC is found, false otherwise.
3844  */
3845 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
3846                               size_t *rem)
3847 {
3848         enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
3849         struct intel_pt_pkt packet;
3850         int ret;
3851
3852         while (len) {
3853                 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
3854                 if (ret <= 0)
3855                         return false;
3856                 if (packet.type == INTEL_PT_TSC) {
3857                         *tsc = packet.payload;
3858                         *rem = len;
3859                         return true;
3860                 }
3861                 if (packet.type == INTEL_PT_PSBEND)
3862                         return false;
3863                 buf += ret;
3864                 len -= ret;
3865         }
3866         return false;
3867 }
3868
3869 /**
3870  * intel_pt_tsc_cmp - compare 7-byte TSCs.
3871  * @tsc1: first TSC to compare
3872  * @tsc2: second TSC to compare
3873  *
3874  * This function compares 7-byte TSC values allowing for the possibility that
3875  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
3876  * around so for that purpose this function assumes the absolute difference is
3877  * less than half the maximum difference.
3878  *
3879  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
3880  * after @tsc2.
3881  */
3882 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
3883 {
3884         const uint64_t halfway = (1ULL << 55);
3885
3886         if (tsc1 == tsc2)
3887                 return 0;
3888
3889         if (tsc1 < tsc2) {
3890                 if (tsc2 - tsc1 < halfway)
3891                         return -1;
3892                 else
3893                         return 1;
3894         } else {
3895                 if (tsc1 - tsc2 < halfway)
3896                         return 1;
3897                 else
3898                         return -1;
3899         }
3900 }
3901
3902 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
3903
3904 /**
3905  * adj_for_padding - adjust overlap to account for padding.
3906  * @buf_b: second buffer
3907  * @buf_a: first buffer
3908  * @len_a: size of first buffer
3909  *
3910  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
3911  * accordingly.
3912  *
3913  * Return: A pointer into @buf_b from where non-overlapped data starts
3914  */
3915 static unsigned char *adj_for_padding(unsigned char *buf_b,
3916                                       unsigned char *buf_a, size_t len_a)
3917 {
3918         unsigned char *p = buf_b - MAX_PADDING;
3919         unsigned char *q = buf_a + len_a - MAX_PADDING;
3920         int i;
3921
3922         for (i = MAX_PADDING; i; i--, p++, q++) {
3923                 if (*p != *q)
3924                         break;
3925         }
3926
3927         return p;
3928 }
3929
3930 /**
3931  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
3932  *                             using TSC.
3933  * @buf_a: first buffer
3934  * @len_a: size of first buffer
3935  * @buf_b: second buffer
3936  * @len_b: size of second buffer
3937  * @consecutive: returns true if there is data in buf_b that is consecutive
3938  *               to buf_a
3939  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
3940  *
3941  * If the trace contains TSC we can look at the last TSC of @buf_a and the
3942  * first TSC of @buf_b in order to determine if the buffers overlap, and then
3943  * walk forward in @buf_b until a later TSC is found.  A precondition is that
3944  * @buf_a and @buf_b are positioned at a PSB.
3945  *
3946  * Return: A pointer into @buf_b from where non-overlapped data starts, or
3947  * @buf_b + @len_b if there is no non-overlapped data.
3948  */
3949 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
3950                                                 size_t len_a,
3951                                                 unsigned char *buf_b,
3952                                                 size_t len_b, bool *consecutive,
3953                                                 bool ooo_tsc)
3954 {
3955         uint64_t tsc_a, tsc_b;
3956         unsigned char *p;
3957         size_t len, rem_a, rem_b;
3958
3959         p = intel_pt_last_psb(buf_a, len_a);
3960         if (!p)
3961                 return buf_b; /* No PSB in buf_a => no overlap */
3962
3963         len = len_a - (p - buf_a);
3964         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
3965                 /* The last PSB+ in buf_a is incomplete, so go back one more */
3966                 len_a -= len;
3967                 p = intel_pt_last_psb(buf_a, len_a);
3968                 if (!p)
3969                         return buf_b; /* No full PSB+ => assume no overlap */
3970                 len = len_a - (p - buf_a);
3971                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
3972                         return buf_b; /* No TSC in buf_a => assume no overlap */
3973         }
3974
3975         while (1) {
3976                 /* Ignore PSB+ with no TSC */
3977                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
3978                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
3979
3980                         /* Same TSC, so buffers are consecutive */
3981                         if (!cmp && rem_b >= rem_a) {
3982                                 unsigned char *start;
3983
3984                                 *consecutive = true;
3985                                 start = buf_b + len_b - (rem_b - rem_a);
3986                                 return adj_for_padding(start, buf_a, len_a);
3987                         }
3988                         if (cmp < 0 && !ooo_tsc)
3989                                 return buf_b; /* tsc_a < tsc_b => no overlap */
3990                 }
3991
3992                 if (!intel_pt_step_psb(&buf_b, &len_b))
3993                         return buf_b + len_b; /* No PSB in buf_b => no data */
3994         }
3995 }
3996
3997 /**
3998  * intel_pt_find_overlap - determine start of non-overlapped trace data.
3999  * @buf_a: first buffer
4000  * @len_a: size of first buffer
4001  * @buf_b: second buffer
4002  * @len_b: size of second buffer
4003  * @have_tsc: can use TSC packets to detect overlap
4004  * @consecutive: returns true if there is data in buf_b that is consecutive
4005  *               to buf_a
4006  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4007  *
4008  * When trace samples or snapshots are recorded there is the possibility that
4009  * the data overlaps.  Note that, for the purposes of decoding, data is only
4010  * useful if it begins with a PSB packet.
4011  *
4012  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4013  * @buf_b + @len_b if there is no non-overlapped data.
4014  */
4015 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
4016                                      unsigned char *buf_b, size_t len_b,
4017                                      bool have_tsc, bool *consecutive,
4018                                      bool ooo_tsc)
4019 {
4020         unsigned char *found;
4021
4022         /* Buffer 'b' must start at PSB so throw away everything before that */
4023         if (!intel_pt_next_psb(&buf_b, &len_b))
4024                 return buf_b + len_b; /* No PSB */
4025
4026         if (!intel_pt_next_psb(&buf_a, &len_a))
4027                 return buf_b; /* No overlap */
4028
4029         if (have_tsc) {
4030                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4031                                                   consecutive, ooo_tsc);
4032                 if (found)
4033                         return found;
4034         }
4035
4036         /*
4037          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4038          * we can ignore the first part of buffer 'a'.
4039          */
4040         while (len_b < len_a) {
4041                 if (!intel_pt_step_psb(&buf_a, &len_a))
4042                         return buf_b; /* No overlap */
4043         }
4044
4045         /* Now len_b >= len_a */
4046         while (1) {
4047                 /* Potential overlap so check the bytes */
4048                 found = memmem(buf_a, len_a, buf_b, len_a);
4049                 if (found) {
4050                         *consecutive = true;
4051                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
4052                 }
4053
4054                 /* Try again at next PSB in buffer 'a' */
4055                 if (!intel_pt_step_psb(&buf_a, &len_a))
4056                         return buf_b; /* No overlap */
4057         }
4058 }
4059
4060 /**
4061  * struct fast_forward_data - data used by intel_pt_ff_cb().
4062  * @timestamp: timestamp to fast forward towards
4063  * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4064  *                 the fast forward timestamp.
4065  */
4066 struct fast_forward_data {
4067         uint64_t timestamp;
4068         uint64_t buf_timestamp;
4069 };
4070
4071 /**
4072  * intel_pt_ff_cb - fast forward lookahead callback.
4073  * @buffer: Intel PT trace buffer
4074  * @data: opaque pointer to fast forward data (struct fast_forward_data)
4075  *
4076  * Determine if @buffer trace is past the fast forward timestamp.
4077  *
4078  * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4079  *         timestamp, and 0 otherwise.
4080  */
4081 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
4082 {
4083         struct fast_forward_data *d = data;
4084         unsigned char *buf;
4085         uint64_t tsc;
4086         size_t rem;
4087         size_t len;
4088
4089         buf = (unsigned char *)buffer->buf;
4090         len = buffer->len;
4091
4092         if (!intel_pt_next_psb(&buf, &len) ||
4093             !intel_pt_next_tsc(buf, len, &tsc, &rem))
4094                 return 0;
4095
4096         tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4097
4098         intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4099                      tsc, buffer->ref_timestamp);
4100
4101         /*
4102          * If the buffer contains a timestamp earlier that the fast forward
4103          * timestamp, then record it, else stop.
4104          */
4105         if (tsc < d->timestamp)
4106                 d->buf_timestamp = buffer->ref_timestamp;
4107         else
4108                 return 1;
4109
4110         return 0;
4111 }
4112
4113 /**
4114  * intel_pt_fast_forward - reposition decoder forwards.
4115  * @decoder: Intel PT decoder
4116  * @timestamp: timestamp to fast forward towards
4117  *
4118  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4119  *
4120  * Return: 0 on success or negative error code on failure.
4121  */
4122 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4123 {
4124         struct fast_forward_data d = { .timestamp = timestamp };
4125         unsigned char *buf;
4126         size_t len;
4127         int err;
4128
4129         intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4130
4131         /* Find buffer timestamp of buffer to fast forward to */
4132         err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4133         if (err < 0)
4134                 return err;
4135
4136         /* Walk to buffer with same buffer timestamp */
4137         if (d.buf_timestamp) {
4138                 do {
4139                         decoder->pos += decoder->len;
4140                         decoder->len = 0;
4141                         err = intel_pt_get_next_data(decoder, true);
4142                         /* -ENOLINK means non-consecutive trace */
4143                         if (err && err != -ENOLINK)
4144                                 return err;
4145                 } while (decoder->buf_timestamp != d.buf_timestamp);
4146         }
4147
4148         if (!decoder->buf)
4149                 return 0;
4150
4151         buf = (unsigned char *)decoder->buf;
4152         len = decoder->len;
4153
4154         if (!intel_pt_next_psb(&buf, &len))
4155                 return 0;
4156
4157         /*
4158          * Walk PSBs while the PSB timestamp is less than the fast forward
4159          * timestamp.
4160          */
4161         do {
4162                 uint64_t tsc;
4163                 size_t rem;
4164
4165                 if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4166                         break;
4167                 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4168                 /*
4169                  * A TSC packet can slip past MTC packets but, after fast
4170                  * forward, decoding starts at the TSC timestamp. That means
4171                  * the timestamps may not be exactly the same as the timestamps
4172                  * that would have been decoded without fast forward.
4173                  */
4174                 if (tsc < timestamp) {
4175                         intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4176                         decoder->pos += decoder->len - len;
4177                         decoder->buf = buf;
4178                         decoder->len = len;
4179                         intel_pt_reposition(decoder);
4180                 } else {
4181                         break;
4182                 }
4183         } while (intel_pt_step_psb(&buf, &len));
4184
4185         return 0;
4186 }