tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_pt_decoder.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2014, Intel Corporation.
   5  */
   6
   7 #ifndef _GNU_SOURCE
   8 #define _GNU_SOURCE
   9 #endif
  10 #include <stdlib.h>
  11 #include <stdbool.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <linux/compiler.h>
  17 #include <linux/string.h>
  18 #include <linux/zalloc.h>
  19
  20 #include "../auxtrace.h"
  21
  22 #include "intel-pt-insn-decoder.h"
  23 #include "intel-pt-pkt-decoder.h"
  24 #include "intel-pt-decoder.h"
  25 #include "intel-pt-log.h"
  26
  27 #define BITULL(x) (1ULL << (x))
  28
  29 /* IA32_RTIT_CTL MSR bits */
  30 #define INTEL_PT_CYC_ENABLE             BITULL(1)
  31 #define INTEL_PT_CYC_THRESHOLD          (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
  32 #define INTEL_PT_CYC_THRESHOLD_SHIFT    19
  33
  34 #define INTEL_PT_BLK_SIZE 1024
  35
  36 #define BIT63 (((uint64_t)1 << 63))
  37
  38 #define SEVEN_BYTES 0xffffffffffffffULL
  39
  40 #define NO_VMCS 0xffffffffffULL
  41
  42 #define INTEL_PT_RETURN 1
  43
  44 /*
  45  * Default maximum number of loops with no packets consumed i.e. stuck in a
  46  * loop.
  47  */
  48 #define INTEL_PT_MAX_LOOPS 100000
  49
  50 struct intel_pt_blk {
  51         struct intel_pt_blk *prev;
  52         uint64_t ip[INTEL_PT_BLK_SIZE];
  53 };
  54
  55 struct intel_pt_stack {
  56         struct intel_pt_blk *blk;
  57         struct intel_pt_blk *spare;
  58         int pos;
  59 };
  60
  61 enum intel_pt_p_once {
  62         INTEL_PT_PRT_ONCE_UNK_VMCS,
  63         INTEL_PT_PRT_ONCE_ERANGE,
  64 };
  65
  66 enum intel_pt_pkt_state {
  67         INTEL_PT_STATE_NO_PSB,
  68         INTEL_PT_STATE_NO_IP,
  69         INTEL_PT_STATE_ERR_RESYNC,
  70         INTEL_PT_STATE_IN_SYNC,
  71         INTEL_PT_STATE_TNT_CONT,
  72         INTEL_PT_STATE_TNT,
  73         INTEL_PT_STATE_TIP,
  74         INTEL_PT_STATE_TIP_PGD,
  75         INTEL_PT_STATE_FUP,
  76         INTEL_PT_STATE_FUP_NO_TIP,
  77         INTEL_PT_STATE_FUP_IN_PSB,
  78         INTEL_PT_STATE_RESAMPLE,
  79         INTEL_PT_STATE_VM_TIME_CORRELATION,
  80 };
  81
  82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  83 {
  84         switch (pkt_state) {
  85         case INTEL_PT_STATE_NO_PSB:
  86         case INTEL_PT_STATE_NO_IP:
  87         case INTEL_PT_STATE_ERR_RESYNC:
  88         case INTEL_PT_STATE_IN_SYNC:
  89         case INTEL_PT_STATE_TNT_CONT:
  90         case INTEL_PT_STATE_RESAMPLE:
  91         case INTEL_PT_STATE_VM_TIME_CORRELATION:
  92                 return true;
  93         case INTEL_PT_STATE_TNT:
  94         case INTEL_PT_STATE_TIP:
  95         case INTEL_PT_STATE_TIP_PGD:
  96         case INTEL_PT_STATE_FUP:
  97         case INTEL_PT_STATE_FUP_NO_TIP:
  98         case INTEL_PT_STATE_FUP_IN_PSB:
  99                 return false;
 100         default:
 101                 return true;
 102         };
 103 }
 104
 105 #ifdef INTEL_PT_STRICT
 106 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
 107 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
 108 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
 109 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
 110 #else
 111 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
 112 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
 113 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
 114 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
 115 #endif
 116
 117 struct intel_pt_decoder {
 118         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 119         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 120                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 121                          uint64_t max_insn_cnt, void *data);
 122         bool (*pgd_ip)(uint64_t ip, void *data);
 123         int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
 124         struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);
 125         void *data;
 126         struct intel_pt_state state;
 127         const unsigned char *buf;
 128         size_t len;
 129         bool return_compression;
 130         bool branch_enable;
 131         bool mtc_insn;
 132         bool pge;
 133         bool have_tma;
 134         bool have_cyc;
 135         bool fixup_last_mtc;
 136         bool have_last_ip;
 137         bool in_psb;
 138         bool hop;
 139         bool leap;
 140         bool vm_time_correlation;
 141         bool vm_tm_corr_dry_run;
 142         bool vm_tm_corr_reliable;
 143         bool vm_tm_corr_same_buf;
 144         bool vm_tm_corr_continuous;
 145         bool nr;
 146         bool next_nr;
 147         enum intel_pt_param_flags flags;
 148         uint64_t pos;
 149         uint64_t last_ip;
 150         uint64_t ip;
 151         uint64_t pip_payload;
 152         uint64_t timestamp;
 153         uint64_t tsc_timestamp;
 154         uint64_t ref_timestamp;
 155         uint64_t buf_timestamp;
 156         uint64_t sample_timestamp;
 157         uint64_t ret_addr;
 158         uint64_t ctc_timestamp;
 159         uint64_t ctc_delta;
 160         uint64_t cycle_cnt;
 161         uint64_t cyc_ref_timestamp;
 162         uint64_t first_timestamp;
 163         uint64_t last_reliable_timestamp;
 164         uint64_t vmcs;
 165         uint64_t print_once;
 166         uint64_t last_ctc;
 167         uint32_t last_mtc;
 168         uint32_t tsc_ctc_ratio_n;
 169         uint32_t tsc_ctc_ratio_d;
 170         uint32_t tsc_ctc_mult;
 171         uint32_t tsc_slip;
 172         uint32_t ctc_rem_mask;
 173         int mtc_shift;
 174         struct intel_pt_stack stack;
 175         enum intel_pt_pkt_state pkt_state;
 176         enum intel_pt_pkt_ctx pkt_ctx;
 177         enum intel_pt_pkt_ctx prev_pkt_ctx;
 178         enum intel_pt_blk_type blk_type;
 179         int blk_type_pos;
 180         struct intel_pt_pkt packet;
 181         struct intel_pt_pkt tnt;
 182         int pkt_step;
 183         int pkt_len;
 184         int last_packet_type;
 185         unsigned int cbr;
 186         unsigned int cbr_seen;
 187         unsigned int max_non_turbo_ratio;
 188         double max_non_turbo_ratio_fp;
 189         double cbr_cyc_to_tsc;
 190         double calc_cyc_to_tsc;
 191         bool have_calc_cyc_to_tsc;
 192         int exec_mode;
 193         unsigned int insn_bytes;
 194         uint64_t period;
 195         enum intel_pt_period_type period_type;
 196         uint64_t tot_insn_cnt;
 197         uint64_t period_insn_cnt;
 198         uint64_t period_mask;
 199         uint64_t period_ticks;
 200         uint64_t last_masked_timestamp;
 201         uint64_t tot_cyc_cnt;
 202         uint64_t sample_tot_cyc_cnt;
 203         uint64_t base_cyc_cnt;
 204         uint64_t cyc_cnt_timestamp;
 205         uint64_t ctl;
 206         uint64_t cyc_threshold;
 207         double tsc_to_cyc;
 208         bool continuous_period;
 209         bool overflow;
 210         bool set_fup_tx_flags;
 211         bool set_fup_ptw;
 212         bool set_fup_mwait;
 213         bool set_fup_pwre;
 214         bool set_fup_exstop;
 215         bool set_fup_bep;
 216         bool sample_cyc;
 217         unsigned int fup_tx_flags;
 218         unsigned int tx_flags;
 219         uint64_t fup_ptw_payload;
 220         uint64_t fup_mwait_payload;
 221         uint64_t fup_pwre_payload;
 222         uint64_t cbr_payload;
 223         uint64_t timestamp_insn_cnt;
 224         uint64_t sample_insn_cnt;
 225         uint64_t stuck_ip;
 226         int max_loops;
 227         int no_progress;
 228         int stuck_ip_prd;
 229         int stuck_ip_cnt;
 230         uint64_t psb_ip;
 231         const unsigned char *next_buf;
 232         size_t next_len;
 233         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 234 };
 235
 236 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 237 {
 238         int i;
 239
 240         for (i = 0; x != 1; i++)
 241                 x >>= 1;
 242
 243         return x << i;
 244 }
 245
 246 __printf(1, 2)
 247 static void p_log(const char *fmt, ...)
 248 {
 249         char buf[512];
 250         va_list args;
 251
 252         va_start(args, fmt);
 253         vsnprintf(buf, sizeof(buf), fmt, args);
 254         va_end(args);
 255
 256         fprintf(stderr, "%s\n", buf);
 257         intel_pt_log("%s\n", buf);
 258 }
 259
 260 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
 261                                 enum intel_pt_p_once id)
 262 {
 263         uint64_t bit = 1ULL << id;
 264
 265         if (decoder->print_once & bit)
 266                 return false;
 267         decoder->print_once |= bit;
 268         return true;
 269 }
 270
 271 static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
 272 {
 273         if (!(ctl & INTEL_PT_CYC_ENABLE))
 274                 return 0;
 275
 276         return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
 277 }
 278
 279 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 280 {
 281         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 282                 uint64_t period;
 283
 284                 period = intel_pt_lower_power_of_2(decoder->period);
 285                 decoder->period_mask  = ~(period - 1);
 286                 decoder->period_ticks = period;
 287         }
 288 }
 289
 290 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 291 {
 292         if (!d)
 293                 return 0;
 294         return (t / d) * n + ((t % d) * n) / d;
 295 }
 296
 297 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 298 {
 299         struct intel_pt_decoder *decoder;
 300
 301         if (!params->get_trace || !params->walk_insn)
 302                 return NULL;
 303
 304         decoder = zalloc(sizeof(struct intel_pt_decoder));
 305         if (!decoder)
 306                 return NULL;
 307
 308         decoder->get_trace          = params->get_trace;
 309         decoder->walk_insn          = params->walk_insn;
 310         decoder->pgd_ip             = params->pgd_ip;
 311         decoder->lookahead          = params->lookahead;
 312         decoder->findnew_vmcs_info  = params->findnew_vmcs_info;
 313         decoder->data               = params->data;
 314         decoder->return_compression = params->return_compression;
 315         decoder->branch_enable      = params->branch_enable;
 316         decoder->hop                = params->quick >= 1;
 317         decoder->leap               = params->quick >= 2;
 318         decoder->vm_time_correlation = params->vm_time_correlation;
 319         decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
 320         decoder->first_timestamp    = params->first_timestamp;
 321         decoder->last_reliable_timestamp = params->first_timestamp;
 322         decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
 323
 324         decoder->flags              = params->flags;
 325
 326         decoder->ctl                = params->ctl;
 327         decoder->period             = params->period;
 328         decoder->period_type        = params->period_type;
 329
 330         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 331         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 332
 333         decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
 334
 335         intel_pt_setup_period(decoder);
 336
 337         decoder->mtc_shift = params->mtc_period;
 338         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 339
 340         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 341         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 342
 343         if (!decoder->tsc_ctc_ratio_n)
 344                 decoder->tsc_ctc_ratio_d = 0;
 345
 346         if (decoder->tsc_ctc_ratio_d) {
 347                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 348                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 349                                                 decoder->tsc_ctc_ratio_d;
 350         }
 351
 352         /*
 353          * A TSC packet can slip past MTC packets so that the timestamp appears
 354          * to go backwards. One estimate is that can be up to about 40 CPU
 355          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 356          * slippage an order of magnitude more to be on the safe side.
 357          */
 358         decoder->tsc_slip = 0x10000;
 359
 360         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 361         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 362         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 363         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 364         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 365
 366         if (decoder->hop)
 367                 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
 368
 369         return decoder;
 370 }
 371
 372 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
 373                                   uint64_t first_timestamp)
 374 {
 375         decoder->first_timestamp = first_timestamp;
 376 }
 377
 378 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 379 {
 380         struct intel_pt_blk *blk = stack->blk;
 381
 382         stack->blk = blk->prev;
 383         if (!stack->spare)
 384                 stack->spare = blk;
 385         else
 386                 free(blk);
 387 }
 388
 389 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 390 {
 391         if (!stack->pos) {
 392                 if (!stack->blk)
 393                         return 0;
 394                 intel_pt_pop_blk(stack);
 395                 if (!stack->blk)
 396                         return 0;
 397                 stack->pos = INTEL_PT_BLK_SIZE;
 398         }
 399         return stack->blk->ip[--stack->pos];
 400 }
 401
 402 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 403 {
 404         struct intel_pt_blk *blk;
 405
 406         if (stack->spare) {
 407                 blk = stack->spare;
 408                 stack->spare = NULL;
 409         } else {
 410                 blk = malloc(sizeof(struct intel_pt_blk));
 411                 if (!blk)
 412                         return -ENOMEM;
 413         }
 414
 415         blk->prev = stack->blk;
 416         stack->blk = blk;
 417         stack->pos = 0;
 418         return 0;
 419 }
 420
 421 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 422 {
 423         int err;
 424
 425         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 426                 err = intel_pt_alloc_blk(stack);
 427                 if (err)
 428                         return err;
 429         }
 430
 431         stack->blk->ip[stack->pos++] = ip;
 432         return 0;
 433 }
 434
 435 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 436 {
 437         while (stack->blk)
 438                 intel_pt_pop_blk(stack);
 439         stack->pos = 0;
 440 }
 441
 442 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 443 {
 444         intel_pt_clear_stack(stack);
 445         zfree(&stack->blk);
 446         zfree(&stack->spare);
 447 }
 448
 449 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 450 {
 451         intel_pt_free_stack(&decoder->stack);
 452         free(decoder);
 453 }
 454
 455 static int intel_pt_ext_err(int code)
 456 {
 457         switch (code) {
 458         case -ENOMEM:
 459                 return INTEL_PT_ERR_NOMEM;
 460         case -ENOSYS:
 461                 return INTEL_PT_ERR_INTERN;
 462         case -EBADMSG:
 463                 return INTEL_PT_ERR_BADPKT;
 464         case -ENODATA:
 465                 return INTEL_PT_ERR_NODATA;
 466         case -EILSEQ:
 467                 return INTEL_PT_ERR_NOINSN;
 468         case -ENOENT:
 469                 return INTEL_PT_ERR_MISMAT;
 470         case -EOVERFLOW:
 471                 return INTEL_PT_ERR_OVR;
 472         case -ENOSPC:
 473                 return INTEL_PT_ERR_LOST;
 474         case -ELOOP:
 475                 return INTEL_PT_ERR_NELOOP;
 476         default:
 477                 return INTEL_PT_ERR_UNK;
 478         }
 479 }
 480
 481 static const char *intel_pt_err_msgs[] = {
 482         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 483         [INTEL_PT_ERR_INTERN] = "Internal error",
 484         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 485         [INTEL_PT_ERR_NODATA] = "No more data",
 486         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 487         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 488         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 489         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 490         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 491         [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
 492 };
 493
 494 int intel_pt__strerror(int code, char *buf, size_t buflen)
 495 {
 496         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 497                 code = INTEL_PT_ERR_UNK;
 498         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 499         return 0;
 500 }
 501
 502 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 503                                  uint64_t last_ip)
 504 {
 505         uint64_t ip;
 506
 507         switch (packet->count) {
 508         case 1:
 509                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 510                      packet->payload;
 511                 break;
 512         case 2:
 513                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 514                      packet->payload;
 515                 break;
 516         case 3:
 517                 ip = packet->payload;
 518                 /* Sign-extend 6-byte ip */
 519                 if (ip & (uint64_t)0x800000000000ULL)
 520                         ip |= (uint64_t)0xffff000000000000ULL;
 521                 break;
 522         case 4:
 523                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 524                      packet->payload;
 525                 break;
 526         case 6:
 527                 ip = packet->payload;
 528                 break;
 529         default:
 530                 return 0;
 531         }
 532
 533         return ip;
 534 }
 535
 536 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 537 {
 538         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 539         decoder->have_last_ip = true;
 540 }
 541
 542 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 543 {
 544         intel_pt_set_last_ip(decoder);
 545         decoder->ip = decoder->last_ip;
 546 }
 547
 548 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 549 {
 550         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 551                             decoder->buf);
 552 }
 553
 554 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 555 {
 556         intel_pt_log("ERROR: Internal error\n");
 557         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 558         return -ENOSYS;
 559 }
 560
 561 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 562 {
 563         decoder->tx_flags = 0;
 564 }
 565
 566 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 567 {
 568         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 569 }
 570
 571 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
 572 {
 573         decoder->pip_payload = decoder->packet.payload;
 574 }
 575
 576 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
 577 {
 578         decoder->next_nr = decoder->pip_payload & 1;
 579 }
 580
 581 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
 582 {
 583         decoder->nr = decoder->pip_payload & 1;
 584         decoder->next_nr = decoder->nr;
 585 }
 586
 587 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
 588 {
 589         intel_pt_update_pip(decoder);
 590         intel_pt_set_nr(decoder);
 591 }
 592
 593 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 594 {
 595         intel_pt_clear_tx_flags(decoder);
 596         decoder->have_tma = false;
 597         decoder->pkt_len = 1;
 598         decoder->pkt_step = 1;
 599         intel_pt_decoder_log_packet(decoder);
 600         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 601                 intel_pt_log("ERROR: Bad packet\n");
 602                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 603         }
 604         return -EBADMSG;
 605 }
 606
 607 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
 608 {
 609         decoder->sample_timestamp = decoder->timestamp;
 610         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
 611 }
 612
 613 static void intel_pt_reposition(struct intel_pt_decoder *decoder)
 614 {
 615         decoder->ip = 0;
 616         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 617         decoder->timestamp = 0;
 618         decoder->have_tma = false;
 619 }
 620
 621 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
 622 {
 623         struct intel_pt_buffer buffer = { .buf = 0, };
 624         int ret;
 625
 626         decoder->pkt_step = 0;
 627
 628         intel_pt_log("Getting more data\n");
 629         ret = decoder->get_trace(&buffer, decoder->data);
 630         if (ret)
 631                 return ret;
 632         decoder->buf = buffer.buf;
 633         decoder->len = buffer.len;
 634         if (!decoder->len) {
 635                 intel_pt_log("No more data\n");
 636                 return -ENODATA;
 637         }
 638         decoder->buf_timestamp = buffer.ref_timestamp;
 639         if (!buffer.consecutive || reposition) {
 640                 intel_pt_reposition(decoder);
 641                 decoder->ref_timestamp = buffer.ref_timestamp;
 642                 decoder->state.trace_nr = buffer.trace_nr;
 643                 decoder->vm_tm_corr_same_buf = false;
 644                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 645                              decoder->ref_timestamp);
 646                 return -ENOLINK;
 647         }
 648
 649         return 0;
 650 }
 651
 652 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
 653                                   bool reposition)
 654 {
 655         if (!decoder->next_buf)
 656                 return intel_pt_get_data(decoder, reposition);
 657
 658         decoder->buf = decoder->next_buf;
 659         decoder->len = decoder->next_len;
 660         decoder->next_buf = 0;
 661         decoder->next_len = 0;
 662         return 0;
 663 }
 664
 665 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 666 {
 667         unsigned char *buf = decoder->temp_buf;
 668         size_t old_len, len, n;
 669         int ret;
 670
 671         old_len = decoder->len;
 672         len = decoder->len;
 673         memcpy(buf, decoder->buf, len);
 674
 675         ret = intel_pt_get_data(decoder, false);
 676         if (ret) {
 677                 decoder->pos += old_len;
 678                 return ret < 0 ? ret : -EINVAL;
 679         }
 680
 681         n = INTEL_PT_PKT_MAX_SZ - len;
 682         if (n > decoder->len)
 683                 n = decoder->len;
 684         memcpy(buf + len, decoder->buf, n);
 685         len += n;
 686
 687         decoder->prev_pkt_ctx = decoder->pkt_ctx;
 688         ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
 689         if (ret < (int)old_len) {
 690                 decoder->next_buf = decoder->buf;
 691                 decoder->next_len = decoder->len;
 692                 decoder->buf = buf;
 693                 decoder->len = old_len;
 694                 return intel_pt_bad_packet(decoder);
 695         }
 696
 697         decoder->next_buf = decoder->buf + (ret - old_len);
 698         decoder->next_len = decoder->len - (ret - old_len);
 699
 700         decoder->buf = buf;
 701         decoder->len = ret;
 702
 703         return ret;
 704 }
 705
 706 struct intel_pt_pkt_info {
 707         struct intel_pt_decoder   *decoder;
 708         struct intel_pt_pkt       packet;
 709         uint64_t                  pos;
 710         int                       pkt_len;
 711         int                       last_packet_type;
 712         void                      *data;
 713 };
 714
 715 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 716
 717 /* Lookahead packets in current buffer */
 718 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 719                                   intel_pt_pkt_cb_t cb, void *data)
 720 {
 721         struct intel_pt_pkt_info pkt_info;
 722         const unsigned char *buf = decoder->buf;
 723         enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
 724         size_t len = decoder->len;
 725         int ret;
 726
 727         pkt_info.decoder          = decoder;
 728         pkt_info.pos              = decoder->pos;
 729         pkt_info.pkt_len          = decoder->pkt_step;
 730         pkt_info.last_packet_type = decoder->last_packet_type;
 731         pkt_info.data             = data;
 732
 733         while (1) {
 734                 do {
 735                         pkt_info.pos += pkt_info.pkt_len;
 736                         buf          += pkt_info.pkt_len;
 737                         len          -= pkt_info.pkt_len;
 738
 739                         if (!len)
 740                                 return INTEL_PT_NEED_MORE_BYTES;
 741
 742                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
 743                                                   &pkt_ctx);
 744                         if (!ret)
 745                                 return INTEL_PT_NEED_MORE_BYTES;
 746                         if (ret < 0)
 747                                 return ret;
 748
 749                         pkt_info.pkt_len = ret;
 750                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 751
 752                 ret = cb(&pkt_info);
 753                 if (ret)
 754                         return 0;
 755
 756                 pkt_info.last_packet_type = pkt_info.packet.type;
 757         }
 758 }
 759
 760 struct intel_pt_calc_cyc_to_tsc_info {
 761         uint64_t        cycle_cnt;
 762         unsigned int    cbr;
 763         uint32_t        last_mtc;
 764         uint64_t        ctc_timestamp;
 765         uint64_t        ctc_delta;
 766         uint64_t        tsc_timestamp;
 767         uint64_t        timestamp;
 768         bool            have_tma;
 769         bool            fixup_last_mtc;
 770         bool            from_mtc;
 771         double          cbr_cyc_to_tsc;
 772 };
 773
 774 /*
 775  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 776  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 777  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 778  * packet by copying the missing bits from the current MTC assuming the least
 779  * difference between the two, and that the current MTC comes after last_mtc.
 780  */
 781 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 782                                     uint32_t *last_mtc)
 783 {
 784         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 785         uint32_t mask = ~(first_missing_bit - 1);
 786
 787         *last_mtc |= mtc & mask;
 788         if (*last_mtc >= mtc) {
 789                 *last_mtc -= first_missing_bit;
 790                 *last_mtc &= 0xff;
 791         }
 792 }
 793
 794 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 795 {
 796         struct intel_pt_decoder *decoder = pkt_info->decoder;
 797         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 798         uint64_t timestamp;
 799         double cyc_to_tsc;
 800         unsigned int cbr;
 801         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 802
 803         switch (pkt_info->packet.type) {
 804         case INTEL_PT_TNT:
 805         case INTEL_PT_TIP_PGE:
 806         case INTEL_PT_TIP:
 807         case INTEL_PT_FUP:
 808         case INTEL_PT_PSB:
 809         case INTEL_PT_PIP:
 810         case INTEL_PT_MODE_EXEC:
 811         case INTEL_PT_MODE_TSX:
 812         case INTEL_PT_PSBEND:
 813         case INTEL_PT_PAD:
 814         case INTEL_PT_VMCS:
 815         case INTEL_PT_MNT:
 816         case INTEL_PT_PTWRITE:
 817         case INTEL_PT_PTWRITE_IP:
 818         case INTEL_PT_BBP:
 819         case INTEL_PT_BIP:
 820         case INTEL_PT_BEP:
 821         case INTEL_PT_BEP_IP:
 822                 return 0;
 823
 824         case INTEL_PT_MTC:
 825                 if (!data->have_tma)
 826                         return 0;
 827
 828                 mtc = pkt_info->packet.payload;
 829                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 830                         data->fixup_last_mtc = false;
 831                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 832                                                 &data->last_mtc);
 833                 }
 834                 if (mtc > data->last_mtc)
 835                         mtc_delta = mtc - data->last_mtc;
 836                 else
 837                         mtc_delta = mtc + 256 - data->last_mtc;
 838                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 839                 data->last_mtc = mtc;
 840
 841                 if (decoder->tsc_ctc_mult) {
 842                         timestamp = data->ctc_timestamp +
 843                                 data->ctc_delta * decoder->tsc_ctc_mult;
 844                 } else {
 845                         timestamp = data->ctc_timestamp +
 846                                 multdiv(data->ctc_delta,
 847                                         decoder->tsc_ctc_ratio_n,
 848                                         decoder->tsc_ctc_ratio_d);
 849                 }
 850
 851                 if (timestamp < data->timestamp)
 852                         return 1;
 853
 854                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 855                         data->timestamp = timestamp;
 856                         return 0;
 857                 }
 858
 859                 break;
 860
 861         case INTEL_PT_TSC:
 862                 /*
 863                  * For now, do not support using TSC packets - refer
 864                  * intel_pt_calc_cyc_to_tsc().
 865                  */
 866                 if (data->from_mtc)
 867                         return 1;
 868                 timestamp = pkt_info->packet.payload |
 869                             (data->timestamp & (0xffULL << 56));
 870                 if (data->from_mtc && timestamp < data->timestamp &&
 871                     data->timestamp - timestamp < decoder->tsc_slip)
 872                         return 1;
 873                 if (timestamp < data->timestamp)
 874                         timestamp += (1ULL << 56);
 875                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 876                         if (data->from_mtc)
 877                                 return 1;
 878                         data->tsc_timestamp = timestamp;
 879                         data->timestamp = timestamp;
 880                         return 0;
 881                 }
 882                 break;
 883
 884         case INTEL_PT_TMA:
 885                 if (data->from_mtc)
 886                         return 1;
 887
 888                 if (!decoder->tsc_ctc_ratio_d)
 889                         return 0;
 890
 891                 ctc = pkt_info->packet.payload;
 892                 fc = pkt_info->packet.count;
 893                 ctc_rem = ctc & decoder->ctc_rem_mask;
 894
 895                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 896
 897                 data->ctc_timestamp = data->tsc_timestamp - fc;
 898                 if (decoder->tsc_ctc_mult) {
 899                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 900                 } else {
 901                         data->ctc_timestamp -=
 902                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 903                                         decoder->tsc_ctc_ratio_d);
 904                 }
 905
 906                 data->ctc_delta = 0;
 907                 data->have_tma = true;
 908                 data->fixup_last_mtc = true;
 909
 910                 return 0;
 911
 912         case INTEL_PT_CYC:
 913                 data->cycle_cnt += pkt_info->packet.payload;
 914                 return 0;
 915
 916         case INTEL_PT_CBR:
 917                 cbr = pkt_info->packet.payload;
 918                 if (data->cbr && data->cbr != cbr)
 919                         return 1;
 920                 data->cbr = cbr;
 921                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 922                 return 0;
 923
 924         case INTEL_PT_TIP_PGD:
 925         case INTEL_PT_TRACESTOP:
 926         case INTEL_PT_EXSTOP:
 927         case INTEL_PT_EXSTOP_IP:
 928         case INTEL_PT_MWAIT:
 929         case INTEL_PT_PWRE:
 930         case INTEL_PT_PWRX:
 931         case INTEL_PT_OVF:
 932         case INTEL_PT_BAD: /* Does not happen */
 933         default:
 934                 return 1;
 935         }
 936
 937         if (!data->cbr && decoder->cbr) {
 938                 data->cbr = decoder->cbr;
 939                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 940         }
 941
 942         if (!data->cycle_cnt)
 943                 return 1;
 944
 945         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 946
 947         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 948             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 949                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 950                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 951                 return 1;
 952         }
 953
 954         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 955         decoder->have_calc_cyc_to_tsc = true;
 956
 957         if (data->cbr) {
 958                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 959                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 960         } else {
 961                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 962                              cyc_to_tsc, pkt_info->pos);
 963         }
 964
 965         return 1;
 966 }
 967
 968 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 969                                      bool from_mtc)
 970 {
 971         struct intel_pt_calc_cyc_to_tsc_info data = {
 972                 .cycle_cnt      = 0,
 973                 .cbr            = 0,
 974                 .last_mtc       = decoder->last_mtc,
 975                 .ctc_timestamp  = decoder->ctc_timestamp,
 976                 .ctc_delta      = decoder->ctc_delta,
 977                 .tsc_timestamp  = decoder->tsc_timestamp,
 978                 .timestamp      = decoder->timestamp,
 979                 .have_tma       = decoder->have_tma,
 980                 .fixup_last_mtc = decoder->fixup_last_mtc,
 981                 .from_mtc       = from_mtc,
 982                 .cbr_cyc_to_tsc = 0,
 983         };
 984
 985         /*
 986          * For now, do not support using TSC packets for at least the reasons:
 987          * 1) timing might have stopped
 988          * 2) TSC packets within PSB+ can slip against CYC packets
 989          */
 990         if (!from_mtc)
 991                 return;
 992
 993         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 994 }
 995
 996 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 997 {
 998         int ret;
 999
1000         decoder->last_packet_type = decoder->packet.type;
1001
1002         do {
1003                 decoder->pos += decoder->pkt_step;
1004                 decoder->buf += decoder->pkt_step;
1005                 decoder->len -= decoder->pkt_step;
1006
1007                 if (!decoder->len) {
1008                         ret = intel_pt_get_next_data(decoder, false);
1009                         if (ret)
1010                                 return ret;
1011                 }
1012
1013                 decoder->prev_pkt_ctx = decoder->pkt_ctx;
1014                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
1015                                           &decoder->packet, &decoder->pkt_ctx);
1016                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1017                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1018                         ret = intel_pt_get_split_packet(decoder);
1019                         if (ret < 0)
1020                                 return ret;
1021                 }
1022                 if (ret <= 0)
1023                         return intel_pt_bad_packet(decoder);
1024
1025                 decoder->pkt_len = ret;
1026                 decoder->pkt_step = ret;
1027                 intel_pt_decoder_log_packet(decoder);
1028         } while (decoder->packet.type == INTEL_PT_PAD);
1029
1030         return 0;
1031 }
1032
1033 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1034 {
1035         uint64_t timestamp, masked_timestamp;
1036
1037         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1038         masked_timestamp = timestamp & decoder->period_mask;
1039         if (decoder->continuous_period) {
1040                 if (masked_timestamp > decoder->last_masked_timestamp)
1041                         return 1;
1042         } else {
1043                 timestamp += 1;
1044                 masked_timestamp = timestamp & decoder->period_mask;
1045                 if (masked_timestamp > decoder->last_masked_timestamp) {
1046                         decoder->last_masked_timestamp = masked_timestamp;
1047                         decoder->continuous_period = true;
1048                 }
1049         }
1050
1051         if (masked_timestamp < decoder->last_masked_timestamp)
1052                 return decoder->period_ticks;
1053
1054         return decoder->period_ticks - (timestamp - masked_timestamp);
1055 }
1056
1057 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1058 {
1059         switch (decoder->period_type) {
1060         case INTEL_PT_PERIOD_INSTRUCTIONS:
1061                 return decoder->period - decoder->period_insn_cnt;
1062         case INTEL_PT_PERIOD_TICKS:
1063                 return intel_pt_next_period(decoder);
1064         case INTEL_PT_PERIOD_NONE:
1065         case INTEL_PT_PERIOD_MTC:
1066         default:
1067                 return 0;
1068         }
1069 }
1070
1071 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1072 {
1073         uint64_t timestamp, masked_timestamp;
1074
1075         switch (decoder->period_type) {
1076         case INTEL_PT_PERIOD_INSTRUCTIONS:
1077                 decoder->period_insn_cnt = 0;
1078                 break;
1079         case INTEL_PT_PERIOD_TICKS:
1080                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1081                 masked_timestamp = timestamp & decoder->period_mask;
1082                 if (masked_timestamp > decoder->last_masked_timestamp)
1083                         decoder->last_masked_timestamp = masked_timestamp;
1084                 else
1085                         decoder->last_masked_timestamp += decoder->period_ticks;
1086                 break;
1087         case INTEL_PT_PERIOD_NONE:
1088         case INTEL_PT_PERIOD_MTC:
1089         default:
1090                 break;
1091         }
1092
1093         decoder->state.type |= INTEL_PT_INSTRUCTION;
1094 }
1095
1096 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1097                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1098 {
1099         uint64_t max_insn_cnt, insn_cnt = 0;
1100         int err;
1101
1102         if (!decoder->mtc_insn)
1103                 decoder->mtc_insn = true;
1104
1105         max_insn_cnt = intel_pt_next_sample(decoder);
1106
1107         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1108                                  max_insn_cnt, decoder->data);
1109
1110         decoder->tot_insn_cnt += insn_cnt;
1111         decoder->timestamp_insn_cnt += insn_cnt;
1112         decoder->sample_insn_cnt += insn_cnt;
1113         decoder->period_insn_cnt += insn_cnt;
1114
1115         if (err) {
1116                 decoder->no_progress = 0;
1117                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
1118                 intel_pt_log_at("ERROR: Failed to get instruction",
1119                                 decoder->ip);
1120                 if (err == -ENOENT)
1121                         return -ENOLINK;
1122                 return -EILSEQ;
1123         }
1124
1125         if (ip && decoder->ip == ip) {
1126                 err = -EAGAIN;
1127                 goto out;
1128         }
1129
1130         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1131                 intel_pt_sample_insn(decoder);
1132
1133         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1134                 decoder->state.type = INTEL_PT_INSTRUCTION;
1135                 decoder->state.from_ip = decoder->ip;
1136                 decoder->state.to_ip = 0;
1137                 decoder->ip += intel_pt_insn->length;
1138                 err = INTEL_PT_RETURN;
1139                 goto out;
1140         }
1141
1142         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1143                 /* Zero-length calls are excluded */
1144                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1145                     intel_pt_insn->rel) {
1146                         err = intel_pt_push(&decoder->stack, decoder->ip +
1147                                             intel_pt_insn->length);
1148                         if (err)
1149                                 goto out;
1150                 }
1151         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1152                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1153         }
1154
1155         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1156                 int cnt = decoder->no_progress++;
1157
1158                 decoder->state.from_ip = decoder->ip;
1159                 decoder->ip += intel_pt_insn->length +
1160                                 intel_pt_insn->rel;
1161                 decoder->state.to_ip = decoder->ip;
1162                 err = INTEL_PT_RETURN;
1163
1164                 /*
1165                  * Check for being stuck in a loop.  This can happen if a
1166                  * decoder error results in the decoder erroneously setting the
1167                  * ip to an address that is itself in an infinite loop that
1168                  * consumes no packets.  When that happens, there must be an
1169                  * unconditional branch.
1170                  */
1171                 if (cnt) {
1172                         if (cnt == 1) {
1173                                 decoder->stuck_ip = decoder->state.to_ip;
1174                                 decoder->stuck_ip_prd = 1;
1175                                 decoder->stuck_ip_cnt = 1;
1176                         } else if (cnt > decoder->max_loops ||
1177                                    decoder->state.to_ip == decoder->stuck_ip) {
1178                                 intel_pt_log_at("ERROR: Never-ending loop",
1179                                                 decoder->state.to_ip);
1180                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1181                                 err = -ELOOP;
1182                                 goto out;
1183                         } else if (!--decoder->stuck_ip_cnt) {
1184                                 decoder->stuck_ip_prd += 1;
1185                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1186                                 decoder->stuck_ip = decoder->state.to_ip;
1187                         }
1188                 }
1189                 goto out_no_progress;
1190         }
1191 out:
1192         decoder->no_progress = 0;
1193 out_no_progress:
1194         decoder->state.insn_op = intel_pt_insn->op;
1195         decoder->state.insn_len = intel_pt_insn->length;
1196         memcpy(decoder->state.insn, intel_pt_insn->buf,
1197                INTEL_PT_INSN_BUF_SZ);
1198
1199         if (decoder->tx_flags & INTEL_PT_IN_TX)
1200                 decoder->state.flags |= INTEL_PT_IN_TX;
1201
1202         return err;
1203 }
1204
1205 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1206 {
1207         bool ret = false;
1208
1209         if (decoder->set_fup_tx_flags) {
1210                 decoder->set_fup_tx_flags = false;
1211                 decoder->tx_flags = decoder->fup_tx_flags;
1212                 decoder->state.type = INTEL_PT_TRANSACTION;
1213                 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1214                         decoder->state.type |= INTEL_PT_BRANCH;
1215                 decoder->state.from_ip = decoder->ip;
1216                 decoder->state.to_ip = 0;
1217                 decoder->state.flags = decoder->fup_tx_flags;
1218                 return true;
1219         }
1220         if (decoder->set_fup_ptw) {
1221                 decoder->set_fup_ptw = false;
1222                 decoder->state.type = INTEL_PT_PTW;
1223                 decoder->state.flags |= INTEL_PT_FUP_IP;
1224                 decoder->state.from_ip = decoder->ip;
1225                 decoder->state.to_ip = 0;
1226                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1227                 return true;
1228         }
1229         if (decoder->set_fup_mwait) {
1230                 decoder->set_fup_mwait = false;
1231                 decoder->state.type = INTEL_PT_MWAIT_OP;
1232                 decoder->state.from_ip = decoder->ip;
1233                 decoder->state.to_ip = 0;
1234                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1235                 ret = true;
1236         }
1237         if (decoder->set_fup_pwre) {
1238                 decoder->set_fup_pwre = false;
1239                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1240                 decoder->state.type &= ~INTEL_PT_BRANCH;
1241                 decoder->state.from_ip = decoder->ip;
1242                 decoder->state.to_ip = 0;
1243                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1244                 ret = true;
1245         }
1246         if (decoder->set_fup_exstop) {
1247                 decoder->set_fup_exstop = false;
1248                 decoder->state.type |= INTEL_PT_EX_STOP;
1249                 decoder->state.type &= ~INTEL_PT_BRANCH;
1250                 decoder->state.flags |= INTEL_PT_FUP_IP;
1251                 decoder->state.from_ip = decoder->ip;
1252                 decoder->state.to_ip = 0;
1253                 ret = true;
1254         }
1255         if (decoder->set_fup_bep) {
1256                 decoder->set_fup_bep = false;
1257                 decoder->state.type |= INTEL_PT_BLK_ITEMS;
1258                 decoder->state.type &= ~INTEL_PT_BRANCH;
1259                 decoder->state.from_ip = decoder->ip;
1260                 decoder->state.to_ip = 0;
1261                 ret = true;
1262         }
1263         return ret;
1264 }
1265
1266 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1267                                           struct intel_pt_insn *intel_pt_insn,
1268                                           uint64_t ip, int err)
1269 {
1270         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1271                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1272                ip == decoder->ip + intel_pt_insn->length;
1273 }
1274
1275 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1276 {
1277         struct intel_pt_insn intel_pt_insn;
1278         uint64_t ip;
1279         int err;
1280
1281         ip = decoder->last_ip;
1282
1283         while (1) {
1284                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1285                 if (err == INTEL_PT_RETURN)
1286                         return 0;
1287                 if (err == -EAGAIN ||
1288                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1289                         bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1290
1291                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1292                         if (intel_pt_fup_event(decoder) && no_tip)
1293                                 return 0;
1294                         return -EAGAIN;
1295                 }
1296                 decoder->set_fup_tx_flags = false;
1297                 if (err)
1298                         return err;
1299
1300                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1301                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1302                                         decoder->ip);
1303                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1304                         return -ENOENT;
1305                 }
1306
1307                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1308                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1309                                         decoder->ip);
1310                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1311                         return -ENOENT;
1312                 }
1313
1314                 intel_pt_bug(decoder);
1315         }
1316 }
1317
1318 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1319 {
1320         struct intel_pt_insn intel_pt_insn;
1321         int err;
1322
1323         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1324         if (err == INTEL_PT_RETURN &&
1325             decoder->pgd_ip &&
1326             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1327             (decoder->state.type & INTEL_PT_BRANCH) &&
1328             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1329                 /* Unconditional branch leaving filter region */
1330                 decoder->no_progress = 0;
1331                 decoder->pge = false;
1332                 decoder->continuous_period = false;
1333                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1334                 decoder->state.type |= INTEL_PT_TRACE_END;
1335                 intel_pt_update_nr(decoder);
1336                 return 0;
1337         }
1338         if (err == INTEL_PT_RETURN)
1339                 return 0;
1340         if (err)
1341                 return err;
1342
1343         intel_pt_update_nr(decoder);
1344
1345         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1346                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1347                         decoder->pge = false;
1348                         decoder->continuous_period = false;
1349                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1350                         decoder->state.from_ip = decoder->ip;
1351                         if (decoder->packet.count == 0) {
1352                                 decoder->state.to_ip = 0;
1353                         } else {
1354                                 decoder->state.to_ip = decoder->last_ip;
1355                                 decoder->ip = decoder->last_ip;
1356                         }
1357                         decoder->state.type |= INTEL_PT_TRACE_END;
1358                 } else {
1359                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1360                         decoder->state.from_ip = decoder->ip;
1361                         if (decoder->packet.count == 0) {
1362                                 decoder->state.to_ip = 0;
1363                         } else {
1364                                 decoder->state.to_ip = decoder->last_ip;
1365                                 decoder->ip = decoder->last_ip;
1366                         }
1367                 }
1368                 return 0;
1369         }
1370
1371         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1372                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1373                                  intel_pt_insn.rel;
1374
1375                 if (decoder->pgd_ip &&
1376                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1377                     decoder->pgd_ip(to_ip, decoder->data)) {
1378                         /* Conditional branch leaving filter region */
1379                         decoder->pge = false;
1380                         decoder->continuous_period = false;
1381                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1382                         decoder->ip = to_ip;
1383                         decoder->state.from_ip = decoder->ip;
1384                         decoder->state.to_ip = to_ip;
1385                         decoder->state.type |= INTEL_PT_TRACE_END;
1386                         return 0;
1387                 }
1388                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1389                                 decoder->ip);
1390                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1391                 return -ENOENT;
1392         }
1393
1394         return intel_pt_bug(decoder);
1395 }
1396
1397 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1398 {
1399         struct intel_pt_insn intel_pt_insn;
1400         int err;
1401
1402         while (1) {
1403                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1404                 if (err == INTEL_PT_RETURN)
1405                         return 0;
1406                 if (err)
1407                         return err;
1408
1409                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1410                         if (!decoder->return_compression) {
1411                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1412                                                 decoder->ip);
1413                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1414                                 return -ENOENT;
1415                         }
1416                         if (!decoder->ret_addr) {
1417                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1418                                                 decoder->ip);
1419                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1420                                 return -ENOENT;
1421                         }
1422                         if (!(decoder->tnt.payload & BIT63)) {
1423                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1424                                                 decoder->ip);
1425                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1426                                 return -ENOENT;
1427                         }
1428                         decoder->tnt.count -= 1;
1429                         if (decoder->tnt.count)
1430                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1431                         else
1432                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1433                         decoder->tnt.payload <<= 1;
1434                         decoder->state.from_ip = decoder->ip;
1435                         decoder->ip = decoder->ret_addr;
1436                         decoder->state.to_ip = decoder->ip;
1437                         return 0;
1438                 }
1439
1440                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1441                         /* Handle deferred TIPs */
1442                         err = intel_pt_get_next_packet(decoder);
1443                         if (err)
1444                                 return err;
1445                         if (decoder->packet.type != INTEL_PT_TIP ||
1446                             decoder->packet.count == 0) {
1447                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1448                                                 decoder->ip);
1449                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1450                                 decoder->pkt_step = 0;
1451                                 return -ENOENT;
1452                         }
1453                         intel_pt_set_last_ip(decoder);
1454                         decoder->state.from_ip = decoder->ip;
1455                         decoder->state.to_ip = decoder->last_ip;
1456                         decoder->ip = decoder->last_ip;
1457                         intel_pt_update_nr(decoder);
1458                         return 0;
1459                 }
1460
1461                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1462                         decoder->tnt.count -= 1;
1463                         if (decoder->tnt.count)
1464                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1465                         else
1466                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1467                         if (decoder->tnt.payload & BIT63) {
1468                                 decoder->tnt.payload <<= 1;
1469                                 decoder->state.from_ip = decoder->ip;
1470                                 decoder->ip += intel_pt_insn.length +
1471                                                intel_pt_insn.rel;
1472                                 decoder->state.to_ip = decoder->ip;
1473                                 return 0;
1474                         }
1475                         /* Instruction sample for a non-taken branch */
1476                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1477                                 decoder->tnt.payload <<= 1;
1478                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1479                                 decoder->state.from_ip = decoder->ip;
1480                                 decoder->state.to_ip = 0;
1481                                 decoder->ip += intel_pt_insn.length;
1482                                 return 0;
1483                         }
1484                         decoder->sample_cyc = false;
1485                         decoder->ip += intel_pt_insn.length;
1486                         if (!decoder->tnt.count) {
1487                                 intel_pt_update_sample_time(decoder);
1488                                 return -EAGAIN;
1489                         }
1490                         decoder->tnt.payload <<= 1;
1491                         continue;
1492                 }
1493
1494                 return intel_pt_bug(decoder);
1495         }
1496 }
1497
1498 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1499 {
1500         unsigned int fup_tx_flags;
1501         int err;
1502
1503         fup_tx_flags = decoder->packet.payload &
1504                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1505         err = intel_pt_get_next_packet(decoder);
1506         if (err)
1507                 return err;
1508         if (decoder->packet.type == INTEL_PT_FUP) {
1509                 decoder->fup_tx_flags = fup_tx_flags;
1510                 decoder->set_fup_tx_flags = true;
1511                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1512                         *no_tip = true;
1513         } else {
1514                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1515                                 decoder->pos);
1516                 intel_pt_update_in_tx(decoder);
1517         }
1518         return 0;
1519 }
1520
1521 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1522 {
1523         timestamp |= (ref_timestamp & (0xffULL << 56));
1524
1525         if (timestamp < ref_timestamp) {
1526                 if (ref_timestamp - timestamp > (1ULL << 55))
1527                         timestamp += (1ULL << 56);
1528         } else {
1529                 if (timestamp - ref_timestamp > (1ULL << 55))
1530                         timestamp -= (1ULL << 56);
1531         }
1532
1533         return timestamp;
1534 }
1535
1536 /* For use only when decoder->vm_time_correlation is true */
1537 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1538                                    uint64_t timestamp)
1539 {
1540         uint64_t max_timestamp = decoder->buf_timestamp;
1541
1542         if (!max_timestamp) {
1543                 max_timestamp = decoder->last_reliable_timestamp +
1544                                 0x400000000ULL;
1545         }
1546         return timestamp >= decoder->last_reliable_timestamp &&
1547                timestamp < decoder->buf_timestamp;
1548 }
1549
1550 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1551 {
1552         uint64_t timestamp;
1553         bool bad = false;
1554
1555         decoder->have_tma = false;
1556
1557         if (decoder->ref_timestamp) {
1558                 timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1559                                             decoder->ref_timestamp);
1560                 decoder->tsc_timestamp = timestamp;
1561                 decoder->timestamp = timestamp;
1562                 decoder->ref_timestamp = 0;
1563                 decoder->timestamp_insn_cnt = 0;
1564         } else if (decoder->timestamp) {
1565                 timestamp = decoder->packet.payload |
1566                             (decoder->timestamp & (0xffULL << 56));
1567                 decoder->tsc_timestamp = timestamp;
1568                 if (timestamp < decoder->timestamp &&
1569                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1570                         intel_pt_log_to("Suppressing backwards timestamp",
1571                                         timestamp);
1572                         timestamp = decoder->timestamp;
1573                 }
1574                 if (timestamp < decoder->timestamp) {
1575                         if (!decoder->buf_timestamp ||
1576                             (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1577                                 intel_pt_log_to("Wraparound timestamp", timestamp);
1578                                 timestamp += (1ULL << 56);
1579                                 decoder->tsc_timestamp = timestamp;
1580                         } else {
1581                                 intel_pt_log_to("Suppressing bad timestamp", timestamp);
1582                                 timestamp = decoder->timestamp;
1583                                 bad = true;
1584                         }
1585                 }
1586                 if (decoder->vm_time_correlation &&
1587                     (bad || !intel_pt_time_in_range(decoder, timestamp)) &&
1588                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1589                         p_log("Timestamp out of range");
1590                 decoder->timestamp = timestamp;
1591                 decoder->timestamp_insn_cnt = 0;
1592         }
1593
1594         if (decoder->last_packet_type == INTEL_PT_CYC) {
1595                 decoder->cyc_ref_timestamp = decoder->timestamp;
1596                 decoder->cycle_cnt = 0;
1597                 decoder->have_calc_cyc_to_tsc = false;
1598                 intel_pt_calc_cyc_to_tsc(decoder, false);
1599         }
1600
1601         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1602 }
1603
1604 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1605 {
1606         intel_pt_log("ERROR: Buffer overflow\n");
1607         intel_pt_clear_tx_flags(decoder);
1608         intel_pt_set_nr(decoder);
1609         decoder->timestamp_insn_cnt = 0;
1610         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1611         decoder->overflow = true;
1612         return -EOVERFLOW;
1613 }
1614
1615 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1616 {
1617         if (decoder->have_cyc)
1618                 return;
1619
1620         decoder->cyc_cnt_timestamp = decoder->timestamp;
1621         decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1622 }
1623
1624 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1625 {
1626         decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1627
1628         if (decoder->pge)
1629                 intel_pt_mtc_cyc_cnt_pge(decoder);
1630 }
1631
1632 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1633 {
1634         uint64_t tot_cyc_cnt, tsc_delta;
1635
1636         if (decoder->have_cyc)
1637                 return;
1638
1639         decoder->sample_cyc = true;
1640
1641         if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1642                 return;
1643
1644         tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1645         tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1646
1647         if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1648                 decoder->tot_cyc_cnt = tot_cyc_cnt;
1649 }
1650
1651 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1652 {
1653         uint32_t ctc = decoder->packet.payload;
1654         uint32_t fc = decoder->packet.count;
1655         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1656
1657         if (!decoder->tsc_ctc_ratio_d)
1658                 return;
1659
1660         if (decoder->pge && !decoder->in_psb)
1661                 intel_pt_mtc_cyc_cnt_pge(decoder);
1662         else
1663                 intel_pt_mtc_cyc_cnt_upd(decoder);
1664
1665         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1666         decoder->last_ctc = ctc - ctc_rem;
1667         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1668         if (decoder->tsc_ctc_mult) {
1669                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1670         } else {
1671                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1672                                                   decoder->tsc_ctc_ratio_n,
1673                                                   decoder->tsc_ctc_ratio_d);
1674         }
1675         decoder->ctc_delta = 0;
1676         decoder->have_tma = true;
1677         decoder->fixup_last_mtc = true;
1678         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1679                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1680 }
1681
1682 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1683 {
1684         uint64_t timestamp;
1685         uint32_t mtc, mtc_delta;
1686
1687         if (!decoder->have_tma)
1688                 return;
1689
1690         mtc = decoder->packet.payload;
1691
1692         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1693                 decoder->fixup_last_mtc = false;
1694                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1695                                         &decoder->last_mtc);
1696         }
1697
1698         if (mtc > decoder->last_mtc)
1699                 mtc_delta = mtc - decoder->last_mtc;
1700         else
1701                 mtc_delta = mtc + 256 - decoder->last_mtc;
1702
1703         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1704
1705         if (decoder->tsc_ctc_mult) {
1706                 timestamp = decoder->ctc_timestamp +
1707                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1708         } else {
1709                 timestamp = decoder->ctc_timestamp +
1710                             multdiv(decoder->ctc_delta,
1711                                     decoder->tsc_ctc_ratio_n,
1712                                     decoder->tsc_ctc_ratio_d);
1713         }
1714
1715         if (timestamp < decoder->timestamp)
1716                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1717                              timestamp, decoder->timestamp);
1718         else
1719                 decoder->timestamp = timestamp;
1720
1721         intel_pt_mtc_cyc_cnt_upd(decoder);
1722
1723         decoder->timestamp_insn_cnt = 0;
1724         decoder->last_mtc = mtc;
1725
1726         if (decoder->last_packet_type == INTEL_PT_CYC) {
1727                 decoder->cyc_ref_timestamp = decoder->timestamp;
1728                 decoder->cycle_cnt = 0;
1729                 decoder->have_calc_cyc_to_tsc = false;
1730                 intel_pt_calc_cyc_to_tsc(decoder, true);
1731         }
1732
1733         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1734 }
1735
1736 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1737 {
1738         unsigned int cbr = decoder->packet.payload & 0xff;
1739
1740         decoder->cbr_payload = decoder->packet.payload;
1741
1742         if (decoder->cbr == cbr)
1743                 return;
1744
1745         decoder->cbr = cbr;
1746         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1747
1748         intel_pt_mtc_cyc_cnt_cbr(decoder);
1749 }
1750
1751 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1752 {
1753         uint64_t timestamp = decoder->cyc_ref_timestamp;
1754
1755         decoder->have_cyc = true;
1756
1757         decoder->cycle_cnt += decoder->packet.payload;
1758         if (decoder->pge)
1759                 decoder->tot_cyc_cnt += decoder->packet.payload;
1760         decoder->sample_cyc = true;
1761
1762         if (!decoder->cyc_ref_timestamp)
1763                 return;
1764
1765         if (decoder->have_calc_cyc_to_tsc)
1766                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1767         else if (decoder->cbr)
1768                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1769         else
1770                 return;
1771
1772         if (timestamp < decoder->timestamp)
1773                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1774                              timestamp, decoder->timestamp);
1775         else
1776                 decoder->timestamp = timestamp;
1777
1778         decoder->timestamp_insn_cnt = 0;
1779
1780         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1781 }
1782
1783 static void intel_pt_bbp(struct intel_pt_decoder *decoder)
1784 {
1785         if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
1786                 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
1787                 decoder->state.items.is_32_bit = false;
1788         }
1789         decoder->blk_type = decoder->packet.payload;
1790         decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
1791         if (decoder->blk_type == INTEL_PT_GP_REGS)
1792                 decoder->state.items.is_32_bit = decoder->packet.count;
1793         if (decoder->blk_type_pos < 0) {
1794                 intel_pt_log("WARNING: Unknown block type %u\n",
1795                              decoder->blk_type);
1796         } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
1797                 intel_pt_log("WARNING: Duplicate block type %u\n",
1798                              decoder->blk_type);
1799         }
1800 }
1801
1802 static void intel_pt_bip(struct intel_pt_decoder *decoder)
1803 {
1804         uint32_t id = decoder->packet.count;
1805         uint32_t bit = 1 << id;
1806         int pos = decoder->blk_type_pos;
1807
1808         if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
1809                 intel_pt_log("WARNING: Unknown block item %u type %d\n",
1810                              id, decoder->blk_type);
1811                 return;
1812         }
1813
1814         if (decoder->state.items.mask[pos] & bit) {
1815                 intel_pt_log("WARNING: Duplicate block item %u type %d\n",
1816                              id, decoder->blk_type);
1817         }
1818
1819         decoder->state.items.mask[pos] |= bit;
1820         decoder->state.items.val[pos][id] = decoder->packet.payload;
1821 }
1822
1823 /* Walk PSB+ packets when already in sync. */
1824 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1825 {
1826         int err;
1827
1828         decoder->in_psb = true;
1829
1830         while (1) {
1831                 err = intel_pt_get_next_packet(decoder);
1832                 if (err)
1833                         goto out;
1834
1835                 switch (decoder->packet.type) {
1836                 case INTEL_PT_PSBEND:
1837                         err = 0;
1838                         goto out;
1839
1840                 case INTEL_PT_TIP_PGD:
1841                 case INTEL_PT_TIP_PGE:
1842                 case INTEL_PT_TIP:
1843                 case INTEL_PT_TNT:
1844                 case INTEL_PT_TRACESTOP:
1845                 case INTEL_PT_BAD:
1846                 case INTEL_PT_PSB:
1847                 case INTEL_PT_PTWRITE:
1848                 case INTEL_PT_PTWRITE_IP:
1849                 case INTEL_PT_EXSTOP:
1850                 case INTEL_PT_EXSTOP_IP:
1851                 case INTEL_PT_MWAIT:
1852                 case INTEL_PT_PWRE:
1853                 case INTEL_PT_PWRX:
1854                 case INTEL_PT_BBP:
1855                 case INTEL_PT_BIP:
1856                 case INTEL_PT_BEP:
1857                 case INTEL_PT_BEP_IP:
1858                         decoder->have_tma = false;
1859                         intel_pt_log("ERROR: Unexpected packet\n");
1860                         err = -EAGAIN;
1861                         goto out;
1862
1863                 case INTEL_PT_OVF:
1864                         err = intel_pt_overflow(decoder);
1865                         goto out;
1866
1867                 case INTEL_PT_TSC:
1868                         intel_pt_calc_tsc_timestamp(decoder);
1869                         break;
1870
1871                 case INTEL_PT_TMA:
1872                         intel_pt_calc_tma(decoder);
1873                         break;
1874
1875                 case INTEL_PT_CBR:
1876                         intel_pt_calc_cbr(decoder);
1877                         break;
1878
1879                 case INTEL_PT_MODE_EXEC:
1880                         decoder->exec_mode = decoder->packet.payload;
1881                         break;
1882
1883                 case INTEL_PT_PIP:
1884                         intel_pt_set_pip(decoder);
1885                         break;
1886
1887                 case INTEL_PT_FUP:
1888                         decoder->pge = true;
1889                         if (decoder->packet.count) {
1890                                 intel_pt_set_last_ip(decoder);
1891                                 decoder->psb_ip = decoder->last_ip;
1892                         }
1893                         break;
1894
1895                 case INTEL_PT_MODE_TSX:
1896                         intel_pt_update_in_tx(decoder);
1897                         break;
1898
1899                 case INTEL_PT_MTC:
1900                         intel_pt_calc_mtc_timestamp(decoder);
1901                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1902                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1903                         break;
1904
1905                 case INTEL_PT_CYC:
1906                         intel_pt_calc_cyc_timestamp(decoder);
1907                         break;
1908
1909                 case INTEL_PT_VMCS:
1910                 case INTEL_PT_MNT:
1911                 case INTEL_PT_PAD:
1912                 default:
1913                         break;
1914                 }
1915         }
1916 out:
1917         decoder->in_psb = false;
1918
1919         return err;
1920 }
1921
1922 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1923 {
1924         int err;
1925
1926         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1927                 decoder->tx_flags = 0;
1928                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1929                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1930         } else {
1931                 decoder->state.flags |= INTEL_PT_ASYNC;
1932         }
1933
1934         while (1) {
1935                 err = intel_pt_get_next_packet(decoder);
1936                 if (err)
1937                         return err;
1938
1939                 switch (decoder->packet.type) {
1940                 case INTEL_PT_TNT:
1941                 case INTEL_PT_FUP:
1942                 case INTEL_PT_TRACESTOP:
1943                 case INTEL_PT_PSB:
1944                 case INTEL_PT_TSC:
1945                 case INTEL_PT_TMA:
1946                 case INTEL_PT_MODE_TSX:
1947                 case INTEL_PT_BAD:
1948                 case INTEL_PT_PSBEND:
1949                 case INTEL_PT_PTWRITE:
1950                 case INTEL_PT_PTWRITE_IP:
1951                 case INTEL_PT_EXSTOP:
1952                 case INTEL_PT_EXSTOP_IP:
1953                 case INTEL_PT_MWAIT:
1954                 case INTEL_PT_PWRE:
1955                 case INTEL_PT_PWRX:
1956                 case INTEL_PT_BBP:
1957                 case INTEL_PT_BIP:
1958                 case INTEL_PT_BEP:
1959                 case INTEL_PT_BEP_IP:
1960                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1961                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1962                         decoder->pkt_step = 0;
1963                         return -ENOENT;
1964
1965                 case INTEL_PT_CBR:
1966                         intel_pt_calc_cbr(decoder);
1967                         break;
1968
1969                 case INTEL_PT_OVF:
1970                         return intel_pt_overflow(decoder);
1971
1972                 case INTEL_PT_TIP_PGD:
1973                         decoder->state.from_ip = decoder->ip;
1974                         if (decoder->packet.count == 0) {
1975                                 decoder->state.to_ip = 0;
1976                         } else {
1977                                 intel_pt_set_ip(decoder);
1978                                 decoder->state.to_ip = decoder->ip;
1979                         }
1980                         decoder->pge = false;
1981                         decoder->continuous_period = false;
1982                         decoder->state.type |= INTEL_PT_TRACE_END;
1983                         intel_pt_update_nr(decoder);
1984                         return 0;
1985
1986                 case INTEL_PT_TIP_PGE:
1987                         decoder->pge = true;
1988                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1989                                      decoder->ip);
1990                         decoder->state.from_ip = 0;
1991                         if (decoder->packet.count == 0) {
1992                                 decoder->state.to_ip = 0;
1993                         } else {
1994                                 intel_pt_set_ip(decoder);
1995                                 decoder->state.to_ip = decoder->ip;
1996                         }
1997                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1998                         intel_pt_mtc_cyc_cnt_pge(decoder);
1999                         intel_pt_set_nr(decoder);
2000                         return 0;
2001
2002                 case INTEL_PT_TIP:
2003                         decoder->state.from_ip = decoder->ip;
2004                         if (decoder->packet.count == 0) {
2005                                 decoder->state.to_ip = 0;
2006                         } else {
2007                                 intel_pt_set_ip(decoder);
2008                                 decoder->state.to_ip = decoder->ip;
2009                         }
2010                         intel_pt_update_nr(decoder);
2011                         return 0;
2012
2013                 case INTEL_PT_PIP:
2014                         intel_pt_update_pip(decoder);
2015                         break;
2016
2017                 case INTEL_PT_MTC:
2018                         intel_pt_calc_mtc_timestamp(decoder);
2019                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2020                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2021                         break;
2022
2023                 case INTEL_PT_CYC:
2024                         intel_pt_calc_cyc_timestamp(decoder);
2025                         break;
2026
2027                 case INTEL_PT_MODE_EXEC:
2028                         decoder->exec_mode = decoder->packet.payload;
2029                         break;
2030
2031                 case INTEL_PT_VMCS:
2032                 case INTEL_PT_MNT:
2033                 case INTEL_PT_PAD:
2034                         break;
2035
2036                 default:
2037                         return intel_pt_bug(decoder);
2038                 }
2039         }
2040 }
2041
2042 static int intel_pt_resample(struct intel_pt_decoder *decoder)
2043 {
2044         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2045         decoder->state.type = INTEL_PT_INSTRUCTION;
2046         decoder->state.from_ip = decoder->ip;
2047         decoder->state.to_ip = 0;
2048         return 0;
2049 }
2050
2051 struct intel_pt_vm_tsc_info {
2052         struct intel_pt_pkt pip_packet;
2053         struct intel_pt_pkt vmcs_packet;
2054         struct intel_pt_pkt tma_packet;
2055         bool tsc, pip, vmcs, tma, psbend;
2056         uint64_t ctc_delta;
2057         uint64_t last_ctc;
2058         int max_lookahead;
2059 };
2060
2061 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2062 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2063 {
2064         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2065
2066         switch (pkt_info->packet.type) {
2067         case INTEL_PT_PAD:
2068         case INTEL_PT_MNT:
2069         case INTEL_PT_MODE_EXEC:
2070         case INTEL_PT_MODE_TSX:
2071         case INTEL_PT_MTC:
2072         case INTEL_PT_FUP:
2073         case INTEL_PT_CYC:
2074         case INTEL_PT_CBR:
2075                 break;
2076
2077         case INTEL_PT_TSC:
2078                 data->tsc = true;
2079                 break;
2080
2081         case INTEL_PT_TMA:
2082                 data->tma_packet = pkt_info->packet;
2083                 data->tma = true;
2084                 break;
2085
2086         case INTEL_PT_PIP:
2087                 data->pip_packet = pkt_info->packet;
2088                 data->pip = true;
2089                 break;
2090
2091         case INTEL_PT_VMCS:
2092                 data->vmcs_packet = pkt_info->packet;
2093                 data->vmcs = true;
2094                 break;
2095
2096         case INTEL_PT_PSBEND:
2097                 data->psbend = true;
2098                 return 1;
2099
2100         case INTEL_PT_TIP_PGE:
2101         case INTEL_PT_PTWRITE:
2102         case INTEL_PT_PTWRITE_IP:
2103         case INTEL_PT_EXSTOP:
2104         case INTEL_PT_EXSTOP_IP:
2105         case INTEL_PT_MWAIT:
2106         case INTEL_PT_PWRE:
2107         case INTEL_PT_PWRX:
2108         case INTEL_PT_BBP:
2109         case INTEL_PT_BIP:
2110         case INTEL_PT_BEP:
2111         case INTEL_PT_BEP_IP:
2112         case INTEL_PT_OVF:
2113         case INTEL_PT_BAD:
2114         case INTEL_PT_TNT:
2115         case INTEL_PT_TIP_PGD:
2116         case INTEL_PT_TIP:
2117         case INTEL_PT_PSB:
2118         case INTEL_PT_TRACESTOP:
2119         default:
2120                 return 1;
2121         }
2122
2123         return 0;
2124 }
2125
2126 struct intel_pt_ovf_fup_info {
2127         int max_lookahead;
2128         bool found;
2129 };
2130
2131 /* Lookahead to detect a FUP packet after OVF */
2132 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2133 {
2134         struct intel_pt_ovf_fup_info *data = pkt_info->data;
2135
2136         if (pkt_info->packet.type == INTEL_PT_CYC ||
2137             pkt_info->packet.type == INTEL_PT_MTC ||
2138             pkt_info->packet.type == INTEL_PT_TSC)
2139                 return !--(data->max_lookahead);
2140         data->found = pkt_info->packet.type == INTEL_PT_FUP;
2141         return 1;
2142 }
2143
2144 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2145 {
2146         struct intel_pt_ovf_fup_info data = {
2147                 .max_lookahead = 16,
2148                 .found = false,
2149         };
2150
2151         intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2152         return data.found;
2153 }
2154
2155 /* Lookahead and get the TMA packet after TSC */
2156 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2157 {
2158         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2159
2160         if (pkt_info->packet.type == INTEL_PT_CYC ||
2161             pkt_info->packet.type == INTEL_PT_MTC)
2162                 return !--(data->max_lookahead);
2163
2164         if (pkt_info->packet.type == INTEL_PT_TMA) {
2165                 data->tma_packet = pkt_info->packet;
2166                 data->tma = true;
2167         }
2168         return 1;
2169 }
2170
2171 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2172 {
2173         if (decoder->tsc_ctc_mult)
2174                 return ctc * decoder->tsc_ctc_mult;
2175         else
2176                 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2177 }
2178
2179 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2180                                            uint32_t ctc,
2181                                            uint32_t fc,
2182                                            uint64_t last_ctc_timestamp,
2183                                            uint64_t ctc_delta,
2184                                            uint32_t last_ctc)
2185 {
2186         /* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2187         uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2188         /*
2189          * Number of CTC ticks from there until current TMA packet. We would
2190          * expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2191          * past an MTC, so a sign-extended value is used.
2192          */
2193         uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2194         /* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2195         uint64_t new_ctc_delta = ctc_delta + delta;
2196         uint64_t expected_tsc;
2197
2198         /*
2199          * Convert CTC ticks to TSC ticks, add the starting point
2200          * (last_ctc_timestamp) and the fast counter from the TMA packet.
2201          */
2202         expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2203
2204         if (intel_pt_enable_logging) {
2205                 intel_pt_log_x64(last_mtc_ctc);
2206                 intel_pt_log_x32(last_ctc);
2207                 intel_pt_log_x64(ctc_delta);
2208                 intel_pt_log_x64(delta);
2209                 intel_pt_log_x32(ctc);
2210                 intel_pt_log_x64(new_ctc_delta);
2211                 intel_pt_log_x64(last_ctc_timestamp);
2212                 intel_pt_log_x32(fc);
2213                 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2214                 intel_pt_log_x64(expected_tsc);
2215         }
2216
2217         return expected_tsc;
2218 }
2219
2220 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2221                                       struct intel_pt_vm_tsc_info *data)
2222 {
2223         uint32_t ctc = data->tma_packet.payload;
2224         uint32_t fc = data->tma_packet.count;
2225
2226         return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2227                                           decoder->ctc_timestamp,
2228                                           data->ctc_delta, data->last_ctc);
2229 }
2230
2231 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2232                                       struct intel_pt_vmcs_info *vmcs_info)
2233 {
2234         uint64_t payload = decoder->packet.payload;
2235
2236         /* VMX adds the TSC Offset, so subtract to get host TSC */
2237         decoder->packet.payload -= vmcs_info->tsc_offset;
2238         /* TSC packet has only 7 bytes */
2239         decoder->packet.payload &= SEVEN_BYTES;
2240
2241         /*
2242          * The buffer is mmapped from the data file, so this also updates the
2243          * data file.
2244          */
2245         if (!decoder->vm_tm_corr_dry_run)
2246                 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2247
2248         intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2249                      "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2250                      payload, decoder->packet.payload, vmcs_info->vmcs,
2251                      vmcs_info->tsc_offset);
2252 }
2253
2254 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2255                                              uint64_t tsc_offset)
2256 {
2257         struct intel_pt_vmcs_info vmcs_info = {
2258                 .vmcs = NO_VMCS,
2259                 .tsc_offset = tsc_offset
2260         };
2261
2262         intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2263 }
2264
2265 static inline bool in_vm(uint64_t pip_payload)
2266 {
2267         return pip_payload & 1;
2268 }
2269
2270 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2271 {
2272         return pip_packet->payload & 1;
2273 }
2274
2275 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2276 {
2277         p_log("VMCS: %#" PRIx64 "  TSC Offset %#" PRIx64,
2278               vmcs_info->vmcs, vmcs_info->tsc_offset);
2279 }
2280
2281 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2282                                     struct intel_pt_vm_tsc_info *data)
2283 {
2284         memset(data, 0, sizeof(*data));
2285         data->ctc_delta = decoder->ctc_delta;
2286         data->last_ctc = decoder->last_ctc;
2287         intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2288         if (data->tsc && !data->psbend)
2289                 p_log("ERROR: PSB without PSBEND");
2290         decoder->in_psb = data->psbend;
2291 }
2292
2293 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2294                                           struct intel_pt_vm_tsc_info *data,
2295                                           struct intel_pt_vmcs_info *vmcs_info,
2296                                           uint64_t host_tsc)
2297 {
2298         if (!decoder->in_psb) {
2299                 /* Can't happen */
2300                 p_log("ERROR: First TSC is not in PSB+");
2301         }
2302
2303         if (data->pip) {
2304                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2305                         if (vmcs_info && vmcs_info->tsc_offset) {
2306                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2307                                 decoder->vm_tm_corr_reliable = true;
2308                         } else {
2309                                 p_log("ERROR: First TSC, unknown TSC Offset");
2310                         }
2311                 } else { /* Host */
2312                         decoder->vm_tm_corr_reliable = true;
2313                 }
2314         } else { /* Host or Guest */
2315                 decoder->vm_tm_corr_reliable = false;
2316                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2317                         /* Assume Host */
2318                 } else {
2319                         /* Assume Guest */
2320                         if (vmcs_info && vmcs_info->tsc_offset)
2321                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2322                         else
2323                                 p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2324                 }
2325         }
2326 }
2327
2328 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2329                                     struct intel_pt_vm_tsc_info *data)
2330 {
2331         struct intel_pt_vmcs_info *vmcs_info;
2332         uint64_t tsc_offset = 0;
2333         uint64_t vmcs;
2334         bool reliable = true;
2335         uint64_t expected_tsc;
2336         uint64_t host_tsc;
2337         uint64_t ref_timestamp;
2338
2339         bool assign = false;
2340         bool assign_reliable = false;
2341
2342         /* Already have 'data' for the in_psb case */
2343         if (!decoder->in_psb) {
2344                 memset(data, 0, sizeof(*data));
2345                 data->ctc_delta = decoder->ctc_delta;
2346                 data->last_ctc = decoder->last_ctc;
2347                 data->max_lookahead = 16;
2348                 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2349                 if (decoder->pge) {
2350                         data->pip = true;
2351                         data->pip_packet.payload = decoder->pip_payload;
2352                 }
2353         }
2354
2355         /* Calculations depend on having TMA packets */
2356         if (!data->tma) {
2357                 p_log("ERROR: TSC without TMA");
2358                 return;
2359         }
2360
2361         vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2362         if (vmcs == NO_VMCS)
2363                 vmcs = 0;
2364
2365         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2366
2367         ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2368         host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2369
2370         if (!decoder->ctc_timestamp) {
2371                 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2372                 return;
2373         }
2374
2375         expected_tsc = intel_pt_expected_tsc(decoder, data);
2376
2377         tsc_offset = host_tsc - expected_tsc;
2378
2379         /* Determine if TSC is from Host or Guest */
2380         if (data->pip) {
2381                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2382                         if (!vmcs_info) {
2383                                 /* PIP NR=1 without VMCS cannot happen */
2384                                 p_log("ERROR: Missing VMCS");
2385                                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2386                                 decoder->vm_tm_corr_reliable = false;
2387                                 return;
2388                         }
2389                 } else { /* Host */
2390                         decoder->last_reliable_timestamp = host_tsc;
2391                         decoder->vm_tm_corr_reliable = true;
2392                         return;
2393                 }
2394         } else { /* Host or Guest */
2395                 reliable = false; /* Host/Guest is a guess, so not reliable */
2396                 if (decoder->in_psb) {
2397                         if (!tsc_offset)
2398                                 return; /* Zero TSC Offset, assume Host */
2399                         /*
2400                          * TSC packet has only 7 bytes of TSC. We have no
2401                          * information about the Guest's 8th byte, but it
2402                          * doesn't matter because we only need 7 bytes.
2403                          * Here, since the 8th byte is unreliable and
2404                          * irrelevant, compare only 7 byes.
2405                          */
2406                         if (vmcs_info &&
2407                             (tsc_offset & SEVEN_BYTES) ==
2408                             (vmcs_info->tsc_offset & SEVEN_BYTES)) {
2409                                 /* Same TSC Offset as last VMCS, assume Guest */
2410                                 goto guest;
2411                         }
2412                 }
2413                 /*
2414                  * Check if the host_tsc is within the expected range.
2415                  * Note, we could narrow the range more by looking ahead for
2416                  * the next host TSC in the same buffer, but we don't bother to
2417                  * do that because this is probably good enough.
2418                  */
2419                 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2420                         /* Within expected range for Host TSC, assume Host */
2421                         decoder->vm_tm_corr_reliable = false;
2422                         return;
2423                 }
2424         }
2425
2426 guest: /* Assuming Guest */
2427
2428         /* Determine whether to assign TSC Offset */
2429         if (vmcs_info && vmcs_info->vmcs) {
2430                 if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2431                         assign = false;
2432                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2433                            decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2434                         /* Continuous tracing, TSC in a PSB is not a time loss */
2435                         assign = true;
2436                         assign_reliable = true;
2437                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2438                         /*
2439                          * Unlikely to be a time loss TSC in a PSB which is not
2440                          * at the start of a buffer.
2441                          */
2442                         assign = true;
2443                         assign_reliable = false;
2444                 }
2445         }
2446
2447         /* Record VMCS TSC Offset */
2448         if (assign && (vmcs_info->tsc_offset != tsc_offset ||
2449                        vmcs_info->reliable != assign_reliable)) {
2450                 bool print = vmcs_info->tsc_offset != tsc_offset;
2451
2452                 vmcs_info->tsc_offset = tsc_offset;
2453                 vmcs_info->reliable = assign_reliable;
2454                 if (print)
2455                         intel_pt_print_vmcs_info(vmcs_info);
2456         }
2457
2458         /* Determine what TSC Offset to use */
2459         if (vmcs_info && vmcs_info->tsc_offset) {
2460                 if (!vmcs_info->reliable)
2461                         reliable = false;
2462                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2463         } else {
2464                 reliable = false;
2465                 if (vmcs_info) {
2466                         if (!vmcs_info->error_printed) {
2467                                 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2468                                       vmcs_info->vmcs);
2469                                 vmcs_info->error_printed = true;
2470                         }
2471                 } else {
2472                         if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2473                                 p_log("ERROR: Unknown VMCS");
2474                 }
2475                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2476         }
2477
2478         decoder->vm_tm_corr_reliable = reliable;
2479 }
2480
2481 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2482 {
2483         uint64_t host_tsc = decoder->packet.payload;
2484         uint64_t guest_tsc = decoder->packet.payload;
2485         struct intel_pt_vmcs_info *vmcs_info;
2486         uint64_t vmcs;
2487
2488         vmcs = decoder->vmcs;
2489         if (vmcs == NO_VMCS)
2490                 vmcs = 0;
2491
2492         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2493
2494         if (decoder->pge) {
2495                 if (in_vm(decoder->pip_payload)) { /* Guest */
2496                         if (!vmcs_info) {
2497                                 /* PIP NR=1 without VMCS cannot happen */
2498                                 p_log("ERROR: Missing VMCS");
2499                         }
2500                 } else { /* Host */
2501                         return;
2502                 }
2503         } else { /* Host or Guest */
2504                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2505                         /* Within expected range for Host TSC, assume Host */
2506                         return;
2507                 }
2508         }
2509
2510         if (vmcs_info) {
2511                 /* Translate Guest TSC to Host TSC */
2512                 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2513                 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2514                 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2515                              "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2516                              guest_tsc, host_tsc, vmcs_info->vmcs,
2517                              vmcs_info->tsc_offset);
2518                 if (!intel_pt_time_in_range(decoder, host_tsc) &&
2519                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2520                         p_log("Timestamp out of range");
2521         } else {
2522                 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2523                         p_log("ERROR: Unknown VMCS");
2524                 host_tsc = decoder->timestamp;
2525         }
2526
2527         decoder->packet.payload = host_tsc;
2528
2529         if (!decoder->vm_tm_corr_dry_run)
2530                 memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2531 }
2532
2533 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2534 {
2535         struct intel_pt_vm_tsc_info data = { .psbend = false };
2536         bool pge;
2537         int err;
2538
2539         if (decoder->in_psb)
2540                 intel_pt_vm_tm_corr_psb(decoder, &data);
2541
2542         while (1) {
2543                 err = intel_pt_get_next_packet(decoder);
2544                 if (err == -ENOLINK)
2545                         continue;
2546                 if (err)
2547                         break;
2548
2549                 switch (decoder->packet.type) {
2550                 case INTEL_PT_TIP_PGD:
2551                         decoder->pge = false;
2552                         decoder->vm_tm_corr_continuous = false;
2553                         break;
2554
2555                 case INTEL_PT_TNT:
2556                 case INTEL_PT_TIP:
2557                 case INTEL_PT_TIP_PGE:
2558                         decoder->pge = true;
2559                         break;
2560
2561                 case INTEL_PT_OVF:
2562                         decoder->in_psb = false;
2563                         pge = decoder->pge;
2564                         decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2565                         if (pge != decoder->pge)
2566                                 intel_pt_log("Surprising PGE change in OVF!");
2567                         if (!decoder->pge)
2568                                 decoder->vm_tm_corr_continuous = false;
2569                         break;
2570
2571                 case INTEL_PT_FUP:
2572                         if (decoder->in_psb)
2573                                 decoder->pge = true;
2574                         break;
2575
2576                 case INTEL_PT_TRACESTOP:
2577                         decoder->pge = false;
2578                         decoder->vm_tm_corr_continuous = false;
2579                         decoder->have_tma = false;
2580                         break;
2581
2582                 case INTEL_PT_PSB:
2583                         intel_pt_vm_tm_corr_psb(decoder, &data);
2584                         break;
2585
2586                 case INTEL_PT_PIP:
2587                         decoder->pip_payload = decoder->packet.payload;
2588                         break;
2589
2590                 case INTEL_PT_MTC:
2591                         intel_pt_calc_mtc_timestamp(decoder);
2592                         break;
2593
2594                 case INTEL_PT_TSC:
2595                         intel_pt_vm_tm_corr_tsc(decoder, &data);
2596                         intel_pt_calc_tsc_timestamp(decoder);
2597                         decoder->vm_tm_corr_same_buf = true;
2598                         decoder->vm_tm_corr_continuous = decoder->pge;
2599                         break;
2600
2601                 case INTEL_PT_TMA:
2602                         intel_pt_calc_tma(decoder);
2603                         break;
2604
2605                 case INTEL_PT_CYC:
2606                         intel_pt_calc_cyc_timestamp(decoder);
2607                         break;
2608
2609                 case INTEL_PT_CBR:
2610                         intel_pt_calc_cbr(decoder);
2611                         break;
2612
2613                 case INTEL_PT_PSBEND:
2614                         decoder->in_psb = false;
2615                         data.psbend = false;
2616                         break;
2617
2618                 case INTEL_PT_VMCS:
2619                         if (decoder->packet.payload != NO_VMCS)
2620                                 decoder->vmcs = decoder->packet.payload;
2621                         break;
2622
2623                 case INTEL_PT_BBP:
2624                         decoder->blk_type = decoder->packet.payload;
2625                         break;
2626
2627                 case INTEL_PT_BIP:
2628                         if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2629                             decoder->packet.count == 2)
2630                                 intel_pt_vm_tm_corr_pebs_tsc(decoder);
2631                         break;
2632
2633                 case INTEL_PT_BEP:
2634                 case INTEL_PT_BEP_IP:
2635                         decoder->blk_type = 0;
2636                         break;
2637
2638                 case INTEL_PT_MODE_EXEC:
2639                 case INTEL_PT_MODE_TSX:
2640                 case INTEL_PT_MNT:
2641                 case INTEL_PT_PAD:
2642                 case INTEL_PT_PTWRITE_IP:
2643                 case INTEL_PT_PTWRITE:
2644                 case INTEL_PT_MWAIT:
2645                 case INTEL_PT_PWRE:
2646                 case INTEL_PT_EXSTOP_IP:
2647                 case INTEL_PT_EXSTOP:
2648                 case INTEL_PT_PWRX:
2649                 case INTEL_PT_BAD: /* Does not happen */
2650                 default:
2651                         break;
2652                 }
2653         }
2654
2655         return err;
2656 }
2657
2658 #define HOP_PROCESS     0
2659 #define HOP_IGNORE      1
2660 #define HOP_RETURN      2
2661 #define HOP_AGAIN       3
2662
2663 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2664
2665 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2666 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
2667 {
2668         /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2669         if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2670                 *err = intel_pt_scan_for_psb(decoder);
2671                 if (*err)
2672                         return HOP_RETURN;
2673         }
2674
2675         switch (decoder->packet.type) {
2676         case INTEL_PT_TNT:
2677                 return HOP_IGNORE;
2678
2679         case INTEL_PT_TIP_PGD:
2680                 if (!decoder->packet.count) {
2681                         intel_pt_set_nr(decoder);
2682                         return HOP_IGNORE;
2683                 }
2684                 intel_pt_set_ip(decoder);
2685                 decoder->state.type |= INTEL_PT_TRACE_END;
2686                 decoder->state.from_ip = 0;
2687                 decoder->state.to_ip = decoder->ip;
2688                 intel_pt_update_nr(decoder);
2689                 return HOP_RETURN;
2690
2691         case INTEL_PT_TIP:
2692                 if (!decoder->packet.count) {
2693                         intel_pt_set_nr(decoder);
2694                         return HOP_IGNORE;
2695                 }
2696                 intel_pt_set_ip(decoder);
2697                 decoder->state.type = INTEL_PT_INSTRUCTION;
2698                 decoder->state.from_ip = decoder->ip;
2699                 decoder->state.to_ip = 0;
2700                 intel_pt_update_nr(decoder);
2701                 return HOP_RETURN;
2702
2703         case INTEL_PT_FUP:
2704                 if (!decoder->packet.count)
2705                         return HOP_IGNORE;
2706                 intel_pt_set_ip(decoder);
2707                 if (intel_pt_fup_event(decoder))
2708                         return HOP_RETURN;
2709                 if (!decoder->branch_enable)
2710                         *no_tip = true;
2711                 if (*no_tip) {
2712                         decoder->state.type = INTEL_PT_INSTRUCTION;
2713                         decoder->state.from_ip = decoder->ip;
2714                         decoder->state.to_ip = 0;
2715                         return HOP_RETURN;
2716                 }
2717                 *err = intel_pt_walk_fup_tip(decoder);
2718                 if (!*err)
2719                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2720                 return HOP_RETURN;
2721
2722         case INTEL_PT_PSB:
2723                 decoder->state.psb_offset = decoder->pos;
2724                 decoder->psb_ip = 0;
2725                 decoder->last_ip = 0;
2726                 decoder->have_last_ip = true;
2727                 *err = intel_pt_walk_psbend(decoder);
2728                 if (*err == -EAGAIN)
2729                         return HOP_AGAIN;
2730                 if (*err)
2731                         return HOP_RETURN;
2732                 decoder->state.type = INTEL_PT_PSB_EVT;
2733                 if (decoder->psb_ip) {
2734                         decoder->state.type |= INTEL_PT_INSTRUCTION;
2735                         decoder->ip = decoder->psb_ip;
2736                 }
2737                 decoder->state.from_ip = decoder->psb_ip;
2738                 decoder->state.to_ip = 0;
2739                 return HOP_RETURN;
2740
2741         case INTEL_PT_BAD:
2742         case INTEL_PT_PAD:
2743         case INTEL_PT_TIP_PGE:
2744         case INTEL_PT_TSC:
2745         case INTEL_PT_TMA:
2746         case INTEL_PT_MODE_EXEC:
2747         case INTEL_PT_MODE_TSX:
2748         case INTEL_PT_MTC:
2749         case INTEL_PT_CYC:
2750         case INTEL_PT_VMCS:
2751         case INTEL_PT_PSBEND:
2752         case INTEL_PT_CBR:
2753         case INTEL_PT_TRACESTOP:
2754         case INTEL_PT_PIP:
2755         case INTEL_PT_OVF:
2756         case INTEL_PT_MNT:
2757         case INTEL_PT_PTWRITE:
2758         case INTEL_PT_PTWRITE_IP:
2759         case INTEL_PT_EXSTOP:
2760         case INTEL_PT_EXSTOP_IP:
2761         case INTEL_PT_MWAIT:
2762         case INTEL_PT_PWRE:
2763         case INTEL_PT_PWRX:
2764         case INTEL_PT_BBP:
2765         case INTEL_PT_BIP:
2766         case INTEL_PT_BEP:
2767         case INTEL_PT_BEP_IP:
2768         default:
2769                 return HOP_PROCESS;
2770         }
2771 }
2772
2773 struct intel_pt_psb_info {
2774         struct intel_pt_pkt fup_packet;
2775         bool fup;
2776         int after_psbend;
2777 };
2778
2779 /* Lookahead and get the FUP packet from PSB+ */
2780 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2781 {
2782         struct intel_pt_psb_info *data = pkt_info->data;
2783
2784         switch (pkt_info->packet.type) {
2785         case INTEL_PT_PAD:
2786         case INTEL_PT_MNT:
2787         case INTEL_PT_TSC:
2788         case INTEL_PT_TMA:
2789         case INTEL_PT_MODE_EXEC:
2790         case INTEL_PT_MODE_TSX:
2791         case INTEL_PT_MTC:
2792         case INTEL_PT_CYC:
2793         case INTEL_PT_VMCS:
2794         case INTEL_PT_CBR:
2795         case INTEL_PT_PIP:
2796                 if (data->after_psbend) {
2797                         data->after_psbend -= 1;
2798                         if (!data->after_psbend)
2799                                 return 1;
2800                 }
2801                 break;
2802
2803         case INTEL_PT_FUP:
2804                 if (data->after_psbend)
2805                         return 1;
2806                 if (data->fup || pkt_info->packet.count == 0)
2807                         return 1;
2808                 data->fup_packet = pkt_info->packet;
2809                 data->fup = true;
2810                 break;
2811
2812         case INTEL_PT_PSBEND:
2813                 if (!data->fup)
2814                         return 1;
2815                 /* Keep going to check for a TIP.PGE */
2816                 data->after_psbend = 6;
2817                 break;
2818
2819         case INTEL_PT_TIP_PGE:
2820                 /* Ignore FUP in PSB+ if followed by TIP.PGE */
2821                 if (data->after_psbend)
2822                         data->fup = false;
2823                 return 1;
2824
2825         case INTEL_PT_PTWRITE:
2826         case INTEL_PT_PTWRITE_IP:
2827         case INTEL_PT_EXSTOP:
2828         case INTEL_PT_EXSTOP_IP:
2829         case INTEL_PT_MWAIT:
2830         case INTEL_PT_PWRE:
2831         case INTEL_PT_PWRX:
2832         case INTEL_PT_BBP:
2833         case INTEL_PT_BIP:
2834         case INTEL_PT_BEP:
2835         case INTEL_PT_BEP_IP:
2836                 if (data->after_psbend) {
2837                         data->after_psbend -= 1;
2838                         if (!data->after_psbend)
2839                                 return 1;
2840                         break;
2841                 }
2842                 return 1;
2843
2844         case INTEL_PT_OVF:
2845         case INTEL_PT_BAD:
2846         case INTEL_PT_TNT:
2847         case INTEL_PT_TIP_PGD:
2848         case INTEL_PT_TIP:
2849         case INTEL_PT_PSB:
2850         case INTEL_PT_TRACESTOP:
2851         default:
2852                 return 1;
2853         }
2854
2855         return 0;
2856 }
2857
2858 static int intel_pt_psb(struct intel_pt_decoder *decoder)
2859 {
2860         int err;
2861
2862         decoder->last_ip = 0;
2863         decoder->psb_ip = 0;
2864         decoder->have_last_ip = true;
2865         intel_pt_clear_stack(&decoder->stack);
2866         err = intel_pt_walk_psbend(decoder);
2867         if (err)
2868                 return err;
2869         decoder->state.type = INTEL_PT_PSB_EVT;
2870         decoder->state.from_ip = decoder->psb_ip;
2871         decoder->state.to_ip = 0;
2872         return 0;
2873 }
2874
2875 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
2876 {
2877         int err;
2878
2879         if (decoder->ip != decoder->last_ip) {
2880                 err = intel_pt_walk_fup(decoder);
2881                 if (!err || err != -EAGAIN)
2882                         return err;
2883         }
2884
2885         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2886         err = intel_pt_psb(decoder);
2887         if (err) {
2888                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2889                 return -ENOENT;
2890         }
2891
2892         return 0;
2893 }
2894
2895 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
2896 {
2897         struct intel_pt_psb_info data = { .fup = false };
2898
2899         if (!decoder->branch_enable || !decoder->pge)
2900                 return false;
2901
2902         intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
2903         if (!data.fup)
2904                 return false;
2905
2906         decoder->packet = data.fup_packet;
2907         intel_pt_set_last_ip(decoder);
2908         decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
2909
2910         *err = intel_pt_fup_in_psb(decoder);
2911
2912         return true;
2913 }
2914
2915 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
2916 {
2917         int last_packet_type = INTEL_PT_PAD;
2918         bool no_tip = false;
2919         int err;
2920
2921         while (1) {
2922                 err = intel_pt_get_next_packet(decoder);
2923                 if (err)
2924                         return err;
2925 next:
2926                 if (decoder->cyc_threshold) {
2927                         if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
2928                                 decoder->sample_cyc = false;
2929                         last_packet_type = decoder->packet.type;
2930                 }
2931
2932                 if (decoder->hop) {
2933                         switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
2934                         case HOP_IGNORE:
2935                                 continue;
2936                         case HOP_RETURN:
2937                                 return err;
2938                         case HOP_AGAIN:
2939                                 goto next;
2940                         default:
2941                                 break;
2942                         }
2943                 }
2944
2945                 switch (decoder->packet.type) {
2946                 case INTEL_PT_TNT:
2947                         if (!decoder->packet.count)
2948                                 break;
2949                         decoder->tnt = decoder->packet;
2950                         decoder->pkt_state = INTEL_PT_STATE_TNT;
2951                         err = intel_pt_walk_tnt(decoder);
2952                         if (err == -EAGAIN)
2953                                 break;
2954                         return err;
2955
2956                 case INTEL_PT_TIP_PGD:
2957                         if (decoder->packet.count != 0)
2958                                 intel_pt_set_last_ip(decoder);
2959                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
2960                         return intel_pt_walk_tip(decoder);
2961
2962                 case INTEL_PT_TIP_PGE: {
2963                         decoder->pge = true;
2964                         intel_pt_mtc_cyc_cnt_pge(decoder);
2965                         intel_pt_set_nr(decoder);
2966                         if (decoder->packet.count == 0) {
2967                                 intel_pt_log_at("Skipping zero TIP.PGE",
2968                                                 decoder->pos);
2969                                 break;
2970                         }
2971                         intel_pt_set_ip(decoder);
2972                         decoder->state.from_ip = 0;
2973                         decoder->state.to_ip = decoder->ip;
2974                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2975                         /*
2976                          * In hop mode, resample to get the to_ip as an
2977                          * "instruction" sample.
2978                          */
2979                         if (decoder->hop)
2980                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2981                         return 0;
2982                 }
2983
2984                 case INTEL_PT_OVF:
2985                         return intel_pt_overflow(decoder);
2986
2987                 case INTEL_PT_TIP:
2988                         if (decoder->packet.count != 0)
2989                                 intel_pt_set_last_ip(decoder);
2990                         decoder->pkt_state = INTEL_PT_STATE_TIP;
2991                         return intel_pt_walk_tip(decoder);
2992
2993                 case INTEL_PT_FUP:
2994                         if (decoder->packet.count == 0) {
2995                                 intel_pt_log_at("Skipping zero FUP",
2996                                                 decoder->pos);
2997                                 no_tip = false;
2998                                 break;
2999                         }
3000                         intel_pt_set_last_ip(decoder);
3001                         if (!decoder->branch_enable) {
3002                                 decoder->ip = decoder->last_ip;
3003                                 if (intel_pt_fup_event(decoder))
3004                                         return 0;
3005                                 no_tip = false;
3006                                 break;
3007                         }
3008                         if (decoder->set_fup_mwait)
3009                                 no_tip = true;
3010                         if (no_tip)
3011                                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3012                         else
3013                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
3014                         err = intel_pt_walk_fup(decoder);
3015                         if (err != -EAGAIN)
3016                                 return err;
3017                         if (no_tip) {
3018                                 no_tip = false;
3019                                 break;
3020                         }
3021                         return intel_pt_walk_fup_tip(decoder);
3022
3023                 case INTEL_PT_TRACESTOP:
3024                         decoder->pge = false;
3025                         decoder->continuous_period = false;
3026                         intel_pt_clear_tx_flags(decoder);
3027                         decoder->have_tma = false;
3028                         break;
3029
3030                 case INTEL_PT_PSB:
3031                         decoder->state.psb_offset = decoder->pos;
3032                         decoder->psb_ip = 0;
3033                         if (intel_pt_psb_with_fup(decoder, &err))
3034                                 return err;
3035                         err = intel_pt_psb(decoder);
3036                         if (err == -EAGAIN)
3037                                 goto next;
3038                         return err;
3039
3040                 case INTEL_PT_PIP:
3041                         intel_pt_update_pip(decoder);
3042                         break;
3043
3044                 case INTEL_PT_MTC:
3045                         intel_pt_calc_mtc_timestamp(decoder);
3046                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3047                                 break;
3048                         /*
3049                          * Ensure that there has been an instruction since the
3050                          * last MTC.
3051                          */
3052                         if (!decoder->mtc_insn)
3053                                 break;
3054                         decoder->mtc_insn = false;
3055                         /* Ensure that there is a timestamp */
3056                         if (!decoder->timestamp)
3057                                 break;
3058                         decoder->state.type = INTEL_PT_INSTRUCTION;
3059                         decoder->state.from_ip = decoder->ip;
3060                         decoder->state.to_ip = 0;
3061                         decoder->mtc_insn = false;
3062                         return 0;
3063
3064                 case INTEL_PT_TSC:
3065                         intel_pt_calc_tsc_timestamp(decoder);
3066                         break;
3067
3068                 case INTEL_PT_TMA:
3069                         intel_pt_calc_tma(decoder);
3070                         break;
3071
3072                 case INTEL_PT_CYC:
3073                         intel_pt_calc_cyc_timestamp(decoder);
3074                         break;
3075
3076                 case INTEL_PT_CBR:
3077                         intel_pt_calc_cbr(decoder);
3078                         if (decoder->cbr != decoder->cbr_seen) {
3079                                 decoder->state.type = 0;
3080                                 return 0;
3081                         }
3082                         break;
3083
3084                 case INTEL_PT_MODE_EXEC:
3085                         decoder->exec_mode = decoder->packet.payload;
3086                         break;
3087
3088                 case INTEL_PT_MODE_TSX:
3089                         /* MODE_TSX need not be followed by FUP */
3090                         if (!decoder->pge || decoder->in_psb) {
3091                                 intel_pt_update_in_tx(decoder);
3092                                 break;
3093                         }
3094                         err = intel_pt_mode_tsx(decoder, &no_tip);
3095                         if (err)
3096                                 return err;
3097                         goto next;
3098
3099                 case INTEL_PT_BAD: /* Does not happen */
3100                         return intel_pt_bug(decoder);
3101
3102                 case INTEL_PT_PSBEND:
3103                 case INTEL_PT_VMCS:
3104                 case INTEL_PT_MNT:
3105                 case INTEL_PT_PAD:
3106                         break;
3107
3108                 case INTEL_PT_PTWRITE_IP:
3109                         decoder->fup_ptw_payload = decoder->packet.payload;
3110                         err = intel_pt_get_next_packet(decoder);
3111                         if (err)
3112                                 return err;
3113                         if (decoder->packet.type == INTEL_PT_FUP) {
3114                                 decoder->set_fup_ptw = true;
3115                                 no_tip = true;
3116                         } else {
3117                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3118                                                 decoder->pos);
3119                         }
3120                         goto next;
3121
3122                 case INTEL_PT_PTWRITE:
3123                         decoder->state.type = INTEL_PT_PTW;
3124                         decoder->state.from_ip = decoder->ip;
3125                         decoder->state.to_ip = 0;
3126                         decoder->state.ptw_payload = decoder->packet.payload;
3127                         return 0;
3128
3129                 case INTEL_PT_MWAIT:
3130                         decoder->fup_mwait_payload = decoder->packet.payload;
3131                         decoder->set_fup_mwait = true;
3132                         break;
3133
3134                 case INTEL_PT_PWRE:
3135                         if (decoder->set_fup_mwait) {
3136                                 decoder->fup_pwre_payload =
3137                                                         decoder->packet.payload;
3138                                 decoder->set_fup_pwre = true;
3139                                 break;
3140                         }
3141                         decoder->state.type = INTEL_PT_PWR_ENTRY;
3142                         decoder->state.from_ip = decoder->ip;
3143                         decoder->state.to_ip = 0;
3144                         decoder->state.pwrx_payload = decoder->packet.payload;
3145                         return 0;
3146
3147                 case INTEL_PT_EXSTOP_IP:
3148                         err = intel_pt_get_next_packet(decoder);
3149                         if (err)
3150                                 return err;
3151                         if (decoder->packet.type == INTEL_PT_FUP) {
3152                                 decoder->set_fup_exstop = true;
3153                                 no_tip = true;
3154                         } else {
3155                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3156                                                 decoder->pos);
3157                         }
3158                         goto next;
3159
3160                 case INTEL_PT_EXSTOP:
3161                         decoder->state.type = INTEL_PT_EX_STOP;
3162                         decoder->state.from_ip = decoder->ip;
3163                         decoder->state.to_ip = 0;
3164                         return 0;
3165
3166                 case INTEL_PT_PWRX:
3167                         decoder->state.type = INTEL_PT_PWR_EXIT;
3168                         decoder->state.from_ip = decoder->ip;
3169                         decoder->state.to_ip = 0;
3170                         decoder->state.pwrx_payload = decoder->packet.payload;
3171                         return 0;
3172
3173                 case INTEL_PT_BBP:
3174                         intel_pt_bbp(decoder);
3175                         break;
3176
3177                 case INTEL_PT_BIP:
3178                         intel_pt_bip(decoder);
3179                         break;
3180
3181                 case INTEL_PT_BEP:
3182                         decoder->state.type = INTEL_PT_BLK_ITEMS;
3183                         decoder->state.from_ip = decoder->ip;
3184                         decoder->state.to_ip = 0;
3185                         return 0;
3186
3187                 case INTEL_PT_BEP_IP:
3188                         err = intel_pt_get_next_packet(decoder);
3189                         if (err)
3190                                 return err;
3191                         if (decoder->packet.type == INTEL_PT_FUP) {
3192                                 decoder->set_fup_bep = true;
3193                                 no_tip = true;
3194                         } else {
3195                                 intel_pt_log_at("ERROR: Missing FUP after BEP",
3196                                                 decoder->pos);
3197                         }
3198                         goto next;
3199
3200                 default:
3201                         return intel_pt_bug(decoder);
3202                 }
3203         }
3204 }
3205
3206 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3207 {
3208         return decoder->packet.count &&
3209                (decoder->have_last_ip || decoder->packet.count == 3 ||
3210                 decoder->packet.count == 6);
3211 }
3212
3213 /* Walk PSB+ packets to get in sync. */
3214 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3215 {
3216         int err;
3217
3218         decoder->in_psb = true;
3219
3220         while (1) {
3221                 err = intel_pt_get_next_packet(decoder);
3222                 if (err)
3223                         goto out;
3224
3225                 switch (decoder->packet.type) {
3226                 case INTEL_PT_TIP_PGD:
3227                         decoder->continuous_period = false;
3228                         __fallthrough;
3229                 case INTEL_PT_TIP_PGE:
3230                 case INTEL_PT_TIP:
3231                 case INTEL_PT_PTWRITE:
3232                 case INTEL_PT_PTWRITE_IP:
3233                 case INTEL_PT_EXSTOP:
3234                 case INTEL_PT_EXSTOP_IP:
3235                 case INTEL_PT_MWAIT:
3236                 case INTEL_PT_PWRE:
3237                 case INTEL_PT_PWRX:
3238                 case INTEL_PT_BBP:
3239                 case INTEL_PT_BIP:
3240                 case INTEL_PT_BEP:
3241                 case INTEL_PT_BEP_IP:
3242                         intel_pt_log("ERROR: Unexpected packet\n");
3243                         err = -ENOENT;
3244                         goto out;
3245
3246                 case INTEL_PT_FUP:
3247                         decoder->pge = true;
3248                         if (intel_pt_have_ip(decoder)) {
3249                                 uint64_t current_ip = decoder->ip;
3250
3251                                 intel_pt_set_ip(decoder);
3252                                 decoder->psb_ip = decoder->ip;
3253                                 if (current_ip)
3254                                         intel_pt_log_to("Setting IP",
3255                                                         decoder->ip);
3256                         }
3257                         break;
3258
3259                 case INTEL_PT_MTC:
3260                         intel_pt_calc_mtc_timestamp(decoder);
3261                         break;
3262
3263                 case INTEL_PT_TSC:
3264                         intel_pt_calc_tsc_timestamp(decoder);
3265                         break;
3266
3267                 case INTEL_PT_TMA:
3268                         intel_pt_calc_tma(decoder);
3269                         break;
3270
3271                 case INTEL_PT_CYC:
3272                         intel_pt_calc_cyc_timestamp(decoder);
3273                         break;
3274
3275                 case INTEL_PT_CBR:
3276                         intel_pt_calc_cbr(decoder);
3277                         break;
3278
3279                 case INTEL_PT_PIP:
3280                         intel_pt_set_pip(decoder);
3281                         break;
3282
3283                 case INTEL_PT_MODE_EXEC:
3284                         decoder->exec_mode = decoder->packet.payload;
3285                         break;
3286
3287                 case INTEL_PT_MODE_TSX:
3288                         intel_pt_update_in_tx(decoder);
3289                         break;
3290
3291                 case INTEL_PT_TRACESTOP:
3292                         decoder->pge = false;
3293                         decoder->continuous_period = false;
3294                         intel_pt_clear_tx_flags(decoder);
3295                         __fallthrough;
3296
3297                 case INTEL_PT_TNT:
3298                         decoder->have_tma = false;
3299                         intel_pt_log("ERROR: Unexpected packet\n");
3300                         if (decoder->ip)
3301                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
3302                         else
3303                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3304                         err = -ENOENT;
3305                         goto out;
3306
3307                 case INTEL_PT_BAD: /* Does not happen */
3308                         err = intel_pt_bug(decoder);
3309                         goto out;
3310
3311                 case INTEL_PT_OVF:
3312                         err = intel_pt_overflow(decoder);
3313                         goto out;
3314
3315                 case INTEL_PT_PSBEND:
3316                         err = 0;
3317                         goto out;
3318
3319                 case INTEL_PT_PSB:
3320                 case INTEL_PT_VMCS:
3321                 case INTEL_PT_MNT:
3322                 case INTEL_PT_PAD:
3323                 default:
3324                         break;
3325                 }
3326         }
3327 out:
3328         decoder->in_psb = false;
3329
3330         return err;
3331 }
3332
3333 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3334 {
3335         int err;
3336
3337         while (1) {
3338                 err = intel_pt_get_next_packet(decoder);
3339                 if (err)
3340                         return err;
3341
3342                 switch (decoder->packet.type) {
3343                 case INTEL_PT_TIP_PGD:
3344                         decoder->continuous_period = false;
3345                         decoder->pge = false;
3346                         if (intel_pt_have_ip(decoder))
3347                                 intel_pt_set_ip(decoder);
3348                         if (!decoder->ip)
3349                                 break;
3350                         decoder->state.type |= INTEL_PT_TRACE_END;
3351                         return 0;
3352
3353                 case INTEL_PT_TIP_PGE:
3354                         decoder->pge = true;
3355                         intel_pt_mtc_cyc_cnt_pge(decoder);
3356                         if (intel_pt_have_ip(decoder))
3357                                 intel_pt_set_ip(decoder);
3358                         if (!decoder->ip)
3359                                 break;
3360                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3361                         return 0;
3362
3363                 case INTEL_PT_TIP:
3364                         decoder->pge = true;
3365                         if (intel_pt_have_ip(decoder))
3366                                 intel_pt_set_ip(decoder);
3367                         if (!decoder->ip)
3368                                 break;
3369                         return 0;
3370
3371                 case INTEL_PT_FUP:
3372                         if (intel_pt_have_ip(decoder))
3373                                 intel_pt_set_ip(decoder);
3374                         if (decoder->ip)
3375                                 return 0;
3376                         break;
3377
3378                 case INTEL_PT_MTC:
3379                         intel_pt_calc_mtc_timestamp(decoder);
3380                         break;
3381
3382                 case INTEL_PT_TSC:
3383                         intel_pt_calc_tsc_timestamp(decoder);
3384                         break;
3385
3386                 case INTEL_PT_TMA:
3387                         intel_pt_calc_tma(decoder);
3388                         break;
3389
3390                 case INTEL_PT_CYC:
3391                         intel_pt_calc_cyc_timestamp(decoder);
3392                         break;
3393
3394                 case INTEL_PT_CBR:
3395                         intel_pt_calc_cbr(decoder);
3396                         break;
3397
3398                 case INTEL_PT_PIP:
3399                         intel_pt_set_pip(decoder);
3400                         break;
3401
3402                 case INTEL_PT_MODE_EXEC:
3403                         decoder->exec_mode = decoder->packet.payload;
3404                         break;
3405
3406                 case INTEL_PT_MODE_TSX:
3407                         intel_pt_update_in_tx(decoder);
3408                         break;
3409
3410                 case INTEL_PT_OVF:
3411                         return intel_pt_overflow(decoder);
3412
3413                 case INTEL_PT_BAD: /* Does not happen */
3414                         return intel_pt_bug(decoder);
3415
3416                 case INTEL_PT_TRACESTOP:
3417                         decoder->pge = false;
3418                         decoder->continuous_period = false;
3419                         intel_pt_clear_tx_flags(decoder);
3420                         decoder->have_tma = false;
3421                         break;
3422
3423                 case INTEL_PT_PSB:
3424                         decoder->state.psb_offset = decoder->pos;
3425                         decoder->psb_ip = 0;
3426                         decoder->last_ip = 0;
3427                         decoder->have_last_ip = true;
3428                         intel_pt_clear_stack(&decoder->stack);
3429                         err = intel_pt_walk_psb(decoder);
3430                         if (err)
3431                                 return err;
3432                         decoder->state.type = INTEL_PT_PSB_EVT;
3433                         decoder->state.from_ip = decoder->psb_ip;
3434                         decoder->state.to_ip = 0;
3435                         return 0;
3436
3437                 case INTEL_PT_TNT:
3438                 case INTEL_PT_PSBEND:
3439                 case INTEL_PT_VMCS:
3440                 case INTEL_PT_MNT:
3441                 case INTEL_PT_PAD:
3442                 case INTEL_PT_PTWRITE:
3443                 case INTEL_PT_PTWRITE_IP:
3444                 case INTEL_PT_EXSTOP:
3445                 case INTEL_PT_EXSTOP_IP:
3446                 case INTEL_PT_MWAIT:
3447                 case INTEL_PT_PWRE:
3448                 case INTEL_PT_PWRX:
3449                 case INTEL_PT_BBP:
3450                 case INTEL_PT_BIP:
3451                 case INTEL_PT_BEP:
3452                 case INTEL_PT_BEP_IP:
3453                 default:
3454                         break;
3455                 }
3456         }
3457 }
3458
3459 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3460 {
3461         int err;
3462
3463         decoder->set_fup_tx_flags = false;
3464         decoder->set_fup_ptw = false;
3465         decoder->set_fup_mwait = false;
3466         decoder->set_fup_pwre = false;
3467         decoder->set_fup_exstop = false;
3468         decoder->set_fup_bep = false;
3469
3470         if (!decoder->branch_enable) {
3471                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3472                 decoder->overflow = false;
3473                 decoder->state.type = 0; /* Do not have a sample */
3474                 return 0;
3475         }
3476
3477         intel_pt_log("Scanning for full IP\n");
3478         err = intel_pt_walk_to_ip(decoder);
3479         if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3480                 return err;
3481
3482         /* In hop mode, resample to get the to_ip as an "instruction" sample */
3483         if (decoder->hop)
3484                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3485         else
3486                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3487         decoder->overflow = false;
3488
3489         decoder->state.from_ip = 0;
3490         decoder->state.to_ip = decoder->ip;
3491         intel_pt_log_to("Setting IP", decoder->ip);
3492
3493         return 0;
3494 }
3495
3496 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3497 {
3498         const unsigned char *end = decoder->buf + decoder->len;
3499         size_t i;
3500
3501         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3502                 if (i > decoder->len)
3503                         continue;
3504                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3505                         return i;
3506         }
3507         return 0;
3508 }
3509
3510 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3511 {
3512         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3513         const char *psb = INTEL_PT_PSB_STR;
3514
3515         if (rest_psb > decoder->len ||
3516             memcmp(decoder->buf, psb + part_psb, rest_psb))
3517                 return 0;
3518
3519         return rest_psb;
3520 }
3521
3522 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3523                                   int part_psb)
3524 {
3525         int rest_psb, ret;
3526
3527         decoder->pos += decoder->len;
3528         decoder->len = 0;
3529
3530         ret = intel_pt_get_next_data(decoder, false);
3531         if (ret)
3532                 return ret;
3533
3534         rest_psb = intel_pt_rest_psb(decoder, part_psb);
3535         if (!rest_psb)
3536                 return 0;
3537
3538         decoder->pos -= part_psb;
3539         decoder->next_buf = decoder->buf + rest_psb;
3540         decoder->next_len = decoder->len - rest_psb;
3541         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3542         decoder->buf = decoder->temp_buf;
3543         decoder->len = INTEL_PT_PSB_LEN;
3544
3545         return 0;
3546 }
3547
3548 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3549 {
3550         unsigned char *next;
3551         int ret;
3552
3553         intel_pt_log("Scanning for PSB\n");
3554         while (1) {
3555                 if (!decoder->len) {
3556                         ret = intel_pt_get_next_data(decoder, false);
3557                         if (ret)
3558                                 return ret;
3559                 }
3560
3561                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3562                               INTEL_PT_PSB_LEN);
3563                 if (!next) {
3564                         int part_psb;
3565
3566                         part_psb = intel_pt_part_psb(decoder);
3567                         if (part_psb) {
3568                                 ret = intel_pt_get_split_psb(decoder, part_psb);
3569                                 if (ret)
3570                                         return ret;
3571                         } else {
3572                                 decoder->pos += decoder->len;
3573                                 decoder->len = 0;
3574                         }
3575                         continue;
3576                 }
3577
3578                 decoder->pkt_step = next - decoder->buf;
3579                 return intel_pt_get_next_packet(decoder);
3580         }
3581 }
3582
3583 static int intel_pt_sync(struct intel_pt_decoder *decoder)
3584 {
3585         int err;
3586
3587         decoder->pge = false;
3588         decoder->continuous_period = false;
3589         decoder->have_last_ip = false;
3590         decoder->last_ip = 0;
3591         decoder->psb_ip = 0;
3592         decoder->ip = 0;
3593         intel_pt_clear_stack(&decoder->stack);
3594
3595         err = intel_pt_scan_for_psb(decoder);
3596         if (err)
3597                 return err;
3598
3599         if (decoder->vm_time_correlation) {
3600                 decoder->in_psb = true;
3601                 if (!decoder->timestamp)
3602                         decoder->timestamp = 1;
3603                 decoder->state.type = 0;
3604                 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3605                 return 0;
3606         }
3607
3608         decoder->have_last_ip = true;
3609         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
3610
3611         err = intel_pt_walk_psb(decoder);
3612         if (err)
3613                 return err;
3614
3615         decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3616         decoder->state.from_ip = decoder->psb_ip;
3617         decoder->state.to_ip = 0;
3618
3619         if (decoder->ip) {
3620                 /*
3621                  * In hop mode, resample to get the PSB FUP ip as an
3622                  * "instruction" sample.
3623                  */
3624                 if (decoder->hop)
3625                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3626                 else
3627                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3628         }
3629
3630         return 0;
3631 }
3632
3633 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3634 {
3635         uint64_t est = decoder->sample_insn_cnt << 1;
3636
3637         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
3638                 goto out;
3639
3640         est *= decoder->max_non_turbo_ratio;
3641         est /= decoder->cbr;
3642 out:
3643         return decoder->sample_timestamp + est;
3644 }
3645
3646 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
3647 {
3648         int err;
3649
3650         do {
3651                 decoder->state.type = INTEL_PT_BRANCH;
3652                 decoder->state.flags = 0;
3653
3654                 switch (decoder->pkt_state) {
3655                 case INTEL_PT_STATE_NO_PSB:
3656                         err = intel_pt_sync(decoder);
3657                         break;
3658                 case INTEL_PT_STATE_NO_IP:
3659                         decoder->have_last_ip = false;
3660                         decoder->last_ip = 0;
3661                         decoder->ip = 0;
3662                         __fallthrough;
3663                 case INTEL_PT_STATE_ERR_RESYNC:
3664                         err = intel_pt_sync_ip(decoder);
3665                         break;
3666                 case INTEL_PT_STATE_IN_SYNC:
3667                         err = intel_pt_walk_trace(decoder);
3668                         break;
3669                 case INTEL_PT_STATE_TNT:
3670                 case INTEL_PT_STATE_TNT_CONT:
3671                         err = intel_pt_walk_tnt(decoder);
3672                         if (err == -EAGAIN)
3673                                 err = intel_pt_walk_trace(decoder);
3674                         break;
3675                 case INTEL_PT_STATE_TIP:
3676                 case INTEL_PT_STATE_TIP_PGD:
3677                         err = intel_pt_walk_tip(decoder);
3678                         break;
3679                 case INTEL_PT_STATE_FUP:
3680                         err = intel_pt_walk_fup(decoder);
3681                         if (err == -EAGAIN)
3682                                 err = intel_pt_walk_fup_tip(decoder);
3683                         break;
3684                 case INTEL_PT_STATE_FUP_NO_TIP:
3685                         err = intel_pt_walk_fup(decoder);
3686                         if (err == -EAGAIN)
3687                                 err = intel_pt_walk_trace(decoder);
3688                         break;
3689                 case INTEL_PT_STATE_FUP_IN_PSB:
3690                         err = intel_pt_fup_in_psb(decoder);
3691                         break;
3692                 case INTEL_PT_STATE_RESAMPLE:
3693                         err = intel_pt_resample(decoder);
3694                         break;
3695                 case INTEL_PT_STATE_VM_TIME_CORRELATION:
3696                         err = intel_pt_vm_time_correlation(decoder);
3697                         break;
3698                 default:
3699                         err = intel_pt_bug(decoder);
3700                         break;
3701                 }
3702         } while (err == -ENOLINK);
3703
3704         if (err) {
3705                 decoder->state.err = intel_pt_ext_err(err);
3706                 decoder->state.from_ip = decoder->ip;
3707                 intel_pt_update_sample_time(decoder);
3708                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3709                 intel_pt_set_nr(decoder);
3710         } else {
3711                 decoder->state.err = 0;
3712                 if (decoder->cbr != decoder->cbr_seen) {
3713                         decoder->cbr_seen = decoder->cbr;
3714                         if (!decoder->state.type) {
3715                                 decoder->state.from_ip = decoder->ip;
3716                                 decoder->state.to_ip = 0;
3717                         }
3718                         decoder->state.type |= INTEL_PT_CBR_CHG;
3719                         decoder->state.cbr_payload = decoder->cbr_payload;
3720                         decoder->state.cbr = decoder->cbr;
3721                 }
3722                 if (intel_pt_sample_time(decoder->pkt_state)) {
3723                         intel_pt_update_sample_time(decoder);
3724                         if (decoder->sample_cyc) {
3725                                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3726                                 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3727                                 decoder->sample_cyc = false;
3728                         }
3729                 }
3730                 /*
3731                  * When using only TSC/MTC to compute cycles, IPC can be
3732                  * sampled as soon as the cycle count changes.
3733                  */
3734                 if (!decoder->have_cyc)
3735                         decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3736         }
3737
3738          /* Let PSB event always have TSC timestamp */
3739         if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
3740                 decoder->sample_timestamp = decoder->tsc_timestamp;
3741
3742         decoder->state.from_nr = decoder->nr;
3743         decoder->state.to_nr = decoder->next_nr;
3744         decoder->nr = decoder->next_nr;
3745
3746         decoder->state.timestamp = decoder->sample_timestamp;
3747         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
3748         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
3749         decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
3750
3751         return &decoder->state;
3752 }
3753
3754 /**
3755  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
3756  * @buf: pointer to buffer pointer
3757  * @len: size of buffer
3758  *
3759  * Updates the buffer pointer to point to the start of the next PSB packet if
3760  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
3761  * @len is adjusted accordingly.
3762  *
3763  * Return: %true if a PSB packet is found, %false otherwise.
3764  */
3765 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
3766 {
3767         unsigned char *next;
3768
3769         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3770         if (next) {
3771                 *len -= next - *buf;
3772                 *buf = next;
3773                 return true;
3774         }
3775         return false;
3776 }
3777
3778 /**
3779  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
3780  *                     packet.
3781  * @buf: pointer to buffer pointer
3782  * @len: size of buffer
3783  *
3784  * Updates the buffer pointer to point to the start of the following PSB packet
3785  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
3786  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
3787  *
3788  * Return: %true if a PSB packet is found, %false otherwise.
3789  */
3790 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
3791 {
3792         unsigned char *next;
3793
3794         if (!*len)
3795                 return false;
3796
3797         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3798         if (next) {
3799                 *len -= next - *buf;
3800                 *buf = next;
3801                 return true;
3802         }
3803         return false;
3804 }
3805
3806 /**
3807  * intel_pt_last_psb - find the last PSB packet in a buffer.
3808  * @buf: buffer
3809  * @len: size of buffer
3810  *
3811  * This function finds the last PSB in a buffer.
3812  *
3813  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
3814  */
3815 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
3816 {
3817         const char *n = INTEL_PT_PSB_STR;
3818         unsigned char *p;
3819         size_t k;
3820
3821         if (len < INTEL_PT_PSB_LEN)
3822                 return NULL;
3823
3824         k = len - INTEL_PT_PSB_LEN + 1;
3825         while (1) {
3826                 p = memrchr(buf, n[0], k);
3827                 if (!p)
3828                         return NULL;
3829                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
3830                         return p;
3831                 k = p - buf;
3832                 if (!k)
3833                         return NULL;
3834         }
3835 }
3836
3837 /**
3838  * intel_pt_next_tsc - find and return next TSC.
3839  * @buf: buffer
3840  * @len: size of buffer
3841  * @tsc: TSC value returned
3842  * @rem: returns remaining size when TSC is found
3843  *
3844  * Find a TSC packet in @buf and return the TSC value.  This function assumes
3845  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
3846  * PSBEND packet is found.
3847  *
3848  * Return: %true if TSC is found, false otherwise.
3849  */
3850 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
3851                               size_t *rem)
3852 {
3853         enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
3854         struct intel_pt_pkt packet;
3855         int ret;
3856
3857         while (len) {
3858                 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
3859                 if (ret <= 0)
3860                         return false;
3861                 if (packet.type == INTEL_PT_TSC) {
3862                         *tsc = packet.payload;
3863                         *rem = len;
3864                         return true;
3865                 }
3866                 if (packet.type == INTEL_PT_PSBEND)
3867                         return false;
3868                 buf += ret;
3869                 len -= ret;
3870         }
3871         return false;
3872 }
3873
3874 /**
3875  * intel_pt_tsc_cmp - compare 7-byte TSCs.
3876  * @tsc1: first TSC to compare
3877  * @tsc2: second TSC to compare
3878  *
3879  * This function compares 7-byte TSC values allowing for the possibility that
3880  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
3881  * around so for that purpose this function assumes the absolute difference is
3882  * less than half the maximum difference.
3883  *
3884  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
3885  * after @tsc2.
3886  */
3887 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
3888 {
3889         const uint64_t halfway = (1ULL << 55);
3890
3891         if (tsc1 == tsc2)
3892                 return 0;
3893
3894         if (tsc1 < tsc2) {
3895                 if (tsc2 - tsc1 < halfway)
3896                         return -1;
3897                 else
3898                         return 1;
3899         } else {
3900                 if (tsc1 - tsc2 < halfway)
3901                         return 1;
3902                 else
3903                         return -1;
3904         }
3905 }
3906
3907 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
3908
3909 /**
3910  * adj_for_padding - adjust overlap to account for padding.
3911  * @buf_b: second buffer
3912  * @buf_a: first buffer
3913  * @len_a: size of first buffer
3914  *
3915  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
3916  * accordingly.
3917  *
3918  * Return: A pointer into @buf_b from where non-overlapped data starts
3919  */
3920 static unsigned char *adj_for_padding(unsigned char *buf_b,
3921                                       unsigned char *buf_a, size_t len_a)
3922 {
3923         unsigned char *p = buf_b - MAX_PADDING;
3924         unsigned char *q = buf_a + len_a - MAX_PADDING;
3925         int i;
3926
3927         for (i = MAX_PADDING; i; i--, p++, q++) {
3928                 if (*p != *q)
3929                         break;
3930         }
3931
3932         return p;
3933 }
3934
3935 /**
3936  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
3937  *                             using TSC.
3938  * @buf_a: first buffer
3939  * @len_a: size of first buffer
3940  * @buf_b: second buffer
3941  * @len_b: size of second buffer
3942  * @consecutive: returns true if there is data in buf_b that is consecutive
3943  *               to buf_a
3944  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
3945  *
3946  * If the trace contains TSC we can look at the last TSC of @buf_a and the
3947  * first TSC of @buf_b in order to determine if the buffers overlap, and then
3948  * walk forward in @buf_b until a later TSC is found.  A precondition is that
3949  * @buf_a and @buf_b are positioned at a PSB.
3950  *
3951  * Return: A pointer into @buf_b from where non-overlapped data starts, or
3952  * @buf_b + @len_b if there is no non-overlapped data.
3953  */
3954 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
3955                                                 size_t len_a,
3956                                                 unsigned char *buf_b,
3957                                                 size_t len_b, bool *consecutive,
3958                                                 bool ooo_tsc)
3959 {
3960         uint64_t tsc_a, tsc_b;
3961         unsigned char *p;
3962         size_t len, rem_a, rem_b;
3963
3964         p = intel_pt_last_psb(buf_a, len_a);
3965         if (!p)
3966                 return buf_b; /* No PSB in buf_a => no overlap */
3967
3968         len = len_a - (p - buf_a);
3969         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
3970                 /* The last PSB+ in buf_a is incomplete, so go back one more */
3971                 len_a -= len;
3972                 p = intel_pt_last_psb(buf_a, len_a);
3973                 if (!p)
3974                         return buf_b; /* No full PSB+ => assume no overlap */
3975                 len = len_a - (p - buf_a);
3976                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
3977                         return buf_b; /* No TSC in buf_a => assume no overlap */
3978         }
3979
3980         while (1) {
3981                 /* Ignore PSB+ with no TSC */
3982                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
3983                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
3984
3985                         /* Same TSC, so buffers are consecutive */
3986                         if (!cmp && rem_b >= rem_a) {
3987                                 unsigned char *start;
3988
3989                                 *consecutive = true;
3990                                 start = buf_b + len_b - (rem_b - rem_a);
3991                                 return adj_for_padding(start, buf_a, len_a);
3992                         }
3993                         if (cmp < 0 && !ooo_tsc)
3994                                 return buf_b; /* tsc_a < tsc_b => no overlap */
3995                 }
3996
3997                 if (!intel_pt_step_psb(&buf_b, &len_b))
3998                         return buf_b + len_b; /* No PSB in buf_b => no data */
3999         }
4000 }
4001
4002 /**
4003  * intel_pt_find_overlap - determine start of non-overlapped trace data.
4004  * @buf_a: first buffer
4005  * @len_a: size of first buffer
4006  * @buf_b: second buffer
4007  * @len_b: size of second buffer
4008  * @have_tsc: can use TSC packets to detect overlap
4009  * @consecutive: returns true if there is data in buf_b that is consecutive
4010  *               to buf_a
4011  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4012  *
4013  * When trace samples or snapshots are recorded there is the possibility that
4014  * the data overlaps.  Note that, for the purposes of decoding, data is only
4015  * useful if it begins with a PSB packet.
4016  *
4017  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4018  * @buf_b + @len_b if there is no non-overlapped data.
4019  */
4020 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
4021                                      unsigned char *buf_b, size_t len_b,
4022                                      bool have_tsc, bool *consecutive,
4023                                      bool ooo_tsc)
4024 {
4025         unsigned char *found;
4026
4027         /* Buffer 'b' must start at PSB so throw away everything before that */
4028         if (!intel_pt_next_psb(&buf_b, &len_b))
4029                 return buf_b + len_b; /* No PSB */
4030
4031         if (!intel_pt_next_psb(&buf_a, &len_a))
4032                 return buf_b; /* No overlap */
4033
4034         if (have_tsc) {
4035                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4036                                                   consecutive, ooo_tsc);
4037                 if (found)
4038                         return found;
4039         }
4040
4041         /*
4042          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4043          * we can ignore the first part of buffer 'a'.
4044          */
4045         while (len_b < len_a) {
4046                 if (!intel_pt_step_psb(&buf_a, &len_a))
4047                         return buf_b; /* No overlap */
4048         }
4049
4050         /* Now len_b >= len_a */
4051         while (1) {
4052                 /* Potential overlap so check the bytes */
4053                 found = memmem(buf_a, len_a, buf_b, len_a);
4054                 if (found) {
4055                         *consecutive = true;
4056                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
4057                 }
4058
4059                 /* Try again at next PSB in buffer 'a' */
4060                 if (!intel_pt_step_psb(&buf_a, &len_a))
4061                         return buf_b; /* No overlap */
4062         }
4063 }
4064
4065 /**
4066  * struct fast_forward_data - data used by intel_pt_ff_cb().
4067  * @timestamp: timestamp to fast forward towards
4068  * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4069  *                 the fast forward timestamp.
4070  */
4071 struct fast_forward_data {
4072         uint64_t timestamp;
4073         uint64_t buf_timestamp;
4074 };
4075
4076 /**
4077  * intel_pt_ff_cb - fast forward lookahead callback.
4078  * @buffer: Intel PT trace buffer
4079  * @data: opaque pointer to fast forward data (struct fast_forward_data)
4080  *
4081  * Determine if @buffer trace is past the fast forward timestamp.
4082  *
4083  * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4084  *         timestamp, and 0 otherwise.
4085  */
4086 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
4087 {
4088         struct fast_forward_data *d = data;
4089         unsigned char *buf;
4090         uint64_t tsc;
4091         size_t rem;
4092         size_t len;
4093
4094         buf = (unsigned char *)buffer->buf;
4095         len = buffer->len;
4096
4097         if (!intel_pt_next_psb(&buf, &len) ||
4098             !intel_pt_next_tsc(buf, len, &tsc, &rem))
4099                 return 0;
4100
4101         tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4102
4103         intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4104                      tsc, buffer->ref_timestamp);
4105
4106         /*
4107          * If the buffer contains a timestamp earlier that the fast forward
4108          * timestamp, then record it, else stop.
4109          */
4110         if (tsc < d->timestamp)
4111                 d->buf_timestamp = buffer->ref_timestamp;
4112         else
4113                 return 1;
4114
4115         return 0;
4116 }
4117
4118 /**
4119  * intel_pt_fast_forward - reposition decoder forwards.
4120  * @decoder: Intel PT decoder
4121  * @timestamp: timestamp to fast forward towards
4122  *
4123  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4124  *
4125  * Return: 0 on success or negative error code on failure.
4126  */
4127 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4128 {
4129         struct fast_forward_data d = { .timestamp = timestamp };
4130         unsigned char *buf;
4131         size_t len;
4132         int err;
4133
4134         intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4135
4136         /* Find buffer timestamp of buffer to fast forward to */
4137         err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4138         if (err < 0)
4139                 return err;
4140
4141         /* Walk to buffer with same buffer timestamp */
4142         if (d.buf_timestamp) {
4143                 do {
4144                         decoder->pos += decoder->len;
4145                         decoder->len = 0;
4146                         err = intel_pt_get_next_data(decoder, true);
4147                         /* -ENOLINK means non-consecutive trace */
4148                         if (err && err != -ENOLINK)
4149                                 return err;
4150                 } while (decoder->buf_timestamp != d.buf_timestamp);
4151         }
4152
4153         if (!decoder->buf)
4154                 return 0;
4155
4156         buf = (unsigned char *)decoder->buf;
4157         len = decoder->len;
4158
4159         if (!intel_pt_next_psb(&buf, &len))
4160                 return 0;
4161
4162         /*
4163          * Walk PSBs while the PSB timestamp is less than the fast forward
4164          * timestamp.
4165          */
4166         do {
4167                 uint64_t tsc;
4168                 size_t rem;
4169
4170                 if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4171                         break;
4172                 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4173                 /*
4174                  * A TSC packet can slip past MTC packets but, after fast
4175                  * forward, decoding starts at the TSC timestamp. That means
4176                  * the timestamps may not be exactly the same as the timestamps
4177                  * that would have been decoded without fast forward.
4178                  */
4179                 if (tsc < timestamp) {
4180                         intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4181                         decoder->pos += decoder->len - len;
4182                         decoder->buf = buf;
4183                         decoder->len = len;
4184                         intel_pt_reposition(decoder);
4185                 } else {
4186                         break;
4187                 }
4188         } while (intel_pt_step_psb(&buf, &len));
4189
4190         return 0;
4191 }