tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25
  26 #include "../cache.h"
  27 #include "../util.h"
  28
  29 #include "intel-pt-insn-decoder.h"
  30 #include "intel-pt-pkt-decoder.h"
  31 #include "intel-pt-decoder.h"
  32 #include "intel-pt-log.h"
  33
  34 #define INTEL_PT_BLK_SIZE 1024
  35
  36 #define BIT63 (((uint64_t)1 << 63))
  37
  38 #define INTEL_PT_RETURN 1
  39
  40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  41 #define INTEL_PT_MAX_LOOPS 10000
  42
  43 struct intel_pt_blk {
  44         struct intel_pt_blk *prev;
  45         uint64_t ip[INTEL_PT_BLK_SIZE];
  46 };
  47
  48 struct intel_pt_stack {
  49         struct intel_pt_blk *blk;
  50         struct intel_pt_blk *spare;
  51         int pos;
  52 };
  53
  54 enum intel_pt_pkt_state {
  55         INTEL_PT_STATE_NO_PSB,
  56         INTEL_PT_STATE_NO_IP,
  57         INTEL_PT_STATE_ERR_RESYNC,
  58         INTEL_PT_STATE_IN_SYNC,
  59         INTEL_PT_STATE_TNT,
  60         INTEL_PT_STATE_TIP,
  61         INTEL_PT_STATE_TIP_PGD,
  62         INTEL_PT_STATE_FUP,
  63         INTEL_PT_STATE_FUP_NO_TIP,
  64 };
  65
  66 #ifdef INTEL_PT_STRICT
  67 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  68 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  69 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  70 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  71 #else
  72 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  73 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  74 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  75 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  76 #endif
  77
  78 struct intel_pt_decoder {
  79         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
  80         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
  81                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
  82                          uint64_t max_insn_cnt, void *data);
  83         bool (*pgd_ip)(uint64_t ip, void *data);
  84         void *data;
  85         struct intel_pt_state state;
  86         const unsigned char *buf;
  87         size_t len;
  88         bool return_compression;
  89         bool mtc_insn;
  90         bool pge;
  91         bool have_tma;
  92         bool have_cyc;
  93         uint64_t pos;
  94         uint64_t last_ip;
  95         uint64_t ip;
  96         uint64_t cr3;
  97         uint64_t timestamp;
  98         uint64_t tsc_timestamp;
  99         uint64_t ref_timestamp;
 100         uint64_t ret_addr;
 101         uint64_t ctc_timestamp;
 102         uint64_t ctc_delta;
 103         uint64_t cycle_cnt;
 104         uint64_t cyc_ref_timestamp;
 105         uint32_t last_mtc;
 106         uint32_t tsc_ctc_ratio_n;
 107         uint32_t tsc_ctc_ratio_d;
 108         uint32_t tsc_ctc_mult;
 109         uint32_t tsc_slip;
 110         uint32_t ctc_rem_mask;
 111         int mtc_shift;
 112         struct intel_pt_stack stack;
 113         enum intel_pt_pkt_state pkt_state;
 114         struct intel_pt_pkt packet;
 115         struct intel_pt_pkt tnt;
 116         int pkt_step;
 117         int pkt_len;
 118         int last_packet_type;
 119         unsigned int cbr;
 120         unsigned int max_non_turbo_ratio;
 121         double max_non_turbo_ratio_fp;
 122         double cbr_cyc_to_tsc;
 123         double calc_cyc_to_tsc;
 124         bool have_calc_cyc_to_tsc;
 125         int exec_mode;
 126         unsigned int insn_bytes;
 127         uint64_t period;
 128         enum intel_pt_period_type period_type;
 129         uint64_t tot_insn_cnt;
 130         uint64_t period_insn_cnt;
 131         uint64_t period_mask;
 132         uint64_t period_ticks;
 133         uint64_t last_masked_timestamp;
 134         bool continuous_period;
 135         bool overflow;
 136         bool set_fup_tx_flags;
 137         unsigned int fup_tx_flags;
 138         unsigned int tx_flags;
 139         uint64_t timestamp_insn_cnt;
 140         uint64_t stuck_ip;
 141         int no_progress;
 142         int stuck_ip_prd;
 143         int stuck_ip_cnt;
 144         const unsigned char *next_buf;
 145         size_t next_len;
 146         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 147 };
 148
 149 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 150 {
 151         int i;
 152
 153         for (i = 0; x != 1; i++)
 154                 x >>= 1;
 155
 156         return x << i;
 157 }
 158
 159 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 160 {
 161         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 162                 uint64_t period;
 163
 164                 period = intel_pt_lower_power_of_2(decoder->period);
 165                 decoder->period_mask  = ~(period - 1);
 166                 decoder->period_ticks = period;
 167         }
 168 }
 169
 170 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 171 {
 172         if (!d)
 173                 return 0;
 174         return (t / d) * n + ((t % d) * n) / d;
 175 }
 176
 177 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 178 {
 179         struct intel_pt_decoder *decoder;
 180
 181         if (!params->get_trace || !params->walk_insn)
 182                 return NULL;
 183
 184         decoder = zalloc(sizeof(struct intel_pt_decoder));
 185         if (!decoder)
 186                 return NULL;
 187
 188         decoder->get_trace          = params->get_trace;
 189         decoder->walk_insn          = params->walk_insn;
 190         decoder->pgd_ip             = params->pgd_ip;
 191         decoder->data               = params->data;
 192         decoder->return_compression = params->return_compression;
 193
 194         decoder->period             = params->period;
 195         decoder->period_type        = params->period_type;
 196
 197         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 198         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 199
 200         intel_pt_setup_period(decoder);
 201
 202         decoder->mtc_shift = params->mtc_period;
 203         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 204
 205         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 206         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 207
 208         if (!decoder->tsc_ctc_ratio_n)
 209                 decoder->tsc_ctc_ratio_d = 0;
 210
 211         if (decoder->tsc_ctc_ratio_d) {
 212                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 213                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 214                                                 decoder->tsc_ctc_ratio_d;
 215
 216                 /*
 217                  * Allow for timestamps appearing to backwards because a TSC
 218                  * packet has slipped past a MTC packet, so allow 2 MTC ticks
 219                  * or ...
 220                  */
 221                 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
 222                                         decoder->tsc_ctc_ratio_n,
 223                                         decoder->tsc_ctc_ratio_d);
 224         }
 225         /* ... or 0x100 paranoia */
 226         if (decoder->tsc_slip < 0x100)
 227                 decoder->tsc_slip = 0x100;
 228
 229         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 230         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 231         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 232         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 233         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 234
 235         return decoder;
 236 }
 237
 238 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 239 {
 240         struct intel_pt_blk *blk = stack->blk;
 241
 242         stack->blk = blk->prev;
 243         if (!stack->spare)
 244                 stack->spare = blk;
 245         else
 246                 free(blk);
 247 }
 248
 249 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 250 {
 251         if (!stack->pos) {
 252                 if (!stack->blk)
 253                         return 0;
 254                 intel_pt_pop_blk(stack);
 255                 if (!stack->blk)
 256                         return 0;
 257                 stack->pos = INTEL_PT_BLK_SIZE;
 258         }
 259         return stack->blk->ip[--stack->pos];
 260 }
 261
 262 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 263 {
 264         struct intel_pt_blk *blk;
 265
 266         if (stack->spare) {
 267                 blk = stack->spare;
 268                 stack->spare = NULL;
 269         } else {
 270                 blk = malloc(sizeof(struct intel_pt_blk));
 271                 if (!blk)
 272                         return -ENOMEM;
 273         }
 274
 275         blk->prev = stack->blk;
 276         stack->blk = blk;
 277         stack->pos = 0;
 278         return 0;
 279 }
 280
 281 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 282 {
 283         int err;
 284
 285         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 286                 err = intel_pt_alloc_blk(stack);
 287                 if (err)
 288                         return err;
 289         }
 290
 291         stack->blk->ip[stack->pos++] = ip;
 292         return 0;
 293 }
 294
 295 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 296 {
 297         while (stack->blk)
 298                 intel_pt_pop_blk(stack);
 299         stack->pos = 0;
 300 }
 301
 302 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 303 {
 304         intel_pt_clear_stack(stack);
 305         zfree(&stack->blk);
 306         zfree(&stack->spare);
 307 }
 308
 309 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 310 {
 311         intel_pt_free_stack(&decoder->stack);
 312         free(decoder);
 313 }
 314
 315 static int intel_pt_ext_err(int code)
 316 {
 317         switch (code) {
 318         case -ENOMEM:
 319                 return INTEL_PT_ERR_NOMEM;
 320         case -ENOSYS:
 321                 return INTEL_PT_ERR_INTERN;
 322         case -EBADMSG:
 323                 return INTEL_PT_ERR_BADPKT;
 324         case -ENODATA:
 325                 return INTEL_PT_ERR_NODATA;
 326         case -EILSEQ:
 327                 return INTEL_PT_ERR_NOINSN;
 328         case -ENOENT:
 329                 return INTEL_PT_ERR_MISMAT;
 330         case -EOVERFLOW:
 331                 return INTEL_PT_ERR_OVR;
 332         case -ENOSPC:
 333                 return INTEL_PT_ERR_LOST;
 334         case -ELOOP:
 335                 return INTEL_PT_ERR_NELOOP;
 336         default:
 337                 return INTEL_PT_ERR_UNK;
 338         }
 339 }
 340
 341 static const char *intel_pt_err_msgs[] = {
 342         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 343         [INTEL_PT_ERR_INTERN] = "Internal error",
 344         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 345         [INTEL_PT_ERR_NODATA] = "No more data",
 346         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 347         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 348         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 349         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 350         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 351         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 352 };
 353
 354 int intel_pt__strerror(int code, char *buf, size_t buflen)
 355 {
 356         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 357                 code = INTEL_PT_ERR_UNK;
 358         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 359         return 0;
 360 }
 361
 362 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 363                                  uint64_t last_ip)
 364 {
 365         uint64_t ip;
 366
 367         switch (packet->count) {
 368         case 1:
 369                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 370                      packet->payload;
 371                 break;
 372         case 2:
 373                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 374                      packet->payload;
 375                 break;
 376         case 3:
 377                 ip = packet->payload;
 378                 /* Sign-extend 6-byte ip */
 379                 if (ip & (uint64_t)0x800000000000ULL)
 380                         ip |= (uint64_t)0xffff000000000000ULL;
 381                 break;
 382         case 4:
 383                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 384                      packet->payload;
 385                 break;
 386         case 6:
 387                 ip = packet->payload;
 388                 break;
 389         default:
 390                 return 0;
 391         }
 392
 393         return ip;
 394 }
 395
 396 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 397 {
 398         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 399 }
 400
 401 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 402 {
 403         intel_pt_set_last_ip(decoder);
 404         decoder->ip = decoder->last_ip;
 405 }
 406
 407 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 408 {
 409         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 410                             decoder->buf);
 411 }
 412
 413 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 414 {
 415         intel_pt_log("ERROR: Internal error\n");
 416         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 417         return -ENOSYS;
 418 }
 419
 420 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 421 {
 422         decoder->tx_flags = 0;
 423 }
 424
 425 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 426 {
 427         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 428 }
 429
 430 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 431 {
 432         intel_pt_clear_tx_flags(decoder);
 433         decoder->have_tma = false;
 434         decoder->pkt_len = 1;
 435         decoder->pkt_step = 1;
 436         intel_pt_decoder_log_packet(decoder);
 437         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 438                 intel_pt_log("ERROR: Bad packet\n");
 439                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 440         }
 441         return -EBADMSG;
 442 }
 443
 444 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 445 {
 446         struct intel_pt_buffer buffer = { .buf = 0, };
 447         int ret;
 448
 449         decoder->pkt_step = 0;
 450
 451         intel_pt_log("Getting more data\n");
 452         ret = decoder->get_trace(&buffer, decoder->data);
 453         if (ret)
 454                 return ret;
 455         decoder->buf = buffer.buf;
 456         decoder->len = buffer.len;
 457         if (!decoder->len) {
 458                 intel_pt_log("No more data\n");
 459                 return -ENODATA;
 460         }
 461         if (!buffer.consecutive) {
 462                 decoder->ip = 0;
 463                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 464                 decoder->ref_timestamp = buffer.ref_timestamp;
 465                 decoder->timestamp = 0;
 466                 decoder->have_tma = false;
 467                 decoder->state.trace_nr = buffer.trace_nr;
 468                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 469                              decoder->ref_timestamp);
 470                 return -ENOLINK;
 471         }
 472
 473         return 0;
 474 }
 475
 476 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 477 {
 478         if (!decoder->next_buf)
 479                 return intel_pt_get_data(decoder);
 480
 481         decoder->buf = decoder->next_buf;
 482         decoder->len = decoder->next_len;
 483         decoder->next_buf = 0;
 484         decoder->next_len = 0;
 485         return 0;
 486 }
 487
 488 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 489 {
 490         unsigned char *buf = decoder->temp_buf;
 491         size_t old_len, len, n;
 492         int ret;
 493
 494         old_len = decoder->len;
 495         len = decoder->len;
 496         memcpy(buf, decoder->buf, len);
 497
 498         ret = intel_pt_get_data(decoder);
 499         if (ret) {
 500                 decoder->pos += old_len;
 501                 return ret < 0 ? ret : -EINVAL;
 502         }
 503
 504         n = INTEL_PT_PKT_MAX_SZ - len;
 505         if (n > decoder->len)
 506                 n = decoder->len;
 507         memcpy(buf + len, decoder->buf, n);
 508         len += n;
 509
 510         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 511         if (ret < (int)old_len) {
 512                 decoder->next_buf = decoder->buf;
 513                 decoder->next_len = decoder->len;
 514                 decoder->buf = buf;
 515                 decoder->len = old_len;
 516                 return intel_pt_bad_packet(decoder);
 517         }
 518
 519         decoder->next_buf = decoder->buf + (ret - old_len);
 520         decoder->next_len = decoder->len - (ret - old_len);
 521
 522         decoder->buf = buf;
 523         decoder->len = ret;
 524
 525         return ret;
 526 }
 527
 528 struct intel_pt_pkt_info {
 529         struct intel_pt_decoder   *decoder;
 530         struct intel_pt_pkt       packet;
 531         uint64_t                  pos;
 532         int                       pkt_len;
 533         int                       last_packet_type;
 534         void                      *data;
 535 };
 536
 537 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 538
 539 /* Lookahead packets in current buffer */
 540 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 541                                   intel_pt_pkt_cb_t cb, void *data)
 542 {
 543         struct intel_pt_pkt_info pkt_info;
 544         const unsigned char *buf = decoder->buf;
 545         size_t len = decoder->len;
 546         int ret;
 547
 548         pkt_info.decoder          = decoder;
 549         pkt_info.pos              = decoder->pos;
 550         pkt_info.pkt_len          = decoder->pkt_step;
 551         pkt_info.last_packet_type = decoder->last_packet_type;
 552         pkt_info.data             = data;
 553
 554         while (1) {
 555                 do {
 556                         pkt_info.pos += pkt_info.pkt_len;
 557                         buf          += pkt_info.pkt_len;
 558                         len          -= pkt_info.pkt_len;
 559
 560                         if (!len)
 561                                 return INTEL_PT_NEED_MORE_BYTES;
 562
 563                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 564                         if (!ret)
 565                                 return INTEL_PT_NEED_MORE_BYTES;
 566                         if (ret < 0)
 567                                 return ret;
 568
 569                         pkt_info.pkt_len = ret;
 570                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 571
 572                 ret = cb(&pkt_info);
 573                 if (ret)
 574                         return 0;
 575
 576                 pkt_info.last_packet_type = pkt_info.packet.type;
 577         }
 578 }
 579
 580 struct intel_pt_calc_cyc_to_tsc_info {
 581         uint64_t        cycle_cnt;
 582         unsigned int    cbr;
 583         uint32_t        last_mtc;
 584         uint64_t        ctc_timestamp;
 585         uint64_t        ctc_delta;
 586         uint64_t        tsc_timestamp;
 587         uint64_t        timestamp;
 588         bool            have_tma;
 589         bool            from_mtc;
 590         double          cbr_cyc_to_tsc;
 591 };
 592
 593 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 594 {
 595         struct intel_pt_decoder *decoder = pkt_info->decoder;
 596         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 597         uint64_t timestamp;
 598         double cyc_to_tsc;
 599         unsigned int cbr;
 600         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 601
 602         switch (pkt_info->packet.type) {
 603         case INTEL_PT_TNT:
 604         case INTEL_PT_TIP_PGE:
 605         case INTEL_PT_TIP:
 606         case INTEL_PT_FUP:
 607         case INTEL_PT_PSB:
 608         case INTEL_PT_PIP:
 609         case INTEL_PT_MODE_EXEC:
 610         case INTEL_PT_MODE_TSX:
 611         case INTEL_PT_PSBEND:
 612         case INTEL_PT_PAD:
 613         case INTEL_PT_VMCS:
 614         case INTEL_PT_MNT:
 615                 return 0;
 616
 617         case INTEL_PT_MTC:
 618                 if (!data->have_tma)
 619                         return 0;
 620
 621                 mtc = pkt_info->packet.payload;
 622                 if (mtc > data->last_mtc)
 623                         mtc_delta = mtc - data->last_mtc;
 624                 else
 625                         mtc_delta = mtc + 256 - data->last_mtc;
 626                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 627                 data->last_mtc = mtc;
 628
 629                 if (decoder->tsc_ctc_mult) {
 630                         timestamp = data->ctc_timestamp +
 631                                 data->ctc_delta * decoder->tsc_ctc_mult;
 632                 } else {
 633                         timestamp = data->ctc_timestamp +
 634                                 multdiv(data->ctc_delta,
 635                                         decoder->tsc_ctc_ratio_n,
 636                                         decoder->tsc_ctc_ratio_d);
 637                 }
 638
 639                 if (timestamp < data->timestamp)
 640                         return 1;
 641
 642                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 643                         data->timestamp = timestamp;
 644                         return 0;
 645                 }
 646
 647                 break;
 648
 649         case INTEL_PT_TSC:
 650                 timestamp = pkt_info->packet.payload |
 651                             (data->timestamp & (0xffULL << 56));
 652                 if (data->from_mtc && timestamp < data->timestamp &&
 653                     data->timestamp - timestamp < decoder->tsc_slip)
 654                         return 1;
 655                 if (timestamp < data->timestamp)
 656                         timestamp += (1ULL << 56);
 657                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 658                         if (data->from_mtc)
 659                                 return 1;
 660                         data->tsc_timestamp = timestamp;
 661                         data->timestamp = timestamp;
 662                         return 0;
 663                 }
 664                 break;
 665
 666         case INTEL_PT_TMA:
 667                 if (data->from_mtc)
 668                         return 1;
 669
 670                 if (!decoder->tsc_ctc_ratio_d)
 671                         return 0;
 672
 673                 ctc = pkt_info->packet.payload;
 674                 fc = pkt_info->packet.count;
 675                 ctc_rem = ctc & decoder->ctc_rem_mask;
 676
 677                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 678
 679                 data->ctc_timestamp = data->tsc_timestamp - fc;
 680                 if (decoder->tsc_ctc_mult) {
 681                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 682                 } else {
 683                         data->ctc_timestamp -=
 684                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 685                                         decoder->tsc_ctc_ratio_d);
 686                 }
 687
 688                 data->ctc_delta = 0;
 689                 data->have_tma = true;
 690
 691                 return 0;
 692
 693         case INTEL_PT_CYC:
 694                 data->cycle_cnt += pkt_info->packet.payload;
 695                 return 0;
 696
 697         case INTEL_PT_CBR:
 698                 cbr = pkt_info->packet.payload;
 699                 if (data->cbr && data->cbr != cbr)
 700                         return 1;
 701                 data->cbr = cbr;
 702                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 703                 return 0;
 704
 705         case INTEL_PT_TIP_PGD:
 706         case INTEL_PT_TRACESTOP:
 707         case INTEL_PT_OVF:
 708         case INTEL_PT_BAD: /* Does not happen */
 709         default:
 710                 return 1;
 711         }
 712
 713         if (!data->cbr && decoder->cbr) {
 714                 data->cbr = decoder->cbr;
 715                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 716         }
 717
 718         if (!data->cycle_cnt)
 719                 return 1;
 720
 721         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 722
 723         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 724             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 725                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 726                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 727                 return 1;
 728         }
 729
 730         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 731         decoder->have_calc_cyc_to_tsc = true;
 732
 733         if (data->cbr) {
 734                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 735                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 736         } else {
 737                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 738                              cyc_to_tsc, pkt_info->pos);
 739         }
 740
 741         return 1;
 742 }
 743
 744 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 745                                      bool from_mtc)
 746 {
 747         struct intel_pt_calc_cyc_to_tsc_info data = {
 748                 .cycle_cnt      = 0,
 749                 .cbr            = 0,
 750                 .last_mtc       = decoder->last_mtc,
 751                 .ctc_timestamp  = decoder->ctc_timestamp,
 752                 .ctc_delta      = decoder->ctc_delta,
 753                 .tsc_timestamp  = decoder->tsc_timestamp,
 754                 .timestamp      = decoder->timestamp,
 755                 .have_tma       = decoder->have_tma,
 756                 .from_mtc       = from_mtc,
 757                 .cbr_cyc_to_tsc = 0,
 758         };
 759
 760         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 761 }
 762
 763 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 764 {
 765         int ret;
 766
 767         decoder->last_packet_type = decoder->packet.type;
 768
 769         do {
 770                 decoder->pos += decoder->pkt_step;
 771                 decoder->buf += decoder->pkt_step;
 772                 decoder->len -= decoder->pkt_step;
 773
 774                 if (!decoder->len) {
 775                         ret = intel_pt_get_next_data(decoder);
 776                         if (ret)
 777                                 return ret;
 778                 }
 779
 780                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 781                                           &decoder->packet);
 782                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 783                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 784                         ret = intel_pt_get_split_packet(decoder);
 785                         if (ret < 0)
 786                                 return ret;
 787                 }
 788                 if (ret <= 0)
 789                         return intel_pt_bad_packet(decoder);
 790
 791                 decoder->pkt_len = ret;
 792                 decoder->pkt_step = ret;
 793                 intel_pt_decoder_log_packet(decoder);
 794         } while (decoder->packet.type == INTEL_PT_PAD);
 795
 796         return 0;
 797 }
 798
 799 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 800 {
 801         uint64_t timestamp, masked_timestamp;
 802
 803         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 804         masked_timestamp = timestamp & decoder->period_mask;
 805         if (decoder->continuous_period) {
 806                 if (masked_timestamp != decoder->last_masked_timestamp)
 807                         return 1;
 808         } else {
 809                 timestamp += 1;
 810                 masked_timestamp = timestamp & decoder->period_mask;
 811                 if (masked_timestamp != decoder->last_masked_timestamp) {
 812                         decoder->last_masked_timestamp = masked_timestamp;
 813                         decoder->continuous_period = true;
 814                 }
 815         }
 816         return decoder->period_ticks - (timestamp - masked_timestamp);
 817 }
 818
 819 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 820 {
 821         switch (decoder->period_type) {
 822         case INTEL_PT_PERIOD_INSTRUCTIONS:
 823                 return decoder->period - decoder->period_insn_cnt;
 824         case INTEL_PT_PERIOD_TICKS:
 825                 return intel_pt_next_period(decoder);
 826         case INTEL_PT_PERIOD_NONE:
 827         case INTEL_PT_PERIOD_MTC:
 828         default:
 829                 return 0;
 830         }
 831 }
 832
 833 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 834 {
 835         uint64_t timestamp, masked_timestamp;
 836
 837         switch (decoder->period_type) {
 838         case INTEL_PT_PERIOD_INSTRUCTIONS:
 839                 decoder->period_insn_cnt = 0;
 840                 break;
 841         case INTEL_PT_PERIOD_TICKS:
 842                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 843                 masked_timestamp = timestamp & decoder->period_mask;
 844                 decoder->last_masked_timestamp = masked_timestamp;
 845                 break;
 846         case INTEL_PT_PERIOD_NONE:
 847         case INTEL_PT_PERIOD_MTC:
 848         default:
 849                 break;
 850         }
 851
 852         decoder->state.type |= INTEL_PT_INSTRUCTION;
 853 }
 854
 855 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 856                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 857 {
 858         uint64_t max_insn_cnt, insn_cnt = 0;
 859         int err;
 860
 861         if (!decoder->mtc_insn)
 862                 decoder->mtc_insn = true;
 863
 864         max_insn_cnt = intel_pt_next_sample(decoder);
 865
 866         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 867                                  max_insn_cnt, decoder->data);
 868
 869         decoder->tot_insn_cnt += insn_cnt;
 870         decoder->timestamp_insn_cnt += insn_cnt;
 871         decoder->period_insn_cnt += insn_cnt;
 872
 873         if (err) {
 874                 decoder->no_progress = 0;
 875                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 876                 intel_pt_log_at("ERROR: Failed to get instruction",
 877                                 decoder->ip);
 878                 if (err == -ENOENT)
 879                         return -ENOLINK;
 880                 return -EILSEQ;
 881         }
 882
 883         if (ip && decoder->ip == ip) {
 884                 err = -EAGAIN;
 885                 goto out;
 886         }
 887
 888         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 889                 intel_pt_sample_insn(decoder);
 890
 891         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 892                 decoder->state.type = INTEL_PT_INSTRUCTION;
 893                 decoder->state.from_ip = decoder->ip;
 894                 decoder->state.to_ip = 0;
 895                 decoder->ip += intel_pt_insn->length;
 896                 err = INTEL_PT_RETURN;
 897                 goto out;
 898         }
 899
 900         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 901                 /* Zero-length calls are excluded */
 902                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 903                     intel_pt_insn->rel) {
 904                         err = intel_pt_push(&decoder->stack, decoder->ip +
 905                                             intel_pt_insn->length);
 906                         if (err)
 907                                 goto out;
 908                 }
 909         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
 910                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
 911         }
 912
 913         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
 914                 int cnt = decoder->no_progress++;
 915
 916                 decoder->state.from_ip = decoder->ip;
 917                 decoder->ip += intel_pt_insn->length +
 918                                 intel_pt_insn->rel;
 919                 decoder->state.to_ip = decoder->ip;
 920                 err = INTEL_PT_RETURN;
 921
 922                 /*
 923                  * Check for being stuck in a loop.  This can happen if a
 924                  * decoder error results in the decoder erroneously setting the
 925                  * ip to an address that is itself in an infinite loop that
 926                  * consumes no packets.  When that happens, there must be an
 927                  * unconditional branch.
 928                  */
 929                 if (cnt) {
 930                         if (cnt == 1) {
 931                                 decoder->stuck_ip = decoder->state.to_ip;
 932                                 decoder->stuck_ip_prd = 1;
 933                                 decoder->stuck_ip_cnt = 1;
 934                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
 935                                    decoder->state.to_ip == decoder->stuck_ip) {
 936                                 intel_pt_log_at("ERROR: Never-ending loop",
 937                                                 decoder->state.to_ip);
 938                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 939                                 err = -ELOOP;
 940                                 goto out;
 941                         } else if (!--decoder->stuck_ip_cnt) {
 942                                 decoder->stuck_ip_prd += 1;
 943                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
 944                                 decoder->stuck_ip = decoder->state.to_ip;
 945                         }
 946                 }
 947                 goto out_no_progress;
 948         }
 949 out:
 950         decoder->no_progress = 0;
 951 out_no_progress:
 952         decoder->state.insn_op = intel_pt_insn->op;
 953         decoder->state.insn_len = intel_pt_insn->length;
 954
 955         if (decoder->tx_flags & INTEL_PT_IN_TX)
 956                 decoder->state.flags |= INTEL_PT_IN_TX;
 957
 958         return err;
 959 }
 960
 961 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
 962 {
 963         struct intel_pt_insn intel_pt_insn;
 964         uint64_t ip;
 965         int err;
 966
 967         ip = decoder->last_ip;
 968
 969         while (1) {
 970                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
 971                 if (err == INTEL_PT_RETURN)
 972                         return 0;
 973                 if (err == -EAGAIN) {
 974                         if (decoder->set_fup_tx_flags) {
 975                                 decoder->set_fup_tx_flags = false;
 976                                 decoder->tx_flags = decoder->fup_tx_flags;
 977                                 decoder->state.type = INTEL_PT_TRANSACTION;
 978                                 decoder->state.from_ip = decoder->ip;
 979                                 decoder->state.to_ip = 0;
 980                                 decoder->state.flags = decoder->fup_tx_flags;
 981                                 return 0;
 982                         }
 983                         return err;
 984                 }
 985                 decoder->set_fup_tx_flags = false;
 986                 if (err)
 987                         return err;
 988
 989                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
 990                         intel_pt_log_at("ERROR: Unexpected indirect branch",
 991                                         decoder->ip);
 992                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 993                         return -ENOENT;
 994                 }
 995
 996                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
 997                         intel_pt_log_at("ERROR: Unexpected conditional branch",
 998                                         decoder->ip);
 999                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1000                         return -ENOENT;
1001                 }
1002
1003                 intel_pt_bug(decoder);
1004         }
1005 }
1006
1007 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1008 {
1009         struct intel_pt_insn intel_pt_insn;
1010         int err;
1011
1012         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1013         if (err == INTEL_PT_RETURN &&
1014             decoder->pgd_ip &&
1015             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1016             (decoder->state.type & INTEL_PT_BRANCH) &&
1017             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1018                 /* Unconditional branch leaving filter region */
1019                 decoder->no_progress = 0;
1020                 decoder->pge = false;
1021                 decoder->continuous_period = false;
1022                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1023                 decoder->state.to_ip = 0;
1024                 return 0;
1025         }
1026         if (err == INTEL_PT_RETURN)
1027                 return 0;
1028         if (err)
1029                 return err;
1030
1031         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1032                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1033                         decoder->pge = false;
1034                         decoder->continuous_period = false;
1035                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1036                         decoder->state.from_ip = decoder->ip;
1037                         decoder->state.to_ip = 0;
1038                         if (decoder->packet.count != 0)
1039                                 decoder->ip = decoder->last_ip;
1040                 } else {
1041                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1042                         decoder->state.from_ip = decoder->ip;
1043                         if (decoder->packet.count == 0) {
1044                                 decoder->state.to_ip = 0;
1045                         } else {
1046                                 decoder->state.to_ip = decoder->last_ip;
1047                                 decoder->ip = decoder->last_ip;
1048                         }
1049                 }
1050                 return 0;
1051         }
1052
1053         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1054                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1055                                  intel_pt_insn.rel;
1056
1057                 if (decoder->pgd_ip &&
1058                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1059                     decoder->pgd_ip(to_ip, decoder->data)) {
1060                         /* Conditional branch leaving filter region */
1061                         decoder->pge = false;
1062                         decoder->continuous_period = false;
1063                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1064                         decoder->ip = to_ip;
1065                         decoder->state.from_ip = decoder->ip;
1066                         decoder->state.to_ip = 0;
1067                         return 0;
1068                 }
1069                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1070                                 decoder->ip);
1071                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1072                 return -ENOENT;
1073         }
1074
1075         return intel_pt_bug(decoder);
1076 }
1077
1078 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1079 {
1080         struct intel_pt_insn intel_pt_insn;
1081         int err;
1082
1083         while (1) {
1084                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1085                 if (err == INTEL_PT_RETURN)
1086                         return 0;
1087                 if (err)
1088                         return err;
1089
1090                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1091                         if (!decoder->return_compression) {
1092                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1093                                                 decoder->ip);
1094                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1095                                 return -ENOENT;
1096                         }
1097                         if (!decoder->ret_addr) {
1098                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1099                                                 decoder->ip);
1100                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1101                                 return -ENOENT;
1102                         }
1103                         if (!(decoder->tnt.payload & BIT63)) {
1104                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1105                                                 decoder->ip);
1106                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1107                                 return -ENOENT;
1108                         }
1109                         decoder->tnt.count -= 1;
1110                         if (!decoder->tnt.count)
1111                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1112                         decoder->tnt.payload <<= 1;
1113                         decoder->state.from_ip = decoder->ip;
1114                         decoder->ip = decoder->ret_addr;
1115                         decoder->state.to_ip = decoder->ip;
1116                         return 0;
1117                 }
1118
1119                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1120                         /* Handle deferred TIPs */
1121                         err = intel_pt_get_next_packet(decoder);
1122                         if (err)
1123                                 return err;
1124                         if (decoder->packet.type != INTEL_PT_TIP ||
1125                             decoder->packet.count == 0) {
1126                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1127                                                 decoder->ip);
1128                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1129                                 decoder->pkt_step = 0;
1130                                 return -ENOENT;
1131                         }
1132                         intel_pt_set_last_ip(decoder);
1133                         decoder->state.from_ip = decoder->ip;
1134                         decoder->state.to_ip = decoder->last_ip;
1135                         decoder->ip = decoder->last_ip;
1136                         return 0;
1137                 }
1138
1139                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1140                         decoder->tnt.count -= 1;
1141                         if (!decoder->tnt.count)
1142                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1143                         if (decoder->tnt.payload & BIT63) {
1144                                 decoder->tnt.payload <<= 1;
1145                                 decoder->state.from_ip = decoder->ip;
1146                                 decoder->ip += intel_pt_insn.length +
1147                                                intel_pt_insn.rel;
1148                                 decoder->state.to_ip = decoder->ip;
1149                                 return 0;
1150                         }
1151                         /* Instruction sample for a non-taken branch */
1152                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1153                                 decoder->tnt.payload <<= 1;
1154                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1155                                 decoder->state.from_ip = decoder->ip;
1156                                 decoder->state.to_ip = 0;
1157                                 decoder->ip += intel_pt_insn.length;
1158                                 return 0;
1159                         }
1160                         decoder->ip += intel_pt_insn.length;
1161                         if (!decoder->tnt.count)
1162                                 return -EAGAIN;
1163                         decoder->tnt.payload <<= 1;
1164                         continue;
1165                 }
1166
1167                 return intel_pt_bug(decoder);
1168         }
1169 }
1170
1171 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1172 {
1173         unsigned int fup_tx_flags;
1174         int err;
1175
1176         fup_tx_flags = decoder->packet.payload &
1177                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1178         err = intel_pt_get_next_packet(decoder);
1179         if (err)
1180                 return err;
1181         if (decoder->packet.type == INTEL_PT_FUP) {
1182                 decoder->fup_tx_flags = fup_tx_flags;
1183                 decoder->set_fup_tx_flags = true;
1184                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1185                         *no_tip = true;
1186         } else {
1187                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1188                                 decoder->pos);
1189                 intel_pt_update_in_tx(decoder);
1190         }
1191         return 0;
1192 }
1193
1194 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1195 {
1196         uint64_t timestamp;
1197
1198         decoder->have_tma = false;
1199
1200         if (decoder->ref_timestamp) {
1201                 timestamp = decoder->packet.payload |
1202                             (decoder->ref_timestamp & (0xffULL << 56));
1203                 if (timestamp < decoder->ref_timestamp) {
1204                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1205                                 timestamp += (1ULL << 56);
1206                 } else {
1207                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1208                                 timestamp -= (1ULL << 56);
1209                 }
1210                 decoder->tsc_timestamp = timestamp;
1211                 decoder->timestamp = timestamp;
1212                 decoder->ref_timestamp = 0;
1213                 decoder->timestamp_insn_cnt = 0;
1214         } else if (decoder->timestamp) {
1215                 timestamp = decoder->packet.payload |
1216                             (decoder->timestamp & (0xffULL << 56));
1217                 decoder->tsc_timestamp = timestamp;
1218                 if (timestamp < decoder->timestamp &&
1219                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1220                         intel_pt_log_to("Suppressing backwards timestamp",
1221                                         timestamp);
1222                         timestamp = decoder->timestamp;
1223                 }
1224                 if (timestamp < decoder->timestamp) {
1225                         intel_pt_log_to("Wraparound timestamp", timestamp);
1226                         timestamp += (1ULL << 56);
1227                         decoder->tsc_timestamp = timestamp;
1228                 }
1229                 decoder->timestamp = timestamp;
1230                 decoder->timestamp_insn_cnt = 0;
1231         }
1232
1233         if (decoder->last_packet_type == INTEL_PT_CYC) {
1234                 decoder->cyc_ref_timestamp = decoder->timestamp;
1235                 decoder->cycle_cnt = 0;
1236                 decoder->have_calc_cyc_to_tsc = false;
1237                 intel_pt_calc_cyc_to_tsc(decoder, false);
1238         }
1239
1240         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1241 }
1242
1243 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1244 {
1245         intel_pt_log("ERROR: Buffer overflow\n");
1246         intel_pt_clear_tx_flags(decoder);
1247         decoder->have_tma = false;
1248         decoder->cbr = 0;
1249         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1250         decoder->overflow = true;
1251         return -EOVERFLOW;
1252 }
1253
1254 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1255 {
1256         uint32_t ctc = decoder->packet.payload;
1257         uint32_t fc = decoder->packet.count;
1258         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1259
1260         if (!decoder->tsc_ctc_ratio_d)
1261                 return;
1262
1263         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1264         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1265         if (decoder->tsc_ctc_mult) {
1266                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1267         } else {
1268                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1269                                                   decoder->tsc_ctc_ratio_n,
1270                                                   decoder->tsc_ctc_ratio_d);
1271         }
1272         decoder->ctc_delta = 0;
1273         decoder->have_tma = true;
1274         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1275                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1276 }
1277
1278 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1279 {
1280         uint64_t timestamp;
1281         uint32_t mtc, mtc_delta;
1282
1283         if (!decoder->have_tma)
1284                 return;
1285
1286         mtc = decoder->packet.payload;
1287
1288         if (mtc > decoder->last_mtc)
1289                 mtc_delta = mtc - decoder->last_mtc;
1290         else
1291                 mtc_delta = mtc + 256 - decoder->last_mtc;
1292
1293         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1294
1295         if (decoder->tsc_ctc_mult) {
1296                 timestamp = decoder->ctc_timestamp +
1297                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1298         } else {
1299                 timestamp = decoder->ctc_timestamp +
1300                             multdiv(decoder->ctc_delta,
1301                                     decoder->tsc_ctc_ratio_n,
1302                                     decoder->tsc_ctc_ratio_d);
1303         }
1304
1305         if (timestamp < decoder->timestamp)
1306                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1307                              timestamp, decoder->timestamp);
1308         else
1309                 decoder->timestamp = timestamp;
1310
1311         decoder->timestamp_insn_cnt = 0;
1312         decoder->last_mtc = mtc;
1313
1314         if (decoder->last_packet_type == INTEL_PT_CYC) {
1315                 decoder->cyc_ref_timestamp = decoder->timestamp;
1316                 decoder->cycle_cnt = 0;
1317                 decoder->have_calc_cyc_to_tsc = false;
1318                 intel_pt_calc_cyc_to_tsc(decoder, true);
1319         }
1320 }
1321
1322 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1323 {
1324         unsigned int cbr = decoder->packet.payload;
1325
1326         if (decoder->cbr == cbr)
1327                 return;
1328
1329         decoder->cbr = cbr;
1330         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1331 }
1332
1333 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1334 {
1335         uint64_t timestamp = decoder->cyc_ref_timestamp;
1336
1337         decoder->have_cyc = true;
1338
1339         decoder->cycle_cnt += decoder->packet.payload;
1340
1341         if (!decoder->cyc_ref_timestamp)
1342                 return;
1343
1344         if (decoder->have_calc_cyc_to_tsc)
1345                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1346         else if (decoder->cbr)
1347                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1348         else
1349                 return;
1350
1351         if (timestamp < decoder->timestamp)
1352                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1353                              timestamp, decoder->timestamp);
1354         else
1355                 decoder->timestamp = timestamp;
1356 }
1357
1358 /* Walk PSB+ packets when already in sync. */
1359 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1360 {
1361         int err;
1362
1363         while (1) {
1364                 err = intel_pt_get_next_packet(decoder);
1365                 if (err)
1366                         return err;
1367
1368                 switch (decoder->packet.type) {
1369                 case INTEL_PT_PSBEND:
1370                         return 0;
1371
1372                 case INTEL_PT_TIP_PGD:
1373                 case INTEL_PT_TIP_PGE:
1374                 case INTEL_PT_TIP:
1375                 case INTEL_PT_TNT:
1376                 case INTEL_PT_TRACESTOP:
1377                 case INTEL_PT_BAD:
1378                 case INTEL_PT_PSB:
1379                         decoder->have_tma = false;
1380                         intel_pt_log("ERROR: Unexpected packet\n");
1381                         return -EAGAIN;
1382
1383                 case INTEL_PT_OVF:
1384                         return intel_pt_overflow(decoder);
1385
1386                 case INTEL_PT_TSC:
1387                         intel_pt_calc_tsc_timestamp(decoder);
1388                         break;
1389
1390                 case INTEL_PT_TMA:
1391                         intel_pt_calc_tma(decoder);
1392                         break;
1393
1394                 case INTEL_PT_CBR:
1395                         intel_pt_calc_cbr(decoder);
1396                         break;
1397
1398                 case INTEL_PT_MODE_EXEC:
1399                         decoder->exec_mode = decoder->packet.payload;
1400                         break;
1401
1402                 case INTEL_PT_PIP:
1403                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1404                         break;
1405
1406                 case INTEL_PT_FUP:
1407                         decoder->pge = true;
1408                         intel_pt_set_last_ip(decoder);
1409                         break;
1410
1411                 case INTEL_PT_MODE_TSX:
1412                         intel_pt_update_in_tx(decoder);
1413                         break;
1414
1415                 case INTEL_PT_MTC:
1416                         intel_pt_calc_mtc_timestamp(decoder);
1417                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1418                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1419                         break;
1420
1421                 case INTEL_PT_CYC:
1422                 case INTEL_PT_VMCS:
1423                 case INTEL_PT_MNT:
1424                 case INTEL_PT_PAD:
1425                 default:
1426                         break;
1427                 }
1428         }
1429 }
1430
1431 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1432 {
1433         int err;
1434
1435         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1436                 decoder->tx_flags = 0;
1437                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1438                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1439         } else {
1440                 decoder->state.flags |= INTEL_PT_ASYNC;
1441         }
1442
1443         while (1) {
1444                 err = intel_pt_get_next_packet(decoder);
1445                 if (err)
1446                         return err;
1447
1448                 switch (decoder->packet.type) {
1449                 case INTEL_PT_TNT:
1450                 case INTEL_PT_FUP:
1451                 case INTEL_PT_TRACESTOP:
1452                 case INTEL_PT_PSB:
1453                 case INTEL_PT_TSC:
1454                 case INTEL_PT_TMA:
1455                 case INTEL_PT_CBR:
1456                 case INTEL_PT_MODE_TSX:
1457                 case INTEL_PT_BAD:
1458                 case INTEL_PT_PSBEND:
1459                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1460                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1461                         return -ENOENT;
1462
1463                 case INTEL_PT_OVF:
1464                         return intel_pt_overflow(decoder);
1465
1466                 case INTEL_PT_TIP_PGD:
1467                         decoder->state.from_ip = decoder->ip;
1468                         decoder->state.to_ip = 0;
1469                         if (decoder->packet.count != 0) {
1470                                 intel_pt_set_ip(decoder);
1471                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1472                                              decoder->ip);
1473                         }
1474                         decoder->pge = false;
1475                         decoder->continuous_period = false;
1476                         return 0;
1477
1478                 case INTEL_PT_TIP_PGE:
1479                         decoder->pge = true;
1480                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1481                                      decoder->ip);
1482                         decoder->state.from_ip = 0;
1483                         if (decoder->packet.count == 0) {
1484                                 decoder->state.to_ip = 0;
1485                         } else {
1486                                 intel_pt_set_ip(decoder);
1487                                 decoder->state.to_ip = decoder->ip;
1488                         }
1489                         return 0;
1490
1491                 case INTEL_PT_TIP:
1492                         decoder->state.from_ip = decoder->ip;
1493                         if (decoder->packet.count == 0) {
1494                                 decoder->state.to_ip = 0;
1495                         } else {
1496                                 intel_pt_set_ip(decoder);
1497                                 decoder->state.to_ip = decoder->ip;
1498                         }
1499                         return 0;
1500
1501                 case INTEL_PT_PIP:
1502                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1503                         break;
1504
1505                 case INTEL_PT_MTC:
1506                         intel_pt_calc_mtc_timestamp(decoder);
1507                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1508                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1509                         break;
1510
1511                 case INTEL_PT_CYC:
1512                         intel_pt_calc_cyc_timestamp(decoder);
1513                         break;
1514
1515                 case INTEL_PT_MODE_EXEC:
1516                         decoder->exec_mode = decoder->packet.payload;
1517                         break;
1518
1519                 case INTEL_PT_VMCS:
1520                 case INTEL_PT_MNT:
1521                 case INTEL_PT_PAD:
1522                         break;
1523
1524                 default:
1525                         return intel_pt_bug(decoder);
1526                 }
1527         }
1528 }
1529
1530 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1531 {
1532         bool no_tip = false;
1533         int err;
1534
1535         while (1) {
1536                 err = intel_pt_get_next_packet(decoder);
1537                 if (err)
1538                         return err;
1539 next:
1540                 switch (decoder->packet.type) {
1541                 case INTEL_PT_TNT:
1542                         if (!decoder->packet.count)
1543                                 break;
1544                         decoder->tnt = decoder->packet;
1545                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1546                         err = intel_pt_walk_tnt(decoder);
1547                         if (err == -EAGAIN)
1548                                 break;
1549                         return err;
1550
1551                 case INTEL_PT_TIP_PGD:
1552                         if (decoder->packet.count != 0)
1553                                 intel_pt_set_last_ip(decoder);
1554                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1555                         return intel_pt_walk_tip(decoder);
1556
1557                 case INTEL_PT_TIP_PGE: {
1558                         decoder->pge = true;
1559                         if (decoder->packet.count == 0) {
1560                                 intel_pt_log_at("Skipping zero TIP.PGE",
1561                                                 decoder->pos);
1562                                 break;
1563                         }
1564                         intel_pt_set_ip(decoder);
1565                         decoder->state.from_ip = 0;
1566                         decoder->state.to_ip = decoder->ip;
1567                         return 0;
1568                 }
1569
1570                 case INTEL_PT_OVF:
1571                         return intel_pt_overflow(decoder);
1572
1573                 case INTEL_PT_TIP:
1574                         if (decoder->packet.count != 0)
1575                                 intel_pt_set_last_ip(decoder);
1576                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1577                         return intel_pt_walk_tip(decoder);
1578
1579                 case INTEL_PT_FUP:
1580                         if (decoder->packet.count == 0) {
1581                                 intel_pt_log_at("Skipping zero FUP",
1582                                                 decoder->pos);
1583                                 no_tip = false;
1584                                 break;
1585                         }
1586                         intel_pt_set_last_ip(decoder);
1587                         err = intel_pt_walk_fup(decoder);
1588                         if (err != -EAGAIN) {
1589                                 if (err)
1590                                         return err;
1591                                 if (no_tip)
1592                                         decoder->pkt_state =
1593                                                 INTEL_PT_STATE_FUP_NO_TIP;
1594                                 else
1595                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1596                                 return 0;
1597                         }
1598                         if (no_tip) {
1599                                 no_tip = false;
1600                                 break;
1601                         }
1602                         return intel_pt_walk_fup_tip(decoder);
1603
1604                 case INTEL_PT_TRACESTOP:
1605                         decoder->pge = false;
1606                         decoder->continuous_period = false;
1607                         intel_pt_clear_tx_flags(decoder);
1608                         decoder->have_tma = false;
1609                         break;
1610
1611                 case INTEL_PT_PSB:
1612                         intel_pt_clear_stack(&decoder->stack);
1613                         err = intel_pt_walk_psbend(decoder);
1614                         if (err == -EAGAIN)
1615                                 goto next;
1616                         if (err)
1617                                 return err;
1618                         break;
1619
1620                 case INTEL_PT_PIP:
1621                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1622                         break;
1623
1624                 case INTEL_PT_MTC:
1625                         intel_pt_calc_mtc_timestamp(decoder);
1626                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1627                                 break;
1628                         /*
1629                          * Ensure that there has been an instruction since the
1630                          * last MTC.
1631                          */
1632                         if (!decoder->mtc_insn)
1633                                 break;
1634                         decoder->mtc_insn = false;
1635                         /* Ensure that there is a timestamp */
1636                         if (!decoder->timestamp)
1637                                 break;
1638                         decoder->state.type = INTEL_PT_INSTRUCTION;
1639                         decoder->state.from_ip = decoder->ip;
1640                         decoder->state.to_ip = 0;
1641                         decoder->mtc_insn = false;
1642                         return 0;
1643
1644                 case INTEL_PT_TSC:
1645                         intel_pt_calc_tsc_timestamp(decoder);
1646                         break;
1647
1648                 case INTEL_PT_TMA:
1649                         intel_pt_calc_tma(decoder);
1650                         break;
1651
1652                 case INTEL_PT_CYC:
1653                         intel_pt_calc_cyc_timestamp(decoder);
1654                         break;
1655
1656                 case INTEL_PT_CBR:
1657                         intel_pt_calc_cbr(decoder);
1658                         break;
1659
1660                 case INTEL_PT_MODE_EXEC:
1661                         decoder->exec_mode = decoder->packet.payload;
1662                         break;
1663
1664                 case INTEL_PT_MODE_TSX:
1665                         /* MODE_TSX need not be followed by FUP */
1666                         if (!decoder->pge) {
1667                                 intel_pt_update_in_tx(decoder);
1668                                 break;
1669                         }
1670                         err = intel_pt_mode_tsx(decoder, &no_tip);
1671                         if (err)
1672                                 return err;
1673                         goto next;
1674
1675                 case INTEL_PT_BAD: /* Does not happen */
1676                         return intel_pt_bug(decoder);
1677
1678                 case INTEL_PT_PSBEND:
1679                 case INTEL_PT_VMCS:
1680                 case INTEL_PT_MNT:
1681                 case INTEL_PT_PAD:
1682                         break;
1683
1684                 default:
1685                         return intel_pt_bug(decoder);
1686                 }
1687         }
1688 }
1689
1690 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1691 {
1692         return decoder->last_ip || decoder->packet.count == 0 ||
1693                decoder->packet.count == 3 || decoder->packet.count == 6;
1694 }
1695
1696 /* Walk PSB+ packets to get in sync. */
1697 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1698 {
1699         int err;
1700
1701         while (1) {
1702                 err = intel_pt_get_next_packet(decoder);
1703                 if (err)
1704                         return err;
1705
1706                 switch (decoder->packet.type) {
1707                 case INTEL_PT_TIP_PGD:
1708                         decoder->continuous_period = false;
1709                 case INTEL_PT_TIP_PGE:
1710                 case INTEL_PT_TIP:
1711                         intel_pt_log("ERROR: Unexpected packet\n");
1712                         return -ENOENT;
1713
1714                 case INTEL_PT_FUP:
1715                         decoder->pge = true;
1716                         if (intel_pt_have_ip(decoder)) {
1717                                 uint64_t current_ip = decoder->ip;
1718
1719                                 intel_pt_set_ip(decoder);
1720                                 if (current_ip)
1721                                         intel_pt_log_to("Setting IP",
1722                                                         decoder->ip);
1723                         }
1724                         break;
1725
1726                 case INTEL_PT_MTC:
1727                         intel_pt_calc_mtc_timestamp(decoder);
1728                         break;
1729
1730                 case INTEL_PT_TSC:
1731                         intel_pt_calc_tsc_timestamp(decoder);
1732                         break;
1733
1734                 case INTEL_PT_TMA:
1735                         intel_pt_calc_tma(decoder);
1736                         break;
1737
1738                 case INTEL_PT_CYC:
1739                         intel_pt_calc_cyc_timestamp(decoder);
1740                         break;
1741
1742                 case INTEL_PT_CBR:
1743                         intel_pt_calc_cbr(decoder);
1744                         break;
1745
1746                 case INTEL_PT_PIP:
1747                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1748                         break;
1749
1750                 case INTEL_PT_MODE_EXEC:
1751                         decoder->exec_mode = decoder->packet.payload;
1752                         break;
1753
1754                 case INTEL_PT_MODE_TSX:
1755                         intel_pt_update_in_tx(decoder);
1756                         break;
1757
1758                 case INTEL_PT_TRACESTOP:
1759                         decoder->pge = false;
1760                         decoder->continuous_period = false;
1761                         intel_pt_clear_tx_flags(decoder);
1762                 case INTEL_PT_TNT:
1763                         decoder->have_tma = false;
1764                         intel_pt_log("ERROR: Unexpected packet\n");
1765                         if (decoder->ip)
1766                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1767                         else
1768                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1769                         return -ENOENT;
1770
1771                 case INTEL_PT_BAD: /* Does not happen */
1772                         return intel_pt_bug(decoder);
1773
1774                 case INTEL_PT_OVF:
1775                         return intel_pt_overflow(decoder);
1776
1777                 case INTEL_PT_PSBEND:
1778                         return 0;
1779
1780                 case INTEL_PT_PSB:
1781                 case INTEL_PT_VMCS:
1782                 case INTEL_PT_MNT:
1783                 case INTEL_PT_PAD:
1784                 default:
1785                         break;
1786                 }
1787         }
1788 }
1789
1790 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1791 {
1792         int err;
1793
1794         while (1) {
1795                 err = intel_pt_get_next_packet(decoder);
1796                 if (err)
1797                         return err;
1798
1799                 switch (decoder->packet.type) {
1800                 case INTEL_PT_TIP_PGD:
1801                         decoder->continuous_period = false;
1802                 case INTEL_PT_TIP_PGE:
1803                 case INTEL_PT_TIP:
1804                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1805                         if (intel_pt_have_ip(decoder))
1806                                 intel_pt_set_ip(decoder);
1807                         if (decoder->ip)
1808                                 return 0;
1809                         break;
1810
1811                 case INTEL_PT_FUP:
1812                         if (decoder->overflow) {
1813                                 if (intel_pt_have_ip(decoder))
1814                                         intel_pt_set_ip(decoder);
1815                                 if (decoder->ip)
1816                                         return 0;
1817                         }
1818                         if (decoder->packet.count)
1819                                 intel_pt_set_last_ip(decoder);
1820                         break;
1821
1822                 case INTEL_PT_MTC:
1823                         intel_pt_calc_mtc_timestamp(decoder);
1824                         break;
1825
1826                 case INTEL_PT_TSC:
1827                         intel_pt_calc_tsc_timestamp(decoder);
1828                         break;
1829
1830                 case INTEL_PT_TMA:
1831                         intel_pt_calc_tma(decoder);
1832                         break;
1833
1834                 case INTEL_PT_CYC:
1835                         intel_pt_calc_cyc_timestamp(decoder);
1836                         break;
1837
1838                 case INTEL_PT_CBR:
1839                         intel_pt_calc_cbr(decoder);
1840                         break;
1841
1842                 case INTEL_PT_PIP:
1843                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1844                         break;
1845
1846                 case INTEL_PT_MODE_EXEC:
1847                         decoder->exec_mode = decoder->packet.payload;
1848                         break;
1849
1850                 case INTEL_PT_MODE_TSX:
1851                         intel_pt_update_in_tx(decoder);
1852                         break;
1853
1854                 case INTEL_PT_OVF:
1855                         return intel_pt_overflow(decoder);
1856
1857                 case INTEL_PT_BAD: /* Does not happen */
1858                         return intel_pt_bug(decoder);
1859
1860                 case INTEL_PT_TRACESTOP:
1861                         decoder->pge = false;
1862                         decoder->continuous_period = false;
1863                         intel_pt_clear_tx_flags(decoder);
1864                         decoder->have_tma = false;
1865                         break;
1866
1867                 case INTEL_PT_PSB:
1868                         err = intel_pt_walk_psb(decoder);
1869                         if (err)
1870                                 return err;
1871                         if (decoder->ip) {
1872                                 /* Do not have a sample */
1873                                 decoder->state.type = 0;
1874                                 return 0;
1875                         }
1876                         break;
1877
1878                 case INTEL_PT_TNT:
1879                 case INTEL_PT_PSBEND:
1880                 case INTEL_PT_VMCS:
1881                 case INTEL_PT_MNT:
1882                 case INTEL_PT_PAD:
1883                 default:
1884                         break;
1885                 }
1886         }
1887 }
1888
1889 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1890 {
1891         int err;
1892
1893         intel_pt_log("Scanning for full IP\n");
1894         err = intel_pt_walk_to_ip(decoder);
1895         if (err)
1896                 return err;
1897
1898         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1899         decoder->overflow = false;
1900
1901         decoder->state.from_ip = 0;
1902         decoder->state.to_ip = decoder->ip;
1903         intel_pt_log_to("Setting IP", decoder->ip);
1904
1905         return 0;
1906 }
1907
1908 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1909 {
1910         const unsigned char *end = decoder->buf + decoder->len;
1911         size_t i;
1912
1913         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1914                 if (i > decoder->len)
1915                         continue;
1916                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1917                         return i;
1918         }
1919         return 0;
1920 }
1921
1922 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1923 {
1924         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1925         const char *psb = INTEL_PT_PSB_STR;
1926
1927         if (rest_psb > decoder->len ||
1928             memcmp(decoder->buf, psb + part_psb, rest_psb))
1929                 return 0;
1930
1931         return rest_psb;
1932 }
1933
1934 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1935                                   int part_psb)
1936 {
1937         int rest_psb, ret;
1938
1939         decoder->pos += decoder->len;
1940         decoder->len = 0;
1941
1942         ret = intel_pt_get_next_data(decoder);
1943         if (ret)
1944                 return ret;
1945
1946         rest_psb = intel_pt_rest_psb(decoder, part_psb);
1947         if (!rest_psb)
1948                 return 0;
1949
1950         decoder->pos -= part_psb;
1951         decoder->next_buf = decoder->buf + rest_psb;
1952         decoder->next_len = decoder->len - rest_psb;
1953         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1954         decoder->buf = decoder->temp_buf;
1955         decoder->len = INTEL_PT_PSB_LEN;
1956
1957         return 0;
1958 }
1959
1960 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
1961 {
1962         unsigned char *next;
1963         int ret;
1964
1965         intel_pt_log("Scanning for PSB\n");
1966         while (1) {
1967                 if (!decoder->len) {
1968                         ret = intel_pt_get_next_data(decoder);
1969                         if (ret)
1970                                 return ret;
1971                 }
1972
1973                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
1974                               INTEL_PT_PSB_LEN);
1975                 if (!next) {
1976                         int part_psb;
1977
1978                         part_psb = intel_pt_part_psb(decoder);
1979                         if (part_psb) {
1980                                 ret = intel_pt_get_split_psb(decoder, part_psb);
1981                                 if (ret)
1982                                         return ret;
1983                         } else {
1984                                 decoder->pos += decoder->len;
1985                                 decoder->len = 0;
1986                         }
1987                         continue;
1988                 }
1989
1990                 decoder->pkt_step = next - decoder->buf;
1991                 return intel_pt_get_next_packet(decoder);
1992         }
1993 }
1994
1995 static int intel_pt_sync(struct intel_pt_decoder *decoder)
1996 {
1997         int err;
1998
1999         decoder->pge = false;
2000         decoder->continuous_period = false;
2001         decoder->last_ip = 0;
2002         decoder->ip = 0;
2003         intel_pt_clear_stack(&decoder->stack);
2004
2005         err = intel_pt_scan_for_psb(decoder);
2006         if (err)
2007                 return err;
2008
2009         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2010
2011         err = intel_pt_walk_psb(decoder);
2012         if (err)
2013                 return err;
2014
2015         if (decoder->ip) {
2016                 decoder->state.type = 0; /* Do not have a sample */
2017                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2018         } else {
2019                 return intel_pt_sync_ip(decoder);
2020         }
2021
2022         return 0;
2023 }
2024
2025 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2026 {
2027         uint64_t est = decoder->timestamp_insn_cnt << 1;
2028
2029         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2030                 goto out;
2031
2032         est *= decoder->max_non_turbo_ratio;
2033         est /= decoder->cbr;
2034 out:
2035         return decoder->timestamp + est;
2036 }
2037
2038 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2039 {
2040         int err;
2041
2042         do {
2043                 decoder->state.type = INTEL_PT_BRANCH;
2044                 decoder->state.flags = 0;
2045
2046                 switch (decoder->pkt_state) {
2047                 case INTEL_PT_STATE_NO_PSB:
2048                         err = intel_pt_sync(decoder);
2049                         break;
2050                 case INTEL_PT_STATE_NO_IP:
2051                         decoder->last_ip = 0;
2052                         /* Fall through */
2053                 case INTEL_PT_STATE_ERR_RESYNC:
2054                         err = intel_pt_sync_ip(decoder);
2055                         break;
2056                 case INTEL_PT_STATE_IN_SYNC:
2057                         err = intel_pt_walk_trace(decoder);
2058                         break;
2059                 case INTEL_PT_STATE_TNT:
2060                         err = intel_pt_walk_tnt(decoder);
2061                         if (err == -EAGAIN)
2062                                 err = intel_pt_walk_trace(decoder);
2063                         break;
2064                 case INTEL_PT_STATE_TIP:
2065                 case INTEL_PT_STATE_TIP_PGD:
2066                         err = intel_pt_walk_tip(decoder);
2067                         break;
2068                 case INTEL_PT_STATE_FUP:
2069                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2070                         err = intel_pt_walk_fup(decoder);
2071                         if (err == -EAGAIN)
2072                                 err = intel_pt_walk_fup_tip(decoder);
2073                         else if (!err)
2074                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2075                         break;
2076                 case INTEL_PT_STATE_FUP_NO_TIP:
2077                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2078                         err = intel_pt_walk_fup(decoder);
2079                         if (err == -EAGAIN)
2080                                 err = intel_pt_walk_trace(decoder);
2081                         break;
2082                 default:
2083                         err = intel_pt_bug(decoder);
2084                         break;
2085                 }
2086         } while (err == -ENOLINK);
2087
2088         decoder->state.err = err ? intel_pt_ext_err(err) : 0;
2089         decoder->state.timestamp = decoder->timestamp;
2090         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2091         decoder->state.cr3 = decoder->cr3;
2092         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2093
2094         if (err)
2095                 decoder->state.from_ip = decoder->ip;
2096
2097         return &decoder->state;
2098 }
2099
2100 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2101 {
2102         if (len < INTEL_PT_PSB_LEN)
2103                 return false;
2104         return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2105                       INTEL_PT_PSB_LEN);
2106 }
2107
2108 /**
2109  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2110  * @buf: pointer to buffer pointer
2111  * @len: size of buffer
2112  *
2113  * Updates the buffer pointer to point to the start of the next PSB packet if
2114  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2115  * @len is adjusted accordingly.
2116  *
2117  * Return: %true if a PSB packet is found, %false otherwise.
2118  */
2119 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2120 {
2121         unsigned char *next;
2122
2123         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2124         if (next) {
2125                 *len -= next - *buf;
2126                 *buf = next;
2127                 return true;
2128         }
2129         return false;
2130 }
2131
2132 /**
2133  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2134  *                     packet.
2135  * @buf: pointer to buffer pointer
2136  * @len: size of buffer
2137  *
2138  * Updates the buffer pointer to point to the start of the following PSB packet
2139  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2140  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2141  *
2142  * Return: %true if a PSB packet is found, %false otherwise.
2143  */
2144 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2145 {
2146         unsigned char *next;
2147
2148         if (!*len)
2149                 return false;
2150
2151         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2152         if (next) {
2153                 *len -= next - *buf;
2154                 *buf = next;
2155                 return true;
2156         }
2157         return false;
2158 }
2159
2160 /**
2161  * intel_pt_last_psb - find the last PSB packet in a buffer.
2162  * @buf: buffer
2163  * @len: size of buffer
2164  *
2165  * This function finds the last PSB in a buffer.
2166  *
2167  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2168  */
2169 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2170 {
2171         const char *n = INTEL_PT_PSB_STR;
2172         unsigned char *p;
2173         size_t k;
2174
2175         if (len < INTEL_PT_PSB_LEN)
2176                 return NULL;
2177
2178         k = len - INTEL_PT_PSB_LEN + 1;
2179         while (1) {
2180                 p = memrchr(buf, n[0], k);
2181                 if (!p)
2182                         return NULL;
2183                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2184                         return p;
2185                 k = p - buf;
2186                 if (!k)
2187                         return NULL;
2188         }
2189 }
2190
2191 /**
2192  * intel_pt_next_tsc - find and return next TSC.
2193  * @buf: buffer
2194  * @len: size of buffer
2195  * @tsc: TSC value returned
2196  *
2197  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2198  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2199  * PSBEND packet is found.
2200  *
2201  * Return: %true if TSC is found, false otherwise.
2202  */
2203 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2204 {
2205         struct intel_pt_pkt packet;
2206         int ret;
2207
2208         while (len) {
2209                 ret = intel_pt_get_packet(buf, len, &packet);
2210                 if (ret <= 0)
2211                         return false;
2212                 if (packet.type == INTEL_PT_TSC) {
2213                         *tsc = packet.payload;
2214                         return true;
2215                 }
2216                 if (packet.type == INTEL_PT_PSBEND)
2217                         return false;
2218                 buf += ret;
2219                 len -= ret;
2220         }
2221         return false;
2222 }
2223
2224 /**
2225  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2226  * @tsc1: first TSC to compare
2227  * @tsc2: second TSC to compare
2228  *
2229  * This function compares 7-byte TSC values allowing for the possibility that
2230  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2231  * around so for that purpose this function assumes the absolute difference is
2232  * less than half the maximum difference.
2233  *
2234  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2235  * after @tsc2.
2236  */
2237 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2238 {
2239         const uint64_t halfway = (1ULL << 55);
2240
2241         if (tsc1 == tsc2)
2242                 return 0;
2243
2244         if (tsc1 < tsc2) {
2245                 if (tsc2 - tsc1 < halfway)
2246                         return -1;
2247                 else
2248                         return 1;
2249         } else {
2250                 if (tsc1 - tsc2 < halfway)
2251                         return 1;
2252                 else
2253                         return -1;
2254         }
2255 }
2256
2257 /**
2258  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2259  *                             using TSC.
2260  * @buf_a: first buffer
2261  * @len_a: size of first buffer
2262  * @buf_b: second buffer
2263  * @len_b: size of second buffer
2264  *
2265  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2266  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2267  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2268  * @buf_a and @buf_b are positioned at a PSB.
2269  *
2270  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2271  * @buf_b + @len_b if there is no non-overlapped data.
2272  */
2273 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2274                                                 size_t len_a,
2275                                                 unsigned char *buf_b,
2276                                                 size_t len_b)
2277 {
2278         uint64_t tsc_a, tsc_b;
2279         unsigned char *p;
2280         size_t len;
2281
2282         p = intel_pt_last_psb(buf_a, len_a);
2283         if (!p)
2284                 return buf_b; /* No PSB in buf_a => no overlap */
2285
2286         len = len_a - (p - buf_a);
2287         if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2288                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2289                 len_a -= len;
2290                 p = intel_pt_last_psb(buf_a, len_a);
2291                 if (!p)
2292                         return buf_b; /* No full PSB+ => assume no overlap */
2293                 len = len_a - (p - buf_a);
2294                 if (!intel_pt_next_tsc(p, len, &tsc_a))
2295                         return buf_b; /* No TSC in buf_a => assume no overlap */
2296         }
2297
2298         while (1) {
2299                 /* Ignore PSB+ with no TSC */
2300                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2301                     intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2302                         return buf_b; /* tsc_a < tsc_b => no overlap */
2303
2304                 if (!intel_pt_step_psb(&buf_b, &len_b))
2305                         return buf_b + len_b; /* No PSB in buf_b => no data */
2306         }
2307 }
2308
2309 /**
2310  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2311  * @buf_a: first buffer
2312  * @len_a: size of first buffer
2313  * @buf_b: second buffer
2314  * @len_b: size of second buffer
2315  * @have_tsc: can use TSC packets to detect overlap
2316  *
2317  * When trace samples or snapshots are recorded there is the possibility that
2318  * the data overlaps.  Note that, for the purposes of decoding, data is only
2319  * useful if it begins with a PSB packet.
2320  *
2321  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2322  * @buf_b + @len_b if there is no non-overlapped data.
2323  */
2324 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2325                                      unsigned char *buf_b, size_t len_b,
2326                                      bool have_tsc)
2327 {
2328         unsigned char *found;
2329
2330         /* Buffer 'b' must start at PSB so throw away everything before that */
2331         if (!intel_pt_next_psb(&buf_b, &len_b))
2332                 return buf_b + len_b; /* No PSB */
2333
2334         if (!intel_pt_next_psb(&buf_a, &len_a))
2335                 return buf_b; /* No overlap */
2336
2337         if (have_tsc) {
2338                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2339                 if (found)
2340                         return found;
2341         }
2342
2343         /*
2344          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2345          * we can ignore the first part of buffer 'a'.
2346          */
2347         while (len_b < len_a) {
2348                 if (!intel_pt_step_psb(&buf_a, &len_a))
2349                         return buf_b; /* No overlap */
2350         }
2351
2352         /* Now len_b >= len_a */
2353         if (len_b > len_a) {
2354                 /* The leftover buffer 'b' must start at a PSB */
2355                 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2356                         if (!intel_pt_step_psb(&buf_a, &len_a))
2357                                 return buf_b; /* No overlap */
2358                 }
2359         }
2360
2361         while (1) {
2362                 /* Potential overlap so check the bytes */
2363                 found = memmem(buf_a, len_a, buf_b, len_a);
2364                 if (found)
2365                         return buf_b + len_a;
2366
2367                 /* Try again at next PSB in buffer 'a' */
2368                 if (!intel_pt_step_psb(&buf_a, &len_a))
2369                         return buf_b; /* No overlap */
2370
2371                 /* The leftover buffer 'b' must start at a PSB */
2372                 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2373                         if (!intel_pt_step_psb(&buf_a, &len_a))
2374                                 return buf_b; /* No overlap */
2375                 }
2376         }
2377 }