2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
29 #include "intel-pt-insn-decoder.h"
30 #include "intel-pt-pkt-decoder.h"
31 #include "intel-pt-decoder.h"
32 #include "intel-pt-log.h"
34 #define INTEL_PT_BLK_SIZE 1024
36 #define BIT63 (((uint64_t)1 << 63))
38 #define INTEL_PT_RETURN 1
40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
41 #define INTEL_PT_MAX_LOOPS 10000
44 struct intel_pt_blk *prev;
45 uint64_t ip[INTEL_PT_BLK_SIZE];
48 struct intel_pt_stack {
49 struct intel_pt_blk *blk;
50 struct intel_pt_blk *spare;
54 enum intel_pt_pkt_state {
55 INTEL_PT_STATE_NO_PSB,
57 INTEL_PT_STATE_ERR_RESYNC,
58 INTEL_PT_STATE_IN_SYNC,
61 INTEL_PT_STATE_TIP_PGD,
63 INTEL_PT_STATE_FUP_NO_TIP,
66 #ifdef INTEL_PT_STRICT
67 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
68 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
69 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
70 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
72 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
73 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
74 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
75 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
78 struct intel_pt_decoder {
79 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
80 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
81 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
82 uint64_t max_insn_cnt, void *data);
84 struct intel_pt_state state;
85 const unsigned char *buf;
87 bool return_compression;
97 uint64_t tsc_timestamp;
98 uint64_t ref_timestamp;
100 uint64_t ctc_timestamp;
103 uint64_t cyc_ref_timestamp;
105 uint32_t tsc_ctc_ratio_n;
106 uint32_t tsc_ctc_ratio_d;
107 uint32_t tsc_ctc_mult;
109 uint32_t ctc_rem_mask;
111 struct intel_pt_stack stack;
112 enum intel_pt_pkt_state pkt_state;
113 struct intel_pt_pkt packet;
114 struct intel_pt_pkt tnt;
117 int last_packet_type;
119 unsigned int max_non_turbo_ratio;
120 double max_non_turbo_ratio_fp;
121 double cbr_cyc_to_tsc;
122 double calc_cyc_to_tsc;
123 bool have_calc_cyc_to_tsc;
125 unsigned int insn_bytes;
127 enum intel_pt_period_type period_type;
128 uint64_t tot_insn_cnt;
129 uint64_t period_insn_cnt;
130 uint64_t period_mask;
131 uint64_t period_ticks;
132 uint64_t last_masked_timestamp;
133 bool continuous_period;
135 bool set_fup_tx_flags;
136 unsigned int fup_tx_flags;
137 unsigned int tx_flags;
138 uint64_t timestamp_insn_cnt;
143 const unsigned char *next_buf;
145 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
148 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
152 for (i = 0; x != 1; i++)
158 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
160 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
163 period = intel_pt_lower_power_of_2(decoder->period);
164 decoder->period_mask = ~(period - 1);
165 decoder->period_ticks = period;
169 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
173 return (t / d) * n + ((t % d) * n) / d;
176 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
178 struct intel_pt_decoder *decoder;
180 if (!params->get_trace || !params->walk_insn)
183 decoder = zalloc(sizeof(struct intel_pt_decoder));
187 decoder->get_trace = params->get_trace;
188 decoder->walk_insn = params->walk_insn;
189 decoder->data = params->data;
190 decoder->return_compression = params->return_compression;
192 decoder->period = params->period;
193 decoder->period_type = params->period_type;
195 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
196 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
198 intel_pt_setup_period(decoder);
200 decoder->mtc_shift = params->mtc_period;
201 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
203 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
204 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
206 if (!decoder->tsc_ctc_ratio_n)
207 decoder->tsc_ctc_ratio_d = 0;
209 if (decoder->tsc_ctc_ratio_d) {
210 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
211 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
212 decoder->tsc_ctc_ratio_d;
215 * Allow for timestamps appearing to backwards because a TSC
216 * packet has slipped past a MTC packet, so allow 2 MTC ticks
219 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
220 decoder->tsc_ctc_ratio_n,
221 decoder->tsc_ctc_ratio_d);
223 /* ... or 0x100 paranoia */
224 if (decoder->tsc_slip < 0x100)
225 decoder->tsc_slip = 0x100;
227 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
228 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
229 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
230 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
231 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
236 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
238 struct intel_pt_blk *blk = stack->blk;
240 stack->blk = blk->prev;
247 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
252 intel_pt_pop_blk(stack);
255 stack->pos = INTEL_PT_BLK_SIZE;
257 return stack->blk->ip[--stack->pos];
260 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
262 struct intel_pt_blk *blk;
268 blk = malloc(sizeof(struct intel_pt_blk));
273 blk->prev = stack->blk;
279 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
283 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
284 err = intel_pt_alloc_blk(stack);
289 stack->blk->ip[stack->pos++] = ip;
293 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
296 intel_pt_pop_blk(stack);
300 static void intel_pt_free_stack(struct intel_pt_stack *stack)
302 intel_pt_clear_stack(stack);
304 zfree(&stack->spare);
307 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
309 intel_pt_free_stack(&decoder->stack);
313 static int intel_pt_ext_err(int code)
317 return INTEL_PT_ERR_NOMEM;
319 return INTEL_PT_ERR_INTERN;
321 return INTEL_PT_ERR_BADPKT;
323 return INTEL_PT_ERR_NODATA;
325 return INTEL_PT_ERR_NOINSN;
327 return INTEL_PT_ERR_MISMAT;
329 return INTEL_PT_ERR_OVR;
331 return INTEL_PT_ERR_LOST;
333 return INTEL_PT_ERR_NELOOP;
335 return INTEL_PT_ERR_UNK;
339 static const char *intel_pt_err_msgs[] = {
340 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
341 [INTEL_PT_ERR_INTERN] = "Internal error",
342 [INTEL_PT_ERR_BADPKT] = "Bad packet",
343 [INTEL_PT_ERR_NODATA] = "No more data",
344 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
345 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
346 [INTEL_PT_ERR_OVR] = "Overflow packet",
347 [INTEL_PT_ERR_LOST] = "Lost trace data",
348 [INTEL_PT_ERR_UNK] = "Unknown error!",
349 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
352 int intel_pt__strerror(int code, char *buf, size_t buflen)
354 if (code < 1 || code >= INTEL_PT_ERR_MAX)
355 code = INTEL_PT_ERR_UNK;
356 strlcpy(buf, intel_pt_err_msgs[code], buflen);
360 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
365 switch (packet->count) {
367 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
371 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
375 ip = packet->payload;
376 /* Sign-extend 6-byte ip */
377 if (ip & (uint64_t)0x800000000000ULL)
378 ip |= (uint64_t)0xffff000000000000ULL;
381 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
385 ip = packet->payload;
394 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
396 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
399 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
401 intel_pt_set_last_ip(decoder);
402 decoder->ip = decoder->last_ip;
405 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
407 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
411 static int intel_pt_bug(struct intel_pt_decoder *decoder)
413 intel_pt_log("ERROR: Internal error\n");
414 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
418 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
420 decoder->tx_flags = 0;
423 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
425 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
428 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
430 intel_pt_clear_tx_flags(decoder);
431 decoder->have_tma = false;
432 decoder->pkt_len = 1;
433 decoder->pkt_step = 1;
434 intel_pt_decoder_log_packet(decoder);
435 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
436 intel_pt_log("ERROR: Bad packet\n");
437 decoder->pkt_state = INTEL_PT_STATE_ERR1;
442 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
444 struct intel_pt_buffer buffer = { .buf = 0, };
447 decoder->pkt_step = 0;
449 intel_pt_log("Getting more data\n");
450 ret = decoder->get_trace(&buffer, decoder->data);
453 decoder->buf = buffer.buf;
454 decoder->len = buffer.len;
456 intel_pt_log("No more data\n");
459 if (!buffer.consecutive) {
461 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
462 decoder->ref_timestamp = buffer.ref_timestamp;
463 decoder->timestamp = 0;
464 decoder->have_tma = false;
465 decoder->state.trace_nr = buffer.trace_nr;
466 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
467 decoder->ref_timestamp);
474 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
476 if (!decoder->next_buf)
477 return intel_pt_get_data(decoder);
479 decoder->buf = decoder->next_buf;
480 decoder->len = decoder->next_len;
481 decoder->next_buf = 0;
482 decoder->next_len = 0;
486 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
488 unsigned char *buf = decoder->temp_buf;
489 size_t old_len, len, n;
492 old_len = decoder->len;
494 memcpy(buf, decoder->buf, len);
496 ret = intel_pt_get_data(decoder);
498 decoder->pos += old_len;
499 return ret < 0 ? ret : -EINVAL;
502 n = INTEL_PT_PKT_MAX_SZ - len;
503 if (n > decoder->len)
505 memcpy(buf + len, decoder->buf, n);
508 ret = intel_pt_get_packet(buf, len, &decoder->packet);
509 if (ret < (int)old_len) {
510 decoder->next_buf = decoder->buf;
511 decoder->next_len = decoder->len;
513 decoder->len = old_len;
514 return intel_pt_bad_packet(decoder);
517 decoder->next_buf = decoder->buf + (ret - old_len);
518 decoder->next_len = decoder->len - (ret - old_len);
526 struct intel_pt_pkt_info {
527 struct intel_pt_decoder *decoder;
528 struct intel_pt_pkt packet;
531 int last_packet_type;
535 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
537 /* Lookahead packets in current buffer */
538 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
539 intel_pt_pkt_cb_t cb, void *data)
541 struct intel_pt_pkt_info pkt_info;
542 const unsigned char *buf = decoder->buf;
543 size_t len = decoder->len;
546 pkt_info.decoder = decoder;
547 pkt_info.pos = decoder->pos;
548 pkt_info.pkt_len = decoder->pkt_step;
549 pkt_info.last_packet_type = decoder->last_packet_type;
550 pkt_info.data = data;
554 pkt_info.pos += pkt_info.pkt_len;
555 buf += pkt_info.pkt_len;
556 len -= pkt_info.pkt_len;
559 return INTEL_PT_NEED_MORE_BYTES;
561 ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
563 return INTEL_PT_NEED_MORE_BYTES;
567 pkt_info.pkt_len = ret;
568 } while (pkt_info.packet.type == INTEL_PT_PAD);
574 pkt_info.last_packet_type = pkt_info.packet.type;
578 struct intel_pt_calc_cyc_to_tsc_info {
582 uint64_t ctc_timestamp;
584 uint64_t tsc_timestamp;
588 double cbr_cyc_to_tsc;
591 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
593 struct intel_pt_decoder *decoder = pkt_info->decoder;
594 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
598 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
600 switch (pkt_info->packet.type) {
602 case INTEL_PT_TIP_PGE:
607 case INTEL_PT_MODE_EXEC:
608 case INTEL_PT_MODE_TSX:
609 case INTEL_PT_PSBEND:
619 mtc = pkt_info->packet.payload;
620 if (mtc > data->last_mtc)
621 mtc_delta = mtc - data->last_mtc;
623 mtc_delta = mtc + 256 - data->last_mtc;
624 data->ctc_delta += mtc_delta << decoder->mtc_shift;
625 data->last_mtc = mtc;
627 if (decoder->tsc_ctc_mult) {
628 timestamp = data->ctc_timestamp +
629 data->ctc_delta * decoder->tsc_ctc_mult;
631 timestamp = data->ctc_timestamp +
632 multdiv(data->ctc_delta,
633 decoder->tsc_ctc_ratio_n,
634 decoder->tsc_ctc_ratio_d);
637 if (timestamp < data->timestamp)
640 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
641 data->timestamp = timestamp;
648 timestamp = pkt_info->packet.payload |
649 (data->timestamp & (0xffULL << 56));
650 if (data->from_mtc && timestamp < data->timestamp &&
651 data->timestamp - timestamp < decoder->tsc_slip)
653 if (timestamp < data->timestamp)
654 timestamp += (1ULL << 56);
655 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
658 data->tsc_timestamp = timestamp;
659 data->timestamp = timestamp;
668 if (!decoder->tsc_ctc_ratio_d)
671 ctc = pkt_info->packet.payload;
672 fc = pkt_info->packet.count;
673 ctc_rem = ctc & decoder->ctc_rem_mask;
675 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
677 data->ctc_timestamp = data->tsc_timestamp - fc;
678 if (decoder->tsc_ctc_mult) {
679 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
681 data->ctc_timestamp -=
682 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
683 decoder->tsc_ctc_ratio_d);
687 data->have_tma = true;
692 data->cycle_cnt += pkt_info->packet.payload;
696 cbr = pkt_info->packet.payload;
697 if (data->cbr && data->cbr != cbr)
700 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
703 case INTEL_PT_TIP_PGD:
704 case INTEL_PT_TRACESTOP:
706 case INTEL_PT_BAD: /* Does not happen */
711 if (!data->cbr && decoder->cbr) {
712 data->cbr = decoder->cbr;
713 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
716 if (!data->cycle_cnt)
719 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
721 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
722 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
723 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
724 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
728 decoder->calc_cyc_to_tsc = cyc_to_tsc;
729 decoder->have_calc_cyc_to_tsc = true;
732 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
733 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
735 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
736 cyc_to_tsc, pkt_info->pos);
742 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
745 struct intel_pt_calc_cyc_to_tsc_info data = {
748 .last_mtc = decoder->last_mtc,
749 .ctc_timestamp = decoder->ctc_timestamp,
750 .ctc_delta = decoder->ctc_delta,
751 .tsc_timestamp = decoder->tsc_timestamp,
752 .timestamp = decoder->timestamp,
753 .have_tma = decoder->have_tma,
754 .from_mtc = from_mtc,
758 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
761 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
765 decoder->last_packet_type = decoder->packet.type;
768 decoder->pos += decoder->pkt_step;
769 decoder->buf += decoder->pkt_step;
770 decoder->len -= decoder->pkt_step;
773 ret = intel_pt_get_next_data(decoder);
778 ret = intel_pt_get_packet(decoder->buf, decoder->len,
780 if (ret == INTEL_PT_NEED_MORE_BYTES &&
781 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
782 ret = intel_pt_get_split_packet(decoder);
787 return intel_pt_bad_packet(decoder);
789 decoder->pkt_len = ret;
790 decoder->pkt_step = ret;
791 intel_pt_decoder_log_packet(decoder);
792 } while (decoder->packet.type == INTEL_PT_PAD);
797 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
799 uint64_t timestamp, masked_timestamp;
801 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
802 masked_timestamp = timestamp & decoder->period_mask;
803 if (decoder->continuous_period) {
804 if (masked_timestamp != decoder->last_masked_timestamp)
808 masked_timestamp = timestamp & decoder->period_mask;
809 if (masked_timestamp != decoder->last_masked_timestamp) {
810 decoder->last_masked_timestamp = masked_timestamp;
811 decoder->continuous_period = true;
814 return decoder->period_ticks - (timestamp - masked_timestamp);
817 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
819 switch (decoder->period_type) {
820 case INTEL_PT_PERIOD_INSTRUCTIONS:
821 return decoder->period - decoder->period_insn_cnt;
822 case INTEL_PT_PERIOD_TICKS:
823 return intel_pt_next_period(decoder);
824 case INTEL_PT_PERIOD_NONE:
825 case INTEL_PT_PERIOD_MTC:
831 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
833 uint64_t timestamp, masked_timestamp;
835 switch (decoder->period_type) {
836 case INTEL_PT_PERIOD_INSTRUCTIONS:
837 decoder->period_insn_cnt = 0;
839 case INTEL_PT_PERIOD_TICKS:
840 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
841 masked_timestamp = timestamp & decoder->period_mask;
842 decoder->last_masked_timestamp = masked_timestamp;
844 case INTEL_PT_PERIOD_NONE:
845 case INTEL_PT_PERIOD_MTC:
850 decoder->state.type |= INTEL_PT_INSTRUCTION;
853 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
854 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
856 uint64_t max_insn_cnt, insn_cnt = 0;
859 if (!decoder->mtc_insn)
860 decoder->mtc_insn = true;
862 max_insn_cnt = intel_pt_next_sample(decoder);
864 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
865 max_insn_cnt, decoder->data);
867 decoder->tot_insn_cnt += insn_cnt;
868 decoder->timestamp_insn_cnt += insn_cnt;
869 decoder->period_insn_cnt += insn_cnt;
872 decoder->no_progress = 0;
873 decoder->pkt_state = INTEL_PT_STATE_ERR2;
874 intel_pt_log_at("ERROR: Failed to get instruction",
881 if (ip && decoder->ip == ip) {
886 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
887 intel_pt_sample_insn(decoder);
889 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
890 decoder->state.type = INTEL_PT_INSTRUCTION;
891 decoder->state.from_ip = decoder->ip;
892 decoder->state.to_ip = 0;
893 decoder->ip += intel_pt_insn->length;
894 err = INTEL_PT_RETURN;
898 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
899 /* Zero-length calls are excluded */
900 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
901 intel_pt_insn->rel) {
902 err = intel_pt_push(&decoder->stack, decoder->ip +
903 intel_pt_insn->length);
907 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
908 decoder->ret_addr = intel_pt_pop(&decoder->stack);
911 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
912 int cnt = decoder->no_progress++;
914 decoder->state.from_ip = decoder->ip;
915 decoder->ip += intel_pt_insn->length +
917 decoder->state.to_ip = decoder->ip;
918 err = INTEL_PT_RETURN;
921 * Check for being stuck in a loop. This can happen if a
922 * decoder error results in the decoder erroneously setting the
923 * ip to an address that is itself in an infinite loop that
924 * consumes no packets. When that happens, there must be an
925 * unconditional branch.
929 decoder->stuck_ip = decoder->state.to_ip;
930 decoder->stuck_ip_prd = 1;
931 decoder->stuck_ip_cnt = 1;
932 } else if (cnt > INTEL_PT_MAX_LOOPS ||
933 decoder->state.to_ip == decoder->stuck_ip) {
934 intel_pt_log_at("ERROR: Never-ending loop",
935 decoder->state.to_ip);
936 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
939 } else if (!--decoder->stuck_ip_cnt) {
940 decoder->stuck_ip_prd += 1;
941 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
942 decoder->stuck_ip = decoder->state.to_ip;
945 goto out_no_progress;
948 decoder->no_progress = 0;
950 decoder->state.insn_op = intel_pt_insn->op;
951 decoder->state.insn_len = intel_pt_insn->length;
953 if (decoder->tx_flags & INTEL_PT_IN_TX)
954 decoder->state.flags |= INTEL_PT_IN_TX;
959 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
961 struct intel_pt_insn intel_pt_insn;
965 ip = decoder->last_ip;
968 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
969 if (err == INTEL_PT_RETURN)
971 if (err == -EAGAIN) {
972 if (decoder->set_fup_tx_flags) {
973 decoder->set_fup_tx_flags = false;
974 decoder->tx_flags = decoder->fup_tx_flags;
975 decoder->state.type = INTEL_PT_TRANSACTION;
976 decoder->state.from_ip = decoder->ip;
977 decoder->state.to_ip = 0;
978 decoder->state.flags = decoder->fup_tx_flags;
983 decoder->set_fup_tx_flags = false;
987 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
988 intel_pt_log_at("ERROR: Unexpected indirect branch",
990 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
994 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
995 intel_pt_log_at("ERROR: Unexpected conditional branch",
997 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1001 intel_pt_bug(decoder);
1005 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1007 struct intel_pt_insn intel_pt_insn;
1010 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1011 if (err == INTEL_PT_RETURN)
1016 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1017 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1018 decoder->pge = false;
1019 decoder->continuous_period = false;
1020 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1021 decoder->state.from_ip = decoder->ip;
1022 decoder->state.to_ip = 0;
1023 if (decoder->packet.count != 0)
1024 decoder->ip = decoder->last_ip;
1026 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1027 decoder->state.from_ip = decoder->ip;
1028 if (decoder->packet.count == 0) {
1029 decoder->state.to_ip = 0;
1031 decoder->state.to_ip = decoder->last_ip;
1032 decoder->ip = decoder->last_ip;
1038 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1039 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1041 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1045 return intel_pt_bug(decoder);
1048 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1050 struct intel_pt_insn intel_pt_insn;
1054 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1055 if (err == INTEL_PT_RETURN)
1060 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1061 if (!decoder->return_compression) {
1062 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1064 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1067 if (!decoder->ret_addr) {
1068 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1070 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1073 if (!(decoder->tnt.payload & BIT63)) {
1074 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1076 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1079 decoder->tnt.count -= 1;
1080 if (!decoder->tnt.count)
1081 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1082 decoder->tnt.payload <<= 1;
1083 decoder->state.from_ip = decoder->ip;
1084 decoder->ip = decoder->ret_addr;
1085 decoder->state.to_ip = decoder->ip;
1089 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1090 /* Handle deferred TIPs */
1091 err = intel_pt_get_next_packet(decoder);
1094 if (decoder->packet.type != INTEL_PT_TIP ||
1095 decoder->packet.count == 0) {
1096 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1098 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1099 decoder->pkt_step = 0;
1102 intel_pt_set_last_ip(decoder);
1103 decoder->state.from_ip = decoder->ip;
1104 decoder->state.to_ip = decoder->last_ip;
1105 decoder->ip = decoder->last_ip;
1109 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1110 decoder->tnt.count -= 1;
1111 if (!decoder->tnt.count)
1112 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1113 if (decoder->tnt.payload & BIT63) {
1114 decoder->tnt.payload <<= 1;
1115 decoder->state.from_ip = decoder->ip;
1116 decoder->ip += intel_pt_insn.length +
1118 decoder->state.to_ip = decoder->ip;
1121 /* Instruction sample for a non-taken branch */
1122 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1123 decoder->tnt.payload <<= 1;
1124 decoder->state.type = INTEL_PT_INSTRUCTION;
1125 decoder->state.from_ip = decoder->ip;
1126 decoder->state.to_ip = 0;
1127 decoder->ip += intel_pt_insn.length;
1130 decoder->ip += intel_pt_insn.length;
1131 if (!decoder->tnt.count)
1133 decoder->tnt.payload <<= 1;
1137 return intel_pt_bug(decoder);
1141 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1143 unsigned int fup_tx_flags;
1146 fup_tx_flags = decoder->packet.payload &
1147 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1148 err = intel_pt_get_next_packet(decoder);
1151 if (decoder->packet.type == INTEL_PT_FUP) {
1152 decoder->fup_tx_flags = fup_tx_flags;
1153 decoder->set_fup_tx_flags = true;
1154 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1157 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1159 intel_pt_update_in_tx(decoder);
1164 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1168 decoder->have_tma = false;
1170 if (decoder->ref_timestamp) {
1171 timestamp = decoder->packet.payload |
1172 (decoder->ref_timestamp & (0xffULL << 56));
1173 if (timestamp < decoder->ref_timestamp) {
1174 if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1175 timestamp += (1ULL << 56);
1177 if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1178 timestamp -= (1ULL << 56);
1180 decoder->tsc_timestamp = timestamp;
1181 decoder->timestamp = timestamp;
1182 decoder->ref_timestamp = 0;
1183 decoder->timestamp_insn_cnt = 0;
1184 } else if (decoder->timestamp) {
1185 timestamp = decoder->packet.payload |
1186 (decoder->timestamp & (0xffULL << 56));
1187 decoder->tsc_timestamp = timestamp;
1188 if (timestamp < decoder->timestamp &&
1189 decoder->timestamp - timestamp < decoder->tsc_slip) {
1190 intel_pt_log_to("Suppressing backwards timestamp",
1192 timestamp = decoder->timestamp;
1194 if (timestamp < decoder->timestamp) {
1195 intel_pt_log_to("Wraparound timestamp", timestamp);
1196 timestamp += (1ULL << 56);
1197 decoder->tsc_timestamp = timestamp;
1199 decoder->timestamp = timestamp;
1200 decoder->timestamp_insn_cnt = 0;
1203 if (decoder->last_packet_type == INTEL_PT_CYC) {
1204 decoder->cyc_ref_timestamp = decoder->timestamp;
1205 decoder->cycle_cnt = 0;
1206 decoder->have_calc_cyc_to_tsc = false;
1207 intel_pt_calc_cyc_to_tsc(decoder, false);
1210 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1213 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1215 intel_pt_log("ERROR: Buffer overflow\n");
1216 intel_pt_clear_tx_flags(decoder);
1217 decoder->have_tma = false;
1219 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1220 decoder->overflow = true;
1224 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1226 uint32_t ctc = decoder->packet.payload;
1227 uint32_t fc = decoder->packet.count;
1228 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1230 if (!decoder->tsc_ctc_ratio_d)
1233 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1234 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1235 if (decoder->tsc_ctc_mult) {
1236 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1238 decoder->ctc_timestamp -= multdiv(ctc_rem,
1239 decoder->tsc_ctc_ratio_n,
1240 decoder->tsc_ctc_ratio_d);
1242 decoder->ctc_delta = 0;
1243 decoder->have_tma = true;
1244 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1245 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1248 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1251 uint32_t mtc, mtc_delta;
1253 if (!decoder->have_tma)
1256 mtc = decoder->packet.payload;
1258 if (mtc > decoder->last_mtc)
1259 mtc_delta = mtc - decoder->last_mtc;
1261 mtc_delta = mtc + 256 - decoder->last_mtc;
1263 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1265 if (decoder->tsc_ctc_mult) {
1266 timestamp = decoder->ctc_timestamp +
1267 decoder->ctc_delta * decoder->tsc_ctc_mult;
1269 timestamp = decoder->ctc_timestamp +
1270 multdiv(decoder->ctc_delta,
1271 decoder->tsc_ctc_ratio_n,
1272 decoder->tsc_ctc_ratio_d);
1275 if (timestamp < decoder->timestamp)
1276 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1277 timestamp, decoder->timestamp);
1279 decoder->timestamp = timestamp;
1281 decoder->timestamp_insn_cnt = 0;
1282 decoder->last_mtc = mtc;
1284 if (decoder->last_packet_type == INTEL_PT_CYC) {
1285 decoder->cyc_ref_timestamp = decoder->timestamp;
1286 decoder->cycle_cnt = 0;
1287 decoder->have_calc_cyc_to_tsc = false;
1288 intel_pt_calc_cyc_to_tsc(decoder, true);
1292 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1294 unsigned int cbr = decoder->packet.payload;
1296 if (decoder->cbr == cbr)
1300 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1303 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1305 uint64_t timestamp = decoder->cyc_ref_timestamp;
1307 decoder->have_cyc = true;
1309 decoder->cycle_cnt += decoder->packet.payload;
1311 if (!decoder->cyc_ref_timestamp)
1314 if (decoder->have_calc_cyc_to_tsc)
1315 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1316 else if (decoder->cbr)
1317 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1321 if (timestamp < decoder->timestamp)
1322 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1323 timestamp, decoder->timestamp);
1325 decoder->timestamp = timestamp;
1328 /* Walk PSB+ packets when already in sync. */
1329 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1334 err = intel_pt_get_next_packet(decoder);
1338 switch (decoder->packet.type) {
1339 case INTEL_PT_PSBEND:
1342 case INTEL_PT_TIP_PGD:
1343 case INTEL_PT_TIP_PGE:
1346 case INTEL_PT_TRACESTOP:
1349 decoder->have_tma = false;
1350 intel_pt_log("ERROR: Unexpected packet\n");
1354 return intel_pt_overflow(decoder);
1357 intel_pt_calc_tsc_timestamp(decoder);
1361 intel_pt_calc_tma(decoder);
1365 intel_pt_calc_cbr(decoder);
1368 case INTEL_PT_MODE_EXEC:
1369 decoder->exec_mode = decoder->packet.payload;
1373 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1377 decoder->pge = true;
1378 intel_pt_set_last_ip(decoder);
1381 case INTEL_PT_MODE_TSX:
1382 intel_pt_update_in_tx(decoder);
1386 intel_pt_calc_mtc_timestamp(decoder);
1387 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1388 decoder->state.type |= INTEL_PT_INSTRUCTION;
1401 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1405 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1406 decoder->tx_flags = 0;
1407 decoder->state.flags &= ~INTEL_PT_IN_TX;
1408 decoder->state.flags |= INTEL_PT_ABORT_TX;
1410 decoder->state.flags |= INTEL_PT_ASYNC;
1414 err = intel_pt_get_next_packet(decoder);
1418 switch (decoder->packet.type) {
1421 case INTEL_PT_TRACESTOP:
1426 case INTEL_PT_MODE_TSX:
1428 case INTEL_PT_PSBEND:
1429 intel_pt_log("ERROR: Missing TIP after FUP\n");
1430 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1434 return intel_pt_overflow(decoder);
1436 case INTEL_PT_TIP_PGD:
1437 decoder->state.from_ip = decoder->ip;
1438 decoder->state.to_ip = 0;
1439 if (decoder->packet.count != 0) {
1440 intel_pt_set_ip(decoder);
1441 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1444 decoder->pge = false;
1445 decoder->continuous_period = false;
1448 case INTEL_PT_TIP_PGE:
1449 decoder->pge = true;
1450 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1452 decoder->state.from_ip = 0;
1453 if (decoder->packet.count == 0) {
1454 decoder->state.to_ip = 0;
1456 intel_pt_set_ip(decoder);
1457 decoder->state.to_ip = decoder->ip;
1462 decoder->state.from_ip = decoder->ip;
1463 if (decoder->packet.count == 0) {
1464 decoder->state.to_ip = 0;
1466 intel_pt_set_ip(decoder);
1467 decoder->state.to_ip = decoder->ip;
1472 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1476 intel_pt_calc_mtc_timestamp(decoder);
1477 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1478 decoder->state.type |= INTEL_PT_INSTRUCTION;
1482 intel_pt_calc_cyc_timestamp(decoder);
1485 case INTEL_PT_MODE_EXEC:
1486 decoder->exec_mode = decoder->packet.payload;
1495 return intel_pt_bug(decoder);
1500 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1502 bool no_tip = false;
1506 err = intel_pt_get_next_packet(decoder);
1510 switch (decoder->packet.type) {
1512 if (!decoder->packet.count)
1514 decoder->tnt = decoder->packet;
1515 decoder->pkt_state = INTEL_PT_STATE_TNT;
1516 err = intel_pt_walk_tnt(decoder);
1521 case INTEL_PT_TIP_PGD:
1522 if (decoder->packet.count != 0)
1523 intel_pt_set_last_ip(decoder);
1524 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1525 return intel_pt_walk_tip(decoder);
1527 case INTEL_PT_TIP_PGE: {
1528 decoder->pge = true;
1529 if (decoder->packet.count == 0) {
1530 intel_pt_log_at("Skipping zero TIP.PGE",
1534 intel_pt_set_ip(decoder);
1535 decoder->state.from_ip = 0;
1536 decoder->state.to_ip = decoder->ip;
1541 return intel_pt_overflow(decoder);
1544 if (decoder->packet.count != 0)
1545 intel_pt_set_last_ip(decoder);
1546 decoder->pkt_state = INTEL_PT_STATE_TIP;
1547 return intel_pt_walk_tip(decoder);
1550 if (decoder->packet.count == 0) {
1551 intel_pt_log_at("Skipping zero FUP",
1556 intel_pt_set_last_ip(decoder);
1557 err = intel_pt_walk_fup(decoder);
1558 if (err != -EAGAIN) {
1562 decoder->pkt_state =
1563 INTEL_PT_STATE_FUP_NO_TIP;
1565 decoder->pkt_state = INTEL_PT_STATE_FUP;
1572 return intel_pt_walk_fup_tip(decoder);
1574 case INTEL_PT_TRACESTOP:
1575 decoder->pge = false;
1576 decoder->continuous_period = false;
1577 intel_pt_clear_tx_flags(decoder);
1578 decoder->have_tma = false;
1582 intel_pt_clear_stack(&decoder->stack);
1583 err = intel_pt_walk_psbend(decoder);
1591 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1595 intel_pt_calc_mtc_timestamp(decoder);
1596 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1599 * Ensure that there has been an instruction since the
1602 if (!decoder->mtc_insn)
1604 decoder->mtc_insn = false;
1605 /* Ensure that there is a timestamp */
1606 if (!decoder->timestamp)
1608 decoder->state.type = INTEL_PT_INSTRUCTION;
1609 decoder->state.from_ip = decoder->ip;
1610 decoder->state.to_ip = 0;
1611 decoder->mtc_insn = false;
1615 intel_pt_calc_tsc_timestamp(decoder);
1619 intel_pt_calc_tma(decoder);
1623 intel_pt_calc_cyc_timestamp(decoder);
1627 intel_pt_calc_cbr(decoder);
1630 case INTEL_PT_MODE_EXEC:
1631 decoder->exec_mode = decoder->packet.payload;
1634 case INTEL_PT_MODE_TSX:
1635 /* MODE_TSX need not be followed by FUP */
1636 if (!decoder->pge) {
1637 intel_pt_update_in_tx(decoder);
1640 err = intel_pt_mode_tsx(decoder, &no_tip);
1645 case INTEL_PT_BAD: /* Does not happen */
1646 return intel_pt_bug(decoder);
1648 case INTEL_PT_PSBEND:
1655 return intel_pt_bug(decoder);
1660 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1662 return decoder->last_ip || decoder->packet.count == 0 ||
1663 decoder->packet.count == 3 || decoder->packet.count == 6;
1666 /* Walk PSB+ packets to get in sync. */
1667 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1672 err = intel_pt_get_next_packet(decoder);
1676 switch (decoder->packet.type) {
1677 case INTEL_PT_TIP_PGD:
1678 decoder->continuous_period = false;
1679 case INTEL_PT_TIP_PGE:
1681 intel_pt_log("ERROR: Unexpected packet\n");
1685 decoder->pge = true;
1686 if (intel_pt_have_ip(decoder)) {
1687 uint64_t current_ip = decoder->ip;
1689 intel_pt_set_ip(decoder);
1691 intel_pt_log_to("Setting IP",
1697 intel_pt_calc_mtc_timestamp(decoder);
1701 intel_pt_calc_tsc_timestamp(decoder);
1705 intel_pt_calc_tma(decoder);
1709 intel_pt_calc_cyc_timestamp(decoder);
1713 intel_pt_calc_cbr(decoder);
1717 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1720 case INTEL_PT_MODE_EXEC:
1721 decoder->exec_mode = decoder->packet.payload;
1724 case INTEL_PT_MODE_TSX:
1725 intel_pt_update_in_tx(decoder);
1728 case INTEL_PT_TRACESTOP:
1729 decoder->pge = false;
1730 decoder->continuous_period = false;
1731 intel_pt_clear_tx_flags(decoder);
1733 decoder->have_tma = false;
1734 intel_pt_log("ERROR: Unexpected packet\n");
1736 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1738 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1741 case INTEL_PT_BAD: /* Does not happen */
1742 return intel_pt_bug(decoder);
1745 return intel_pt_overflow(decoder);
1747 case INTEL_PT_PSBEND:
1760 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1765 err = intel_pt_get_next_packet(decoder);
1769 switch (decoder->packet.type) {
1770 case INTEL_PT_TIP_PGD:
1771 decoder->continuous_period = false;
1772 case INTEL_PT_TIP_PGE:
1774 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1775 if (intel_pt_have_ip(decoder))
1776 intel_pt_set_ip(decoder);
1782 if (decoder->overflow) {
1783 if (intel_pt_have_ip(decoder))
1784 intel_pt_set_ip(decoder);
1788 if (decoder->packet.count)
1789 intel_pt_set_last_ip(decoder);
1793 intel_pt_calc_mtc_timestamp(decoder);
1797 intel_pt_calc_tsc_timestamp(decoder);
1801 intel_pt_calc_tma(decoder);
1805 intel_pt_calc_cyc_timestamp(decoder);
1809 intel_pt_calc_cbr(decoder);
1813 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1816 case INTEL_PT_MODE_EXEC:
1817 decoder->exec_mode = decoder->packet.payload;
1820 case INTEL_PT_MODE_TSX:
1821 intel_pt_update_in_tx(decoder);
1825 return intel_pt_overflow(decoder);
1827 case INTEL_PT_BAD: /* Does not happen */
1828 return intel_pt_bug(decoder);
1830 case INTEL_PT_TRACESTOP:
1831 decoder->pge = false;
1832 decoder->continuous_period = false;
1833 intel_pt_clear_tx_flags(decoder);
1834 decoder->have_tma = false;
1838 err = intel_pt_walk_psb(decoder);
1842 /* Do not have a sample */
1843 decoder->state.type = 0;
1849 case INTEL_PT_PSBEND:
1859 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1863 intel_pt_log("Scanning for full IP\n");
1864 err = intel_pt_walk_to_ip(decoder);
1868 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1869 decoder->overflow = false;
1871 decoder->state.from_ip = 0;
1872 decoder->state.to_ip = decoder->ip;
1873 intel_pt_log_to("Setting IP", decoder->ip);
1878 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1880 const unsigned char *end = decoder->buf + decoder->len;
1883 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1884 if (i > decoder->len)
1886 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1892 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1894 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1895 const char *psb = INTEL_PT_PSB_STR;
1897 if (rest_psb > decoder->len ||
1898 memcmp(decoder->buf, psb + part_psb, rest_psb))
1904 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1909 decoder->pos += decoder->len;
1912 ret = intel_pt_get_next_data(decoder);
1916 rest_psb = intel_pt_rest_psb(decoder, part_psb);
1920 decoder->pos -= part_psb;
1921 decoder->next_buf = decoder->buf + rest_psb;
1922 decoder->next_len = decoder->len - rest_psb;
1923 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1924 decoder->buf = decoder->temp_buf;
1925 decoder->len = INTEL_PT_PSB_LEN;
1930 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
1932 unsigned char *next;
1935 intel_pt_log("Scanning for PSB\n");
1937 if (!decoder->len) {
1938 ret = intel_pt_get_next_data(decoder);
1943 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
1948 part_psb = intel_pt_part_psb(decoder);
1950 ret = intel_pt_get_split_psb(decoder, part_psb);
1954 decoder->pos += decoder->len;
1960 decoder->pkt_step = next - decoder->buf;
1961 return intel_pt_get_next_packet(decoder);
1965 static int intel_pt_sync(struct intel_pt_decoder *decoder)
1969 decoder->pge = false;
1970 decoder->continuous_period = false;
1971 decoder->last_ip = 0;
1973 intel_pt_clear_stack(&decoder->stack);
1975 err = intel_pt_scan_for_psb(decoder);
1979 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
1981 err = intel_pt_walk_psb(decoder);
1986 decoder->state.type = 0; /* Do not have a sample */
1987 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1989 return intel_pt_sync_ip(decoder);
1995 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
1997 uint64_t est = decoder->timestamp_insn_cnt << 1;
1999 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2002 est *= decoder->max_non_turbo_ratio;
2003 est /= decoder->cbr;
2005 return decoder->timestamp + est;
2008 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2013 decoder->state.type = INTEL_PT_BRANCH;
2014 decoder->state.flags = 0;
2016 switch (decoder->pkt_state) {
2017 case INTEL_PT_STATE_NO_PSB:
2018 err = intel_pt_sync(decoder);
2020 case INTEL_PT_STATE_NO_IP:
2021 decoder->last_ip = 0;
2023 case INTEL_PT_STATE_ERR_RESYNC:
2024 err = intel_pt_sync_ip(decoder);
2026 case INTEL_PT_STATE_IN_SYNC:
2027 err = intel_pt_walk_trace(decoder);
2029 case INTEL_PT_STATE_TNT:
2030 err = intel_pt_walk_tnt(decoder);
2032 err = intel_pt_walk_trace(decoder);
2034 case INTEL_PT_STATE_TIP:
2035 case INTEL_PT_STATE_TIP_PGD:
2036 err = intel_pt_walk_tip(decoder);
2038 case INTEL_PT_STATE_FUP:
2039 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2040 err = intel_pt_walk_fup(decoder);
2042 err = intel_pt_walk_fup_tip(decoder);
2044 decoder->pkt_state = INTEL_PT_STATE_FUP;
2046 case INTEL_PT_STATE_FUP_NO_TIP:
2047 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2048 err = intel_pt_walk_fup(decoder);
2050 err = intel_pt_walk_trace(decoder);
2053 err = intel_pt_bug(decoder);
2056 } while (err == -ENOLINK);
2058 decoder->state.err = err ? intel_pt_ext_err(err) : 0;
2059 decoder->state.timestamp = decoder->timestamp;
2060 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2061 decoder->state.cr3 = decoder->cr3;
2062 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2065 decoder->state.from_ip = decoder->ip;
2067 return &decoder->state;
2070 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2072 if (len < INTEL_PT_PSB_LEN)
2074 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2079 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2080 * @buf: pointer to buffer pointer
2081 * @len: size of buffer
2083 * Updates the buffer pointer to point to the start of the next PSB packet if
2084 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2085 * @len is adjusted accordingly.
2087 * Return: %true if a PSB packet is found, %false otherwise.
2089 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2091 unsigned char *next;
2093 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2095 *len -= next - *buf;
2103 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2105 * @buf: pointer to buffer pointer
2106 * @len: size of buffer
2108 * Updates the buffer pointer to point to the start of the following PSB packet
2109 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2110 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2112 * Return: %true if a PSB packet is found, %false otherwise.
2114 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2116 unsigned char *next;
2121 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2123 *len -= next - *buf;
2131 * intel_pt_last_psb - find the last PSB packet in a buffer.
2133 * @len: size of buffer
2135 * This function finds the last PSB in a buffer.
2137 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2139 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2141 const char *n = INTEL_PT_PSB_STR;
2145 if (len < INTEL_PT_PSB_LEN)
2148 k = len - INTEL_PT_PSB_LEN + 1;
2150 p = memrchr(buf, n[0], k);
2153 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2162 * intel_pt_next_tsc - find and return next TSC.
2164 * @len: size of buffer
2165 * @tsc: TSC value returned
2167 * Find a TSC packet in @buf and return the TSC value. This function assumes
2168 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2169 * PSBEND packet is found.
2171 * Return: %true if TSC is found, false otherwise.
2173 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2175 struct intel_pt_pkt packet;
2179 ret = intel_pt_get_packet(buf, len, &packet);
2182 if (packet.type == INTEL_PT_TSC) {
2183 *tsc = packet.payload;
2186 if (packet.type == INTEL_PT_PSBEND)
2195 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2196 * @tsc1: first TSC to compare
2197 * @tsc2: second TSC to compare
2199 * This function compares 7-byte TSC values allowing for the possibility that
2200 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2201 * around so for that purpose this function assumes the absolute difference is
2202 * less than half the maximum difference.
2204 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2207 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2209 const uint64_t halfway = (1ULL << 55);
2215 if (tsc2 - tsc1 < halfway)
2220 if (tsc1 - tsc2 < halfway)
2228 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2230 * @buf_a: first buffer
2231 * @len_a: size of first buffer
2232 * @buf_b: second buffer
2233 * @len_b: size of second buffer
2235 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2236 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2237 * walk forward in @buf_b until a later TSC is found. A precondition is that
2238 * @buf_a and @buf_b are positioned at a PSB.
2240 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2241 * @buf_b + @len_b if there is no non-overlapped data.
2243 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2245 unsigned char *buf_b,
2248 uint64_t tsc_a, tsc_b;
2252 p = intel_pt_last_psb(buf_a, len_a);
2254 return buf_b; /* No PSB in buf_a => no overlap */
2256 len = len_a - (p - buf_a);
2257 if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2258 /* The last PSB+ in buf_a is incomplete, so go back one more */
2260 p = intel_pt_last_psb(buf_a, len_a);
2262 return buf_b; /* No full PSB+ => assume no overlap */
2263 len = len_a - (p - buf_a);
2264 if (!intel_pt_next_tsc(p, len, &tsc_a))
2265 return buf_b; /* No TSC in buf_a => assume no overlap */
2269 /* Ignore PSB+ with no TSC */
2270 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2271 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2272 return buf_b; /* tsc_a < tsc_b => no overlap */
2274 if (!intel_pt_step_psb(&buf_b, &len_b))
2275 return buf_b + len_b; /* No PSB in buf_b => no data */
2280 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2281 * @buf_a: first buffer
2282 * @len_a: size of first buffer
2283 * @buf_b: second buffer
2284 * @len_b: size of second buffer
2285 * @have_tsc: can use TSC packets to detect overlap
2287 * When trace samples or snapshots are recorded there is the possibility that
2288 * the data overlaps. Note that, for the purposes of decoding, data is only
2289 * useful if it begins with a PSB packet.
2291 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2292 * @buf_b + @len_b if there is no non-overlapped data.
2294 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2295 unsigned char *buf_b, size_t len_b,
2298 unsigned char *found;
2300 /* Buffer 'b' must start at PSB so throw away everything before that */
2301 if (!intel_pt_next_psb(&buf_b, &len_b))
2302 return buf_b + len_b; /* No PSB */
2304 if (!intel_pt_next_psb(&buf_a, &len_a))
2305 return buf_b; /* No overlap */
2308 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2314 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2315 * we can ignore the first part of buffer 'a'.
2317 while (len_b < len_a) {
2318 if (!intel_pt_step_psb(&buf_a, &len_a))
2319 return buf_b; /* No overlap */
2322 /* Now len_b >= len_a */
2323 if (len_b > len_a) {
2324 /* The leftover buffer 'b' must start at a PSB */
2325 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2326 if (!intel_pt_step_psb(&buf_a, &len_a))
2327 return buf_b; /* No overlap */
2332 /* Potential overlap so check the bytes */
2333 found = memmem(buf_a, len_a, buf_b, len_a);
2335 return buf_b + len_a;
2337 /* Try again at next PSB in buffer 'a' */
2338 if (!intel_pt_step_psb(&buf_a, &len_a))
2339 return buf_b; /* No overlap */
2341 /* The leftover buffer 'b' must start at a PSB */
2342 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2343 if (!intel_pt_step_psb(&buf_a, &len_a))
2344 return buf_b; /* No overlap */