1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * intel_pt_decoder.c: Intel Processor Trace support
4  * Copyright (c) 2013-2014, Intel Corporation.
5  */
6 
7 #ifndef _GNU_SOURCE
8 #define _GNU_SOURCE
9 #endif
10 #include <stdlib.h>
11 #include <stdbool.h>
12 #include <string.h>
13 #include <errno.h>
14 #include <stdint.h>
15 #include <inttypes.h>
16 #include <linux/compiler.h>
17 #include <linux/string.h>
18 #include <linux/zalloc.h>
19 
20 #include "../auxtrace.h"
21 
22 #include "intel-pt-insn-decoder.h"
23 #include "intel-pt-pkt-decoder.h"
24 #include "intel-pt-decoder.h"
25 #include "intel-pt-log.h"
26 
27 #define INTEL_PT_BLK_SIZE 1024
28 
29 #define BIT63 (((uint64_t)1 << 63))
30 
31 #define INTEL_PT_RETURN 1
32 
33 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
34 #define INTEL_PT_MAX_LOOPS 10000
35 
36 struct intel_pt_blk {
37 	struct intel_pt_blk *prev;
38 	uint64_t ip[INTEL_PT_BLK_SIZE];
39 };
40 
41 struct intel_pt_stack {
42 	struct intel_pt_blk *blk;
43 	struct intel_pt_blk *spare;
44 	int pos;
45 };
46 
47 enum intel_pt_pkt_state {
48 	INTEL_PT_STATE_NO_PSB,
49 	INTEL_PT_STATE_NO_IP,
50 	INTEL_PT_STATE_ERR_RESYNC,
51 	INTEL_PT_STATE_IN_SYNC,
52 	INTEL_PT_STATE_TNT_CONT,
53 	INTEL_PT_STATE_TNT,
54 	INTEL_PT_STATE_TIP,
55 	INTEL_PT_STATE_TIP_PGD,
56 	INTEL_PT_STATE_FUP,
57 	INTEL_PT_STATE_FUP_NO_TIP,
58 };
59 
60 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
61 {
62 	switch (pkt_state) {
63 	case INTEL_PT_STATE_NO_PSB:
64 	case INTEL_PT_STATE_NO_IP:
65 	case INTEL_PT_STATE_ERR_RESYNC:
66 	case INTEL_PT_STATE_IN_SYNC:
67 	case INTEL_PT_STATE_TNT_CONT:
68 		return true;
69 	case INTEL_PT_STATE_TNT:
70 	case INTEL_PT_STATE_TIP:
71 	case INTEL_PT_STATE_TIP_PGD:
72 	case INTEL_PT_STATE_FUP:
73 	case INTEL_PT_STATE_FUP_NO_TIP:
74 		return false;
75 	default:
76 		return true;
77 	};
78 }
79 
80 #ifdef INTEL_PT_STRICT
81 #define INTEL_PT_STATE_ERR1	INTEL_PT_STATE_NO_PSB
82 #define INTEL_PT_STATE_ERR2	INTEL_PT_STATE_NO_PSB
83 #define INTEL_PT_STATE_ERR3	INTEL_PT_STATE_NO_PSB
84 #define INTEL_PT_STATE_ERR4	INTEL_PT_STATE_NO_PSB
85 #else
86 #define INTEL_PT_STATE_ERR1	(decoder->pkt_state)
87 #define INTEL_PT_STATE_ERR2	INTEL_PT_STATE_NO_IP
88 #define INTEL_PT_STATE_ERR3	INTEL_PT_STATE_ERR_RESYNC
89 #define INTEL_PT_STATE_ERR4	INTEL_PT_STATE_IN_SYNC
90 #endif
91 
92 struct intel_pt_decoder {
93 	int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
94 	int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
95 			 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
96 			 uint64_t max_insn_cnt, void *data);
97 	bool (*pgd_ip)(uint64_t ip, void *data);
98 	int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
99 	void *data;
100 	struct intel_pt_state state;
101 	const unsigned char *buf;
102 	size_t len;
103 	bool return_compression;
104 	bool branch_enable;
105 	bool mtc_insn;
106 	bool pge;
107 	bool have_tma;
108 	bool have_cyc;
109 	bool fixup_last_mtc;
110 	bool have_last_ip;
111 	bool in_psb;
112 	enum intel_pt_param_flags flags;
113 	uint64_t pos;
114 	uint64_t last_ip;
115 	uint64_t ip;
116 	uint64_t cr3;
117 	uint64_t timestamp;
118 	uint64_t tsc_timestamp;
119 	uint64_t ref_timestamp;
120 	uint64_t buf_timestamp;
121 	uint64_t sample_timestamp;
122 	uint64_t ret_addr;
123 	uint64_t ctc_timestamp;
124 	uint64_t ctc_delta;
125 	uint64_t cycle_cnt;
126 	uint64_t cyc_ref_timestamp;
127 	uint32_t last_mtc;
128 	uint32_t tsc_ctc_ratio_n;
129 	uint32_t tsc_ctc_ratio_d;
130 	uint32_t tsc_ctc_mult;
131 	uint32_t tsc_slip;
132 	uint32_t ctc_rem_mask;
133 	int mtc_shift;
134 	struct intel_pt_stack stack;
135 	enum intel_pt_pkt_state pkt_state;
136 	enum intel_pt_pkt_ctx pkt_ctx;
137 	enum intel_pt_pkt_ctx prev_pkt_ctx;
138 	enum intel_pt_blk_type blk_type;
139 	int blk_type_pos;
140 	struct intel_pt_pkt packet;
141 	struct intel_pt_pkt tnt;
142 	int pkt_step;
143 	int pkt_len;
144 	int last_packet_type;
145 	unsigned int cbr;
146 	unsigned int cbr_seen;
147 	unsigned int max_non_turbo_ratio;
148 	double max_non_turbo_ratio_fp;
149 	double cbr_cyc_to_tsc;
150 	double calc_cyc_to_tsc;
151 	bool have_calc_cyc_to_tsc;
152 	int exec_mode;
153 	unsigned int insn_bytes;
154 	uint64_t period;
155 	enum intel_pt_period_type period_type;
156 	uint64_t tot_insn_cnt;
157 	uint64_t period_insn_cnt;
158 	uint64_t period_mask;
159 	uint64_t period_ticks;
160 	uint64_t last_masked_timestamp;
161 	uint64_t tot_cyc_cnt;
162 	uint64_t sample_tot_cyc_cnt;
163 	uint64_t base_cyc_cnt;
164 	uint64_t cyc_cnt_timestamp;
165 	double tsc_to_cyc;
166 	bool continuous_period;
167 	bool overflow;
168 	bool set_fup_tx_flags;
169 	bool set_fup_ptw;
170 	bool set_fup_mwait;
171 	bool set_fup_pwre;
172 	bool set_fup_exstop;
173 	bool set_fup_bep;
174 	bool sample_cyc;
175 	unsigned int fup_tx_flags;
176 	unsigned int tx_flags;
177 	uint64_t fup_ptw_payload;
178 	uint64_t fup_mwait_payload;
179 	uint64_t fup_pwre_payload;
180 	uint64_t cbr_payload;
181 	uint64_t timestamp_insn_cnt;
182 	uint64_t sample_insn_cnt;
183 	uint64_t stuck_ip;
184 	int no_progress;
185 	int stuck_ip_prd;
186 	int stuck_ip_cnt;
187 	const unsigned char *next_buf;
188 	size_t next_len;
189 	unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
190 };
191 
192 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
193 {
194 	int i;
195 
196 	for (i = 0; x != 1; i++)
197 		x >>= 1;
198 
199 	return x << i;
200 }
201 
202 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
203 {
204 	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
205 		uint64_t period;
206 
207 		period = intel_pt_lower_power_of_2(decoder->period);
208 		decoder->period_mask  = ~(period - 1);
209 		decoder->period_ticks = period;
210 	}
211 }
212 
213 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
214 {
215 	if (!d)
216 		return 0;
217 	return (t / d) * n + ((t % d) * n) / d;
218 }
219 
220 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
221 {
222 	struct intel_pt_decoder *decoder;
223 
224 	if (!params->get_trace || !params->walk_insn)
225 		return NULL;
226 
227 	decoder = zalloc(sizeof(struct intel_pt_decoder));
228 	if (!decoder)
229 		return NULL;
230 
231 	decoder->get_trace          = params->get_trace;
232 	decoder->walk_insn          = params->walk_insn;
233 	decoder->pgd_ip             = params->pgd_ip;
234 	decoder->lookahead          = params->lookahead;
235 	decoder->data               = params->data;
236 	decoder->return_compression = params->return_compression;
237 	decoder->branch_enable      = params->branch_enable;
238 
239 	decoder->flags              = params->flags;
240 
241 	decoder->period             = params->period;
242 	decoder->period_type        = params->period_type;
243 
244 	decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
245 	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
246 
247 	intel_pt_setup_period(decoder);
248 
249 	decoder->mtc_shift = params->mtc_period;
250 	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
251 
252 	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
253 	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
254 
255 	if (!decoder->tsc_ctc_ratio_n)
256 		decoder->tsc_ctc_ratio_d = 0;
257 
258 	if (decoder->tsc_ctc_ratio_d) {
259 		if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
260 			decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
261 						decoder->tsc_ctc_ratio_d;
262 	}
263 
264 	/*
265 	 * A TSC packet can slip past MTC packets so that the timestamp appears
266 	 * to go backwards. One estimate is that can be up to about 40 CPU
267 	 * cycles, which is certainly less than 0x1000 TSC ticks, but accept
268 	 * slippage an order of magnitude more to be on the safe side.
269 	 */
270 	decoder->tsc_slip = 0x10000;
271 
272 	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
273 	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
274 	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
275 	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
276 	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
277 
278 	return decoder;
279 }
280 
281 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
282 {
283 	struct intel_pt_blk *blk = stack->blk;
284 
285 	stack->blk = blk->prev;
286 	if (!stack->spare)
287 		stack->spare = blk;
288 	else
289 		free(blk);
290 }
291 
292 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
293 {
294 	if (!stack->pos) {
295 		if (!stack->blk)
296 			return 0;
297 		intel_pt_pop_blk(stack);
298 		if (!stack->blk)
299 			return 0;
300 		stack->pos = INTEL_PT_BLK_SIZE;
301 	}
302 	return stack->blk->ip[--stack->pos];
303 }
304 
305 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
306 {
307 	struct intel_pt_blk *blk;
308 
309 	if (stack->spare) {
310 		blk = stack->spare;
311 		stack->spare = NULL;
312 	} else {
313 		blk = malloc(sizeof(struct intel_pt_blk));
314 		if (!blk)
315 			return -ENOMEM;
316 	}
317 
318 	blk->prev = stack->blk;
319 	stack->blk = blk;
320 	stack->pos = 0;
321 	return 0;
322 }
323 
324 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
325 {
326 	int err;
327 
328 	if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
329 		err = intel_pt_alloc_blk(stack);
330 		if (err)
331 			return err;
332 	}
333 
334 	stack->blk->ip[stack->pos++] = ip;
335 	return 0;
336 }
337 
338 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
339 {
340 	while (stack->blk)
341 		intel_pt_pop_blk(stack);
342 	stack->pos = 0;
343 }
344 
345 static void intel_pt_free_stack(struct intel_pt_stack *stack)
346 {
347 	intel_pt_clear_stack(stack);
348 	zfree(&stack->blk);
349 	zfree(&stack->spare);
350 }
351 
352 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
353 {
354 	intel_pt_free_stack(&decoder->stack);
355 	free(decoder);
356 }
357 
358 static int intel_pt_ext_err(int code)
359 {
360 	switch (code) {
361 	case -ENOMEM:
362 		return INTEL_PT_ERR_NOMEM;
363 	case -ENOSYS:
364 		return INTEL_PT_ERR_INTERN;
365 	case -EBADMSG:
366 		return INTEL_PT_ERR_BADPKT;
367 	case -ENODATA:
368 		return INTEL_PT_ERR_NODATA;
369 	case -EILSEQ:
370 		return INTEL_PT_ERR_NOINSN;
371 	case -ENOENT:
372 		return INTEL_PT_ERR_MISMAT;
373 	case -EOVERFLOW:
374 		return INTEL_PT_ERR_OVR;
375 	case -ENOSPC:
376 		return INTEL_PT_ERR_LOST;
377 	case -ELOOP:
378 		return INTEL_PT_ERR_NELOOP;
379 	default:
380 		return INTEL_PT_ERR_UNK;
381 	}
382 }
383 
384 static const char *intel_pt_err_msgs[] = {
385 	[INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
386 	[INTEL_PT_ERR_INTERN] = "Internal error",
387 	[INTEL_PT_ERR_BADPKT] = "Bad packet",
388 	[INTEL_PT_ERR_NODATA] = "No more data",
389 	[INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
390 	[INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
391 	[INTEL_PT_ERR_OVR]    = "Overflow packet",
392 	[INTEL_PT_ERR_LOST]   = "Lost trace data",
393 	[INTEL_PT_ERR_UNK]    = "Unknown error!",
394 	[INTEL_PT_ERR_NELOOP] = "Never-ending loop",
395 };
396 
397 int intel_pt__strerror(int code, char *buf, size_t buflen)
398 {
399 	if (code < 1 || code >= INTEL_PT_ERR_MAX)
400 		code = INTEL_PT_ERR_UNK;
401 	strlcpy(buf, intel_pt_err_msgs[code], buflen);
402 	return 0;
403 }
404 
405 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
406 				 uint64_t last_ip)
407 {
408 	uint64_t ip;
409 
410 	switch (packet->count) {
411 	case 1:
412 		ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
413 		     packet->payload;
414 		break;
415 	case 2:
416 		ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
417 		     packet->payload;
418 		break;
419 	case 3:
420 		ip = packet->payload;
421 		/* Sign-extend 6-byte ip */
422 		if (ip & (uint64_t)0x800000000000ULL)
423 			ip |= (uint64_t)0xffff000000000000ULL;
424 		break;
425 	case 4:
426 		ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
427 		     packet->payload;
428 		break;
429 	case 6:
430 		ip = packet->payload;
431 		break;
432 	default:
433 		return 0;
434 	}
435 
436 	return ip;
437 }
438 
439 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
440 {
441 	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
442 	decoder->have_last_ip = true;
443 }
444 
445 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
446 {
447 	intel_pt_set_last_ip(decoder);
448 	decoder->ip = decoder->last_ip;
449 }
450 
451 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
452 {
453 	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
454 			    decoder->buf);
455 }
456 
457 static int intel_pt_bug(struct intel_pt_decoder *decoder)
458 {
459 	intel_pt_log("ERROR: Internal error\n");
460 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
461 	return -ENOSYS;
462 }
463 
464 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
465 {
466 	decoder->tx_flags = 0;
467 }
468 
469 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
470 {
471 	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
472 }
473 
474 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
475 {
476 	intel_pt_clear_tx_flags(decoder);
477 	decoder->have_tma = false;
478 	decoder->pkt_len = 1;
479 	decoder->pkt_step = 1;
480 	intel_pt_decoder_log_packet(decoder);
481 	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
482 		intel_pt_log("ERROR: Bad packet\n");
483 		decoder->pkt_state = INTEL_PT_STATE_ERR1;
484 	}
485 	return -EBADMSG;
486 }
487 
488 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
489 {
490 	decoder->sample_timestamp = decoder->timestamp;
491 	decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
492 }
493 
494 static void intel_pt_reposition(struct intel_pt_decoder *decoder)
495 {
496 	decoder->ip = 0;
497 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
498 	decoder->timestamp = 0;
499 	decoder->have_tma = false;
500 }
501 
502 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
503 {
504 	struct intel_pt_buffer buffer = { .buf = 0, };
505 	int ret;
506 
507 	decoder->pkt_step = 0;
508 
509 	intel_pt_log("Getting more data\n");
510 	ret = decoder->get_trace(&buffer, decoder->data);
511 	if (ret)
512 		return ret;
513 	decoder->buf = buffer.buf;
514 	decoder->len = buffer.len;
515 	if (!decoder->len) {
516 		intel_pt_log("No more data\n");
517 		return -ENODATA;
518 	}
519 	decoder->buf_timestamp = buffer.ref_timestamp;
520 	if (!buffer.consecutive || reposition) {
521 		intel_pt_reposition(decoder);
522 		decoder->ref_timestamp = buffer.ref_timestamp;
523 		decoder->state.trace_nr = buffer.trace_nr;
524 		intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
525 			     decoder->ref_timestamp);
526 		return -ENOLINK;
527 	}
528 
529 	return 0;
530 }
531 
532 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
533 				  bool reposition)
534 {
535 	if (!decoder->next_buf)
536 		return intel_pt_get_data(decoder, reposition);
537 
538 	decoder->buf = decoder->next_buf;
539 	decoder->len = decoder->next_len;
540 	decoder->next_buf = 0;
541 	decoder->next_len = 0;
542 	return 0;
543 }
544 
545 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
546 {
547 	unsigned char *buf = decoder->temp_buf;
548 	size_t old_len, len, n;
549 	int ret;
550 
551 	old_len = decoder->len;
552 	len = decoder->len;
553 	memcpy(buf, decoder->buf, len);
554 
555 	ret = intel_pt_get_data(decoder, false);
556 	if (ret) {
557 		decoder->pos += old_len;
558 		return ret < 0 ? ret : -EINVAL;
559 	}
560 
561 	n = INTEL_PT_PKT_MAX_SZ - len;
562 	if (n > decoder->len)
563 		n = decoder->len;
564 	memcpy(buf + len, decoder->buf, n);
565 	len += n;
566 
567 	decoder->prev_pkt_ctx = decoder->pkt_ctx;
568 	ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
569 	if (ret < (int)old_len) {
570 		decoder->next_buf = decoder->buf;
571 		decoder->next_len = decoder->len;
572 		decoder->buf = buf;
573 		decoder->len = old_len;
574 		return intel_pt_bad_packet(decoder);
575 	}
576 
577 	decoder->next_buf = decoder->buf + (ret - old_len);
578 	decoder->next_len = decoder->len - (ret - old_len);
579 
580 	decoder->buf = buf;
581 	decoder->len = ret;
582 
583 	return ret;
584 }
585 
586 struct intel_pt_pkt_info {
587 	struct intel_pt_decoder	  *decoder;
588 	struct intel_pt_pkt       packet;
589 	uint64_t                  pos;
590 	int                       pkt_len;
591 	int                       last_packet_type;
592 	void                      *data;
593 };
594 
595 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
596 
597 /* Lookahead packets in current buffer */
598 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
599 				  intel_pt_pkt_cb_t cb, void *data)
600 {
601 	struct intel_pt_pkt_info pkt_info;
602 	const unsigned char *buf = decoder->buf;
603 	enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
604 	size_t len = decoder->len;
605 	int ret;
606 
607 	pkt_info.decoder          = decoder;
608 	pkt_info.pos              = decoder->pos;
609 	pkt_info.pkt_len          = decoder->pkt_step;
610 	pkt_info.last_packet_type = decoder->last_packet_type;
611 	pkt_info.data             = data;
612 
613 	while (1) {
614 		do {
615 			pkt_info.pos += pkt_info.pkt_len;
616 			buf          += pkt_info.pkt_len;
617 			len          -= pkt_info.pkt_len;
618 
619 			if (!len)
620 				return INTEL_PT_NEED_MORE_BYTES;
621 
622 			ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
623 						  &pkt_ctx);
624 			if (!ret)
625 				return INTEL_PT_NEED_MORE_BYTES;
626 			if (ret < 0)
627 				return ret;
628 
629 			pkt_info.pkt_len = ret;
630 		} while (pkt_info.packet.type == INTEL_PT_PAD);
631 
632 		ret = cb(&pkt_info);
633 		if (ret)
634 			return 0;
635 
636 		pkt_info.last_packet_type = pkt_info.packet.type;
637 	}
638 }
639 
640 struct intel_pt_calc_cyc_to_tsc_info {
641 	uint64_t        cycle_cnt;
642 	unsigned int    cbr;
643 	uint32_t        last_mtc;
644 	uint64_t        ctc_timestamp;
645 	uint64_t        ctc_delta;
646 	uint64_t        tsc_timestamp;
647 	uint64_t        timestamp;
648 	bool            have_tma;
649 	bool            fixup_last_mtc;
650 	bool            from_mtc;
651 	double          cbr_cyc_to_tsc;
652 };
653 
654 /*
655  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
656  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
657  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
658  * packet by copying the missing bits from the current MTC assuming the least
659  * difference between the two, and that the current MTC comes after last_mtc.
660  */
661 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
662 				    uint32_t *last_mtc)
663 {
664 	uint32_t first_missing_bit = 1U << (16 - mtc_shift);
665 	uint32_t mask = ~(first_missing_bit - 1);
666 
667 	*last_mtc |= mtc & mask;
668 	if (*last_mtc >= mtc) {
669 		*last_mtc -= first_missing_bit;
670 		*last_mtc &= 0xff;
671 	}
672 }
673 
674 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
675 {
676 	struct intel_pt_decoder *decoder = pkt_info->decoder;
677 	struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
678 	uint64_t timestamp;
679 	double cyc_to_tsc;
680 	unsigned int cbr;
681 	uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
682 
683 	switch (pkt_info->packet.type) {
684 	case INTEL_PT_TNT:
685 	case INTEL_PT_TIP_PGE:
686 	case INTEL_PT_TIP:
687 	case INTEL_PT_FUP:
688 	case INTEL_PT_PSB:
689 	case INTEL_PT_PIP:
690 	case INTEL_PT_MODE_EXEC:
691 	case INTEL_PT_MODE_TSX:
692 	case INTEL_PT_PSBEND:
693 	case INTEL_PT_PAD:
694 	case INTEL_PT_VMCS:
695 	case INTEL_PT_MNT:
696 	case INTEL_PT_PTWRITE:
697 	case INTEL_PT_PTWRITE_IP:
698 	case INTEL_PT_BBP:
699 	case INTEL_PT_BIP:
700 	case INTEL_PT_BEP:
701 	case INTEL_PT_BEP_IP:
702 		return 0;
703 
704 	case INTEL_PT_MTC:
705 		if (!data->have_tma)
706 			return 0;
707 
708 		mtc = pkt_info->packet.payload;
709 		if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
710 			data->fixup_last_mtc = false;
711 			intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
712 						&data->last_mtc);
713 		}
714 		if (mtc > data->last_mtc)
715 			mtc_delta = mtc - data->last_mtc;
716 		else
717 			mtc_delta = mtc + 256 - data->last_mtc;
718 		data->ctc_delta += mtc_delta << decoder->mtc_shift;
719 		data->last_mtc = mtc;
720 
721 		if (decoder->tsc_ctc_mult) {
722 			timestamp = data->ctc_timestamp +
723 				data->ctc_delta * decoder->tsc_ctc_mult;
724 		} else {
725 			timestamp = data->ctc_timestamp +
726 				multdiv(data->ctc_delta,
727 					decoder->tsc_ctc_ratio_n,
728 					decoder->tsc_ctc_ratio_d);
729 		}
730 
731 		if (timestamp < data->timestamp)
732 			return 1;
733 
734 		if (pkt_info->last_packet_type != INTEL_PT_CYC) {
735 			data->timestamp = timestamp;
736 			return 0;
737 		}
738 
739 		break;
740 
741 	case INTEL_PT_TSC:
742 		/*
743 		 * For now, do not support using TSC packets - refer
744 		 * intel_pt_calc_cyc_to_tsc().
745 		 */
746 		if (data->from_mtc)
747 			return 1;
748 		timestamp = pkt_info->packet.payload |
749 			    (data->timestamp & (0xffULL << 56));
750 		if (data->from_mtc && timestamp < data->timestamp &&
751 		    data->timestamp - timestamp < decoder->tsc_slip)
752 			return 1;
753 		if (timestamp < data->timestamp)
754 			timestamp += (1ULL << 56);
755 		if (pkt_info->last_packet_type != INTEL_PT_CYC) {
756 			if (data->from_mtc)
757 				return 1;
758 			data->tsc_timestamp = timestamp;
759 			data->timestamp = timestamp;
760 			return 0;
761 		}
762 		break;
763 
764 	case INTEL_PT_TMA:
765 		if (data->from_mtc)
766 			return 1;
767 
768 		if (!decoder->tsc_ctc_ratio_d)
769 			return 0;
770 
771 		ctc = pkt_info->packet.payload;
772 		fc = pkt_info->packet.count;
773 		ctc_rem = ctc & decoder->ctc_rem_mask;
774 
775 		data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
776 
777 		data->ctc_timestamp = data->tsc_timestamp - fc;
778 		if (decoder->tsc_ctc_mult) {
779 			data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
780 		} else {
781 			data->ctc_timestamp -=
782 				multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
783 					decoder->tsc_ctc_ratio_d);
784 		}
785 
786 		data->ctc_delta = 0;
787 		data->have_tma = true;
788 		data->fixup_last_mtc = true;
789 
790 		return 0;
791 
792 	case INTEL_PT_CYC:
793 		data->cycle_cnt += pkt_info->packet.payload;
794 		return 0;
795 
796 	case INTEL_PT_CBR:
797 		cbr = pkt_info->packet.payload;
798 		if (data->cbr && data->cbr != cbr)
799 			return 1;
800 		data->cbr = cbr;
801 		data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
802 		return 0;
803 
804 	case INTEL_PT_TIP_PGD:
805 	case INTEL_PT_TRACESTOP:
806 	case INTEL_PT_EXSTOP:
807 	case INTEL_PT_EXSTOP_IP:
808 	case INTEL_PT_MWAIT:
809 	case INTEL_PT_PWRE:
810 	case INTEL_PT_PWRX:
811 	case INTEL_PT_OVF:
812 	case INTEL_PT_BAD: /* Does not happen */
813 	default:
814 		return 1;
815 	}
816 
817 	if (!data->cbr && decoder->cbr) {
818 		data->cbr = decoder->cbr;
819 		data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
820 	}
821 
822 	if (!data->cycle_cnt)
823 		return 1;
824 
825 	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
826 
827 	if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
828 	    cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
829 		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
830 			     cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
831 		return 1;
832 	}
833 
834 	decoder->calc_cyc_to_tsc = cyc_to_tsc;
835 	decoder->have_calc_cyc_to_tsc = true;
836 
837 	if (data->cbr) {
838 		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
839 			     cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
840 	} else {
841 		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
842 			     cyc_to_tsc, pkt_info->pos);
843 	}
844 
845 	return 1;
846 }
847 
848 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
849 				     bool from_mtc)
850 {
851 	struct intel_pt_calc_cyc_to_tsc_info data = {
852 		.cycle_cnt      = 0,
853 		.cbr            = 0,
854 		.last_mtc       = decoder->last_mtc,
855 		.ctc_timestamp  = decoder->ctc_timestamp,
856 		.ctc_delta      = decoder->ctc_delta,
857 		.tsc_timestamp  = decoder->tsc_timestamp,
858 		.timestamp      = decoder->timestamp,
859 		.have_tma       = decoder->have_tma,
860 		.fixup_last_mtc = decoder->fixup_last_mtc,
861 		.from_mtc       = from_mtc,
862 		.cbr_cyc_to_tsc = 0,
863 	};
864 
865 	/*
866 	 * For now, do not support using TSC packets for at least the reasons:
867 	 * 1) timing might have stopped
868 	 * 2) TSC packets within PSB+ can slip against CYC packets
869 	 */
870 	if (!from_mtc)
871 		return;
872 
873 	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
874 }
875 
876 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
877 {
878 	int ret;
879 
880 	decoder->last_packet_type = decoder->packet.type;
881 
882 	do {
883 		decoder->pos += decoder->pkt_step;
884 		decoder->buf += decoder->pkt_step;
885 		decoder->len -= decoder->pkt_step;
886 
887 		if (!decoder->len) {
888 			ret = intel_pt_get_next_data(decoder, false);
889 			if (ret)
890 				return ret;
891 		}
892 
893 		decoder->prev_pkt_ctx = decoder->pkt_ctx;
894 		ret = intel_pt_get_packet(decoder->buf, decoder->len,
895 					  &decoder->packet, &decoder->pkt_ctx);
896 		if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
897 		    decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
898 			ret = intel_pt_get_split_packet(decoder);
899 			if (ret < 0)
900 				return ret;
901 		}
902 		if (ret <= 0)
903 			return intel_pt_bad_packet(decoder);
904 
905 		decoder->pkt_len = ret;
906 		decoder->pkt_step = ret;
907 		intel_pt_decoder_log_packet(decoder);
908 	} while (decoder->packet.type == INTEL_PT_PAD);
909 
910 	return 0;
911 }
912 
913 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
914 {
915 	uint64_t timestamp, masked_timestamp;
916 
917 	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
918 	masked_timestamp = timestamp & decoder->period_mask;
919 	if (decoder->continuous_period) {
920 		if (masked_timestamp > decoder->last_masked_timestamp)
921 			return 1;
922 	} else {
923 		timestamp += 1;
924 		masked_timestamp = timestamp & decoder->period_mask;
925 		if (masked_timestamp > decoder->last_masked_timestamp) {
926 			decoder->last_masked_timestamp = masked_timestamp;
927 			decoder->continuous_period = true;
928 		}
929 	}
930 
931 	if (masked_timestamp < decoder->last_masked_timestamp)
932 		return decoder->period_ticks;
933 
934 	return decoder->period_ticks - (timestamp - masked_timestamp);
935 }
936 
937 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
938 {
939 	switch (decoder->period_type) {
940 	case INTEL_PT_PERIOD_INSTRUCTIONS:
941 		return decoder->period - decoder->period_insn_cnt;
942 	case INTEL_PT_PERIOD_TICKS:
943 		return intel_pt_next_period(decoder);
944 	case INTEL_PT_PERIOD_NONE:
945 	case INTEL_PT_PERIOD_MTC:
946 	default:
947 		return 0;
948 	}
949 }
950 
951 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
952 {
953 	uint64_t timestamp, masked_timestamp;
954 
955 	switch (decoder->period_type) {
956 	case INTEL_PT_PERIOD_INSTRUCTIONS:
957 		decoder->period_insn_cnt = 0;
958 		break;
959 	case INTEL_PT_PERIOD_TICKS:
960 		timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
961 		masked_timestamp = timestamp & decoder->period_mask;
962 		if (masked_timestamp > decoder->last_masked_timestamp)
963 			decoder->last_masked_timestamp = masked_timestamp;
964 		else
965 			decoder->last_masked_timestamp += decoder->period_ticks;
966 		break;
967 	case INTEL_PT_PERIOD_NONE:
968 	case INTEL_PT_PERIOD_MTC:
969 	default:
970 		break;
971 	}
972 
973 	decoder->state.type |= INTEL_PT_INSTRUCTION;
974 }
975 
976 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
977 			      struct intel_pt_insn *intel_pt_insn, uint64_t ip)
978 {
979 	uint64_t max_insn_cnt, insn_cnt = 0;
980 	int err;
981 
982 	if (!decoder->mtc_insn)
983 		decoder->mtc_insn = true;
984 
985 	max_insn_cnt = intel_pt_next_sample(decoder);
986 
987 	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
988 				 max_insn_cnt, decoder->data);
989 
990 	decoder->tot_insn_cnt += insn_cnt;
991 	decoder->timestamp_insn_cnt += insn_cnt;
992 	decoder->sample_insn_cnt += insn_cnt;
993 	decoder->period_insn_cnt += insn_cnt;
994 
995 	if (err) {
996 		decoder->no_progress = 0;
997 		decoder->pkt_state = INTEL_PT_STATE_ERR2;
998 		intel_pt_log_at("ERROR: Failed to get instruction",
999 				decoder->ip);
1000 		if (err == -ENOENT)
1001 			return -ENOLINK;
1002 		return -EILSEQ;
1003 	}
1004 
1005 	if (ip && decoder->ip == ip) {
1006 		err = -EAGAIN;
1007 		goto out;
1008 	}
1009 
1010 	if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1011 		intel_pt_sample_insn(decoder);
1012 
1013 	if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1014 		decoder->state.type = INTEL_PT_INSTRUCTION;
1015 		decoder->state.from_ip = decoder->ip;
1016 		decoder->state.to_ip = 0;
1017 		decoder->ip += intel_pt_insn->length;
1018 		err = INTEL_PT_RETURN;
1019 		goto out;
1020 	}
1021 
1022 	if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1023 		/* Zero-length calls are excluded */
1024 		if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1025 		    intel_pt_insn->rel) {
1026 			err = intel_pt_push(&decoder->stack, decoder->ip +
1027 					    intel_pt_insn->length);
1028 			if (err)
1029 				goto out;
1030 		}
1031 	} else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1032 		decoder->ret_addr = intel_pt_pop(&decoder->stack);
1033 	}
1034 
1035 	if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1036 		int cnt = decoder->no_progress++;
1037 
1038 		decoder->state.from_ip = decoder->ip;
1039 		decoder->ip += intel_pt_insn->length +
1040 				intel_pt_insn->rel;
1041 		decoder->state.to_ip = decoder->ip;
1042 		err = INTEL_PT_RETURN;
1043 
1044 		/*
1045 		 * Check for being stuck in a loop.  This can happen if a
1046 		 * decoder error results in the decoder erroneously setting the
1047 		 * ip to an address that is itself in an infinite loop that
1048 		 * consumes no packets.  When that happens, there must be an
1049 		 * unconditional branch.
1050 		 */
1051 		if (cnt) {
1052 			if (cnt == 1) {
1053 				decoder->stuck_ip = decoder->state.to_ip;
1054 				decoder->stuck_ip_prd = 1;
1055 				decoder->stuck_ip_cnt = 1;
1056 			} else if (cnt > INTEL_PT_MAX_LOOPS ||
1057 				   decoder->state.to_ip == decoder->stuck_ip) {
1058 				intel_pt_log_at("ERROR: Never-ending loop",
1059 						decoder->state.to_ip);
1060 				decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1061 				err = -ELOOP;
1062 				goto out;
1063 			} else if (!--decoder->stuck_ip_cnt) {
1064 				decoder->stuck_ip_prd += 1;
1065 				decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1066 				decoder->stuck_ip = decoder->state.to_ip;
1067 			}
1068 		}
1069 		goto out_no_progress;
1070 	}
1071 out:
1072 	decoder->no_progress = 0;
1073 out_no_progress:
1074 	decoder->state.insn_op = intel_pt_insn->op;
1075 	decoder->state.insn_len = intel_pt_insn->length;
1076 	memcpy(decoder->state.insn, intel_pt_insn->buf,
1077 	       INTEL_PT_INSN_BUF_SZ);
1078 
1079 	if (decoder->tx_flags & INTEL_PT_IN_TX)
1080 		decoder->state.flags |= INTEL_PT_IN_TX;
1081 
1082 	return err;
1083 }
1084 
1085 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1086 {
1087 	bool ret = false;
1088 
1089 	if (decoder->set_fup_tx_flags) {
1090 		decoder->set_fup_tx_flags = false;
1091 		decoder->tx_flags = decoder->fup_tx_flags;
1092 		decoder->state.type = INTEL_PT_TRANSACTION;
1093 		decoder->state.from_ip = decoder->ip;
1094 		decoder->state.to_ip = 0;
1095 		decoder->state.flags = decoder->fup_tx_flags;
1096 		return true;
1097 	}
1098 	if (decoder->set_fup_ptw) {
1099 		decoder->set_fup_ptw = false;
1100 		decoder->state.type = INTEL_PT_PTW;
1101 		decoder->state.flags |= INTEL_PT_FUP_IP;
1102 		decoder->state.from_ip = decoder->ip;
1103 		decoder->state.to_ip = 0;
1104 		decoder->state.ptw_payload = decoder->fup_ptw_payload;
1105 		return true;
1106 	}
1107 	if (decoder->set_fup_mwait) {
1108 		decoder->set_fup_mwait = false;
1109 		decoder->state.type = INTEL_PT_MWAIT_OP;
1110 		decoder->state.from_ip = decoder->ip;
1111 		decoder->state.to_ip = 0;
1112 		decoder->state.mwait_payload = decoder->fup_mwait_payload;
1113 		ret = true;
1114 	}
1115 	if (decoder->set_fup_pwre) {
1116 		decoder->set_fup_pwre = false;
1117 		decoder->state.type |= INTEL_PT_PWR_ENTRY;
1118 		decoder->state.type &= ~INTEL_PT_BRANCH;
1119 		decoder->state.from_ip = decoder->ip;
1120 		decoder->state.to_ip = 0;
1121 		decoder->state.pwre_payload = decoder->fup_pwre_payload;
1122 		ret = true;
1123 	}
1124 	if (decoder->set_fup_exstop) {
1125 		decoder->set_fup_exstop = false;
1126 		decoder->state.type |= INTEL_PT_EX_STOP;
1127 		decoder->state.type &= ~INTEL_PT_BRANCH;
1128 		decoder->state.flags |= INTEL_PT_FUP_IP;
1129 		decoder->state.from_ip = decoder->ip;
1130 		decoder->state.to_ip = 0;
1131 		ret = true;
1132 	}
1133 	if (decoder->set_fup_bep) {
1134 		decoder->set_fup_bep = false;
1135 		decoder->state.type |= INTEL_PT_BLK_ITEMS;
1136 		decoder->state.type &= ~INTEL_PT_BRANCH;
1137 		decoder->state.from_ip = decoder->ip;
1138 		decoder->state.to_ip = 0;
1139 		ret = true;
1140 	}
1141 	return ret;
1142 }
1143 
1144 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1145 					  struct intel_pt_insn *intel_pt_insn,
1146 					  uint64_t ip, int err)
1147 {
1148 	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1149 	       intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1150 	       ip == decoder->ip + intel_pt_insn->length;
1151 }
1152 
1153 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1154 {
1155 	struct intel_pt_insn intel_pt_insn;
1156 	uint64_t ip;
1157 	int err;
1158 
1159 	ip = decoder->last_ip;
1160 
1161 	while (1) {
1162 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1163 		if (err == INTEL_PT_RETURN)
1164 			return 0;
1165 		if (err == -EAGAIN ||
1166 		    intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1167 			if (intel_pt_fup_event(decoder))
1168 				return 0;
1169 			return -EAGAIN;
1170 		}
1171 		decoder->set_fup_tx_flags = false;
1172 		if (err)
1173 			return err;
1174 
1175 		if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1176 			intel_pt_log_at("ERROR: Unexpected indirect branch",
1177 					decoder->ip);
1178 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1179 			return -ENOENT;
1180 		}
1181 
1182 		if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1183 			intel_pt_log_at("ERROR: Unexpected conditional branch",
1184 					decoder->ip);
1185 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1186 			return -ENOENT;
1187 		}
1188 
1189 		intel_pt_bug(decoder);
1190 	}
1191 }
1192 
1193 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1194 {
1195 	struct intel_pt_insn intel_pt_insn;
1196 	int err;
1197 
1198 	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1199 	if (err == INTEL_PT_RETURN &&
1200 	    decoder->pgd_ip &&
1201 	    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1202 	    (decoder->state.type & INTEL_PT_BRANCH) &&
1203 	    decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1204 		/* Unconditional branch leaving filter region */
1205 		decoder->no_progress = 0;
1206 		decoder->pge = false;
1207 		decoder->continuous_period = false;
1208 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1209 		decoder->state.type |= INTEL_PT_TRACE_END;
1210 		return 0;
1211 	}
1212 	if (err == INTEL_PT_RETURN)
1213 		return 0;
1214 	if (err)
1215 		return err;
1216 
1217 	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1218 		if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1219 			decoder->pge = false;
1220 			decoder->continuous_period = false;
1221 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1222 			decoder->state.from_ip = decoder->ip;
1223 			if (decoder->packet.count == 0) {
1224 				decoder->state.to_ip = 0;
1225 			} else {
1226 				decoder->state.to_ip = decoder->last_ip;
1227 				decoder->ip = decoder->last_ip;
1228 			}
1229 			decoder->state.type |= INTEL_PT_TRACE_END;
1230 		} else {
1231 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1232 			decoder->state.from_ip = decoder->ip;
1233 			if (decoder->packet.count == 0) {
1234 				decoder->state.to_ip = 0;
1235 			} else {
1236 				decoder->state.to_ip = decoder->last_ip;
1237 				decoder->ip = decoder->last_ip;
1238 			}
1239 		}
1240 		return 0;
1241 	}
1242 
1243 	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1244 		uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1245 				 intel_pt_insn.rel;
1246 
1247 		if (decoder->pgd_ip &&
1248 		    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1249 		    decoder->pgd_ip(to_ip, decoder->data)) {
1250 			/* Conditional branch leaving filter region */
1251 			decoder->pge = false;
1252 			decoder->continuous_period = false;
1253 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1254 			decoder->ip = to_ip;
1255 			decoder->state.from_ip = decoder->ip;
1256 			decoder->state.to_ip = to_ip;
1257 			decoder->state.type |= INTEL_PT_TRACE_END;
1258 			return 0;
1259 		}
1260 		intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1261 				decoder->ip);
1262 		decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1263 		return -ENOENT;
1264 	}
1265 
1266 	return intel_pt_bug(decoder);
1267 }
1268 
1269 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1270 {
1271 	struct intel_pt_insn intel_pt_insn;
1272 	int err;
1273 
1274 	while (1) {
1275 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1276 		if (err == INTEL_PT_RETURN)
1277 			return 0;
1278 		if (err)
1279 			return err;
1280 
1281 		if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1282 			if (!decoder->return_compression) {
1283 				intel_pt_log_at("ERROR: RET when expecting conditional branch",
1284 						decoder->ip);
1285 				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1286 				return -ENOENT;
1287 			}
1288 			if (!decoder->ret_addr) {
1289 				intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1290 						decoder->ip);
1291 				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1292 				return -ENOENT;
1293 			}
1294 			if (!(decoder->tnt.payload & BIT63)) {
1295 				intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1296 						decoder->ip);
1297 				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1298 				return -ENOENT;
1299 			}
1300 			decoder->tnt.count -= 1;
1301 			if (decoder->tnt.count)
1302 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1303 			else
1304 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1305 			decoder->tnt.payload <<= 1;
1306 			decoder->state.from_ip = decoder->ip;
1307 			decoder->ip = decoder->ret_addr;
1308 			decoder->state.to_ip = decoder->ip;
1309 			return 0;
1310 		}
1311 
1312 		if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1313 			/* Handle deferred TIPs */
1314 			err = intel_pt_get_next_packet(decoder);
1315 			if (err)
1316 				return err;
1317 			if (decoder->packet.type != INTEL_PT_TIP ||
1318 			    decoder->packet.count == 0) {
1319 				intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1320 						decoder->ip);
1321 				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1322 				decoder->pkt_step = 0;
1323 				return -ENOENT;
1324 			}
1325 			intel_pt_set_last_ip(decoder);
1326 			decoder->state.from_ip = decoder->ip;
1327 			decoder->state.to_ip = decoder->last_ip;
1328 			decoder->ip = decoder->last_ip;
1329 			return 0;
1330 		}
1331 
1332 		if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1333 			decoder->tnt.count -= 1;
1334 			if (decoder->tnt.count)
1335 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1336 			else
1337 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1338 			if (decoder->tnt.payload & BIT63) {
1339 				decoder->tnt.payload <<= 1;
1340 				decoder->state.from_ip = decoder->ip;
1341 				decoder->ip += intel_pt_insn.length +
1342 					       intel_pt_insn.rel;
1343 				decoder->state.to_ip = decoder->ip;
1344 				return 0;
1345 			}
1346 			/* Instruction sample for a non-taken branch */
1347 			if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1348 				decoder->tnt.payload <<= 1;
1349 				decoder->state.type = INTEL_PT_INSTRUCTION;
1350 				decoder->state.from_ip = decoder->ip;
1351 				decoder->state.to_ip = 0;
1352 				decoder->ip += intel_pt_insn.length;
1353 				return 0;
1354 			}
1355 			decoder->sample_cyc = false;
1356 			decoder->ip += intel_pt_insn.length;
1357 			if (!decoder->tnt.count) {
1358 				intel_pt_update_sample_time(decoder);
1359 				return -EAGAIN;
1360 			}
1361 			decoder->tnt.payload <<= 1;
1362 			continue;
1363 		}
1364 
1365 		return intel_pt_bug(decoder);
1366 	}
1367 }
1368 
1369 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1370 {
1371 	unsigned int fup_tx_flags;
1372 	int err;
1373 
1374 	fup_tx_flags = decoder->packet.payload &
1375 		       (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1376 	err = intel_pt_get_next_packet(decoder);
1377 	if (err)
1378 		return err;
1379 	if (decoder->packet.type == INTEL_PT_FUP) {
1380 		decoder->fup_tx_flags = fup_tx_flags;
1381 		decoder->set_fup_tx_flags = true;
1382 		if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1383 			*no_tip = true;
1384 	} else {
1385 		intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1386 				decoder->pos);
1387 		intel_pt_update_in_tx(decoder);
1388 	}
1389 	return 0;
1390 }
1391 
1392 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1393 {
1394 	timestamp |= (ref_timestamp & (0xffULL << 56));
1395 
1396 	if (timestamp < ref_timestamp) {
1397 		if (ref_timestamp - timestamp > (1ULL << 55))
1398 			timestamp += (1ULL << 56);
1399 	} else {
1400 		if (timestamp - ref_timestamp > (1ULL << 55))
1401 			timestamp -= (1ULL << 56);
1402 	}
1403 
1404 	return timestamp;
1405 }
1406 
1407 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1408 {
1409 	uint64_t timestamp;
1410 
1411 	decoder->have_tma = false;
1412 
1413 	if (decoder->ref_timestamp) {
1414 		timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1415 					    decoder->ref_timestamp);
1416 		decoder->tsc_timestamp = timestamp;
1417 		decoder->timestamp = timestamp;
1418 		decoder->ref_timestamp = 0;
1419 		decoder->timestamp_insn_cnt = 0;
1420 	} else if (decoder->timestamp) {
1421 		timestamp = decoder->packet.payload |
1422 			    (decoder->timestamp & (0xffULL << 56));
1423 		decoder->tsc_timestamp = timestamp;
1424 		if (timestamp < decoder->timestamp &&
1425 		    decoder->timestamp - timestamp < decoder->tsc_slip) {
1426 			intel_pt_log_to("Suppressing backwards timestamp",
1427 					timestamp);
1428 			timestamp = decoder->timestamp;
1429 		}
1430 		if (timestamp < decoder->timestamp) {
1431 			intel_pt_log_to("Wraparound timestamp", timestamp);
1432 			timestamp += (1ULL << 56);
1433 			decoder->tsc_timestamp = timestamp;
1434 		}
1435 		decoder->timestamp = timestamp;
1436 		decoder->timestamp_insn_cnt = 0;
1437 	}
1438 
1439 	if (decoder->last_packet_type == INTEL_PT_CYC) {
1440 		decoder->cyc_ref_timestamp = decoder->timestamp;
1441 		decoder->cycle_cnt = 0;
1442 		decoder->have_calc_cyc_to_tsc = false;
1443 		intel_pt_calc_cyc_to_tsc(decoder, false);
1444 	}
1445 
1446 	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1447 }
1448 
1449 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1450 {
1451 	intel_pt_log("ERROR: Buffer overflow\n");
1452 	intel_pt_clear_tx_flags(decoder);
1453 	decoder->timestamp_insn_cnt = 0;
1454 	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1455 	decoder->overflow = true;
1456 	return -EOVERFLOW;
1457 }
1458 
1459 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1460 {
1461 	if (decoder->have_cyc)
1462 		return;
1463 
1464 	decoder->cyc_cnt_timestamp = decoder->timestamp;
1465 	decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1466 }
1467 
1468 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1469 {
1470 	decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1471 
1472 	if (decoder->pge)
1473 		intel_pt_mtc_cyc_cnt_pge(decoder);
1474 }
1475 
1476 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1477 {
1478 	uint64_t tot_cyc_cnt, tsc_delta;
1479 
1480 	if (decoder->have_cyc)
1481 		return;
1482 
1483 	decoder->sample_cyc = true;
1484 
1485 	if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1486 		return;
1487 
1488 	tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1489 	tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1490 
1491 	if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1492 		decoder->tot_cyc_cnt = tot_cyc_cnt;
1493 }
1494 
1495 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1496 {
1497 	uint32_t ctc = decoder->packet.payload;
1498 	uint32_t fc = decoder->packet.count;
1499 	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1500 
1501 	if (!decoder->tsc_ctc_ratio_d)
1502 		return;
1503 
1504 	if (decoder->pge && !decoder->in_psb)
1505 		intel_pt_mtc_cyc_cnt_pge(decoder);
1506 	else
1507 		intel_pt_mtc_cyc_cnt_upd(decoder);
1508 
1509 	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1510 	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1511 	if (decoder->tsc_ctc_mult) {
1512 		decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1513 	} else {
1514 		decoder->ctc_timestamp -= multdiv(ctc_rem,
1515 						  decoder->tsc_ctc_ratio_n,
1516 						  decoder->tsc_ctc_ratio_d);
1517 	}
1518 	decoder->ctc_delta = 0;
1519 	decoder->have_tma = true;
1520 	decoder->fixup_last_mtc = true;
1521 	intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1522 		     decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1523 }
1524 
1525 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1526 {
1527 	uint64_t timestamp;
1528 	uint32_t mtc, mtc_delta;
1529 
1530 	if (!decoder->have_tma)
1531 		return;
1532 
1533 	mtc = decoder->packet.payload;
1534 
1535 	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1536 		decoder->fixup_last_mtc = false;
1537 		intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1538 					&decoder->last_mtc);
1539 	}
1540 
1541 	if (mtc > decoder->last_mtc)
1542 		mtc_delta = mtc - decoder->last_mtc;
1543 	else
1544 		mtc_delta = mtc + 256 - decoder->last_mtc;
1545 
1546 	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1547 
1548 	if (decoder->tsc_ctc_mult) {
1549 		timestamp = decoder->ctc_timestamp +
1550 			    decoder->ctc_delta * decoder->tsc_ctc_mult;
1551 	} else {
1552 		timestamp = decoder->ctc_timestamp +
1553 			    multdiv(decoder->ctc_delta,
1554 				    decoder->tsc_ctc_ratio_n,
1555 				    decoder->tsc_ctc_ratio_d);
1556 	}
1557 
1558 	if (timestamp < decoder->timestamp)
1559 		intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1560 			     timestamp, decoder->timestamp);
1561 	else
1562 		decoder->timestamp = timestamp;
1563 
1564 	intel_pt_mtc_cyc_cnt_upd(decoder);
1565 
1566 	decoder->timestamp_insn_cnt = 0;
1567 	decoder->last_mtc = mtc;
1568 
1569 	if (decoder->last_packet_type == INTEL_PT_CYC) {
1570 		decoder->cyc_ref_timestamp = decoder->timestamp;
1571 		decoder->cycle_cnt = 0;
1572 		decoder->have_calc_cyc_to_tsc = false;
1573 		intel_pt_calc_cyc_to_tsc(decoder, true);
1574 	}
1575 
1576 	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1577 }
1578 
1579 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1580 {
1581 	unsigned int cbr = decoder->packet.payload & 0xff;
1582 
1583 	decoder->cbr_payload = decoder->packet.payload;
1584 
1585 	if (decoder->cbr == cbr)
1586 		return;
1587 
1588 	decoder->cbr = cbr;
1589 	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1590 
1591 	intel_pt_mtc_cyc_cnt_cbr(decoder);
1592 }
1593 
1594 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1595 {
1596 	uint64_t timestamp = decoder->cyc_ref_timestamp;
1597 
1598 	decoder->have_cyc = true;
1599 
1600 	decoder->cycle_cnt += decoder->packet.payload;
1601 	if (decoder->pge)
1602 		decoder->tot_cyc_cnt += decoder->packet.payload;
1603 	decoder->sample_cyc = true;
1604 
1605 	if (!decoder->cyc_ref_timestamp)
1606 		return;
1607 
1608 	if (decoder->have_calc_cyc_to_tsc)
1609 		timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1610 	else if (decoder->cbr)
1611 		timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1612 	else
1613 		return;
1614 
1615 	if (timestamp < decoder->timestamp)
1616 		intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1617 			     timestamp, decoder->timestamp);
1618 	else
1619 		decoder->timestamp = timestamp;
1620 
1621 	decoder->timestamp_insn_cnt = 0;
1622 
1623 	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1624 }
1625 
1626 static void intel_pt_bbp(struct intel_pt_decoder *decoder)
1627 {
1628 	if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
1629 		memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
1630 		decoder->state.items.is_32_bit = false;
1631 	}
1632 	decoder->blk_type = decoder->packet.payload;
1633 	decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
1634 	if (decoder->blk_type == INTEL_PT_GP_REGS)
1635 		decoder->state.items.is_32_bit = decoder->packet.count;
1636 	if (decoder->blk_type_pos < 0) {
1637 		intel_pt_log("WARNING: Unknown block type %u\n",
1638 			     decoder->blk_type);
1639 	} else if (decoder->state.items.mask[decoder->blk_type_pos]) {
1640 		intel_pt_log("WARNING: Duplicate block type %u\n",
1641 			     decoder->blk_type);
1642 	}
1643 }
1644 
1645 static void intel_pt_bip(struct intel_pt_decoder *decoder)
1646 {
1647 	uint32_t id = decoder->packet.count;
1648 	uint32_t bit = 1 << id;
1649 	int pos = decoder->blk_type_pos;
1650 
1651 	if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
1652 		intel_pt_log("WARNING: Unknown block item %u type %d\n",
1653 			     id, decoder->blk_type);
1654 		return;
1655 	}
1656 
1657 	if (decoder->state.items.mask[pos] & bit) {
1658 		intel_pt_log("WARNING: Duplicate block item %u type %d\n",
1659 			     id, decoder->blk_type);
1660 	}
1661 
1662 	decoder->state.items.mask[pos] |= bit;
1663 	decoder->state.items.val[pos][id] = decoder->packet.payload;
1664 }
1665 
1666 /* Walk PSB+ packets when already in sync. */
1667 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1668 {
1669 	int err;
1670 
1671 	decoder->in_psb = true;
1672 
1673 	while (1) {
1674 		err = intel_pt_get_next_packet(decoder);
1675 		if (err)
1676 			goto out;
1677 
1678 		switch (decoder->packet.type) {
1679 		case INTEL_PT_PSBEND:
1680 			err = 0;
1681 			goto out;
1682 
1683 		case INTEL_PT_TIP_PGD:
1684 		case INTEL_PT_TIP_PGE:
1685 		case INTEL_PT_TIP:
1686 		case INTEL_PT_TNT:
1687 		case INTEL_PT_TRACESTOP:
1688 		case INTEL_PT_BAD:
1689 		case INTEL_PT_PSB:
1690 		case INTEL_PT_PTWRITE:
1691 		case INTEL_PT_PTWRITE_IP:
1692 		case INTEL_PT_EXSTOP:
1693 		case INTEL_PT_EXSTOP_IP:
1694 		case INTEL_PT_MWAIT:
1695 		case INTEL_PT_PWRE:
1696 		case INTEL_PT_PWRX:
1697 		case INTEL_PT_BBP:
1698 		case INTEL_PT_BIP:
1699 		case INTEL_PT_BEP:
1700 		case INTEL_PT_BEP_IP:
1701 			decoder->have_tma = false;
1702 			intel_pt_log("ERROR: Unexpected packet\n");
1703 			err = -EAGAIN;
1704 			goto out;
1705 
1706 		case INTEL_PT_OVF:
1707 			err = intel_pt_overflow(decoder);
1708 			goto out;
1709 
1710 		case INTEL_PT_TSC:
1711 			intel_pt_calc_tsc_timestamp(decoder);
1712 			break;
1713 
1714 		case INTEL_PT_TMA:
1715 			intel_pt_calc_tma(decoder);
1716 			break;
1717 
1718 		case INTEL_PT_CBR:
1719 			intel_pt_calc_cbr(decoder);
1720 			break;
1721 
1722 		case INTEL_PT_MODE_EXEC:
1723 			decoder->exec_mode = decoder->packet.payload;
1724 			break;
1725 
1726 		case INTEL_PT_PIP:
1727 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1728 			break;
1729 
1730 		case INTEL_PT_FUP:
1731 			decoder->pge = true;
1732 			if (decoder->packet.count)
1733 				intel_pt_set_last_ip(decoder);
1734 			break;
1735 
1736 		case INTEL_PT_MODE_TSX:
1737 			intel_pt_update_in_tx(decoder);
1738 			break;
1739 
1740 		case INTEL_PT_MTC:
1741 			intel_pt_calc_mtc_timestamp(decoder);
1742 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1743 				decoder->state.type |= INTEL_PT_INSTRUCTION;
1744 			break;
1745 
1746 		case INTEL_PT_CYC:
1747 		case INTEL_PT_VMCS:
1748 		case INTEL_PT_MNT:
1749 		case INTEL_PT_PAD:
1750 		default:
1751 			break;
1752 		}
1753 	}
1754 out:
1755 	decoder->in_psb = false;
1756 
1757 	return err;
1758 }
1759 
1760 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1761 {
1762 	int err;
1763 
1764 	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1765 		decoder->tx_flags = 0;
1766 		decoder->state.flags &= ~INTEL_PT_IN_TX;
1767 		decoder->state.flags |= INTEL_PT_ABORT_TX;
1768 	} else {
1769 		decoder->state.flags |= INTEL_PT_ASYNC;
1770 	}
1771 
1772 	while (1) {
1773 		err = intel_pt_get_next_packet(decoder);
1774 		if (err)
1775 			return err;
1776 
1777 		switch (decoder->packet.type) {
1778 		case INTEL_PT_TNT:
1779 		case INTEL_PT_FUP:
1780 		case INTEL_PT_TRACESTOP:
1781 		case INTEL_PT_PSB:
1782 		case INTEL_PT_TSC:
1783 		case INTEL_PT_TMA:
1784 		case INTEL_PT_MODE_TSX:
1785 		case INTEL_PT_BAD:
1786 		case INTEL_PT_PSBEND:
1787 		case INTEL_PT_PTWRITE:
1788 		case INTEL_PT_PTWRITE_IP:
1789 		case INTEL_PT_EXSTOP:
1790 		case INTEL_PT_EXSTOP_IP:
1791 		case INTEL_PT_MWAIT:
1792 		case INTEL_PT_PWRE:
1793 		case INTEL_PT_PWRX:
1794 		case INTEL_PT_BBP:
1795 		case INTEL_PT_BIP:
1796 		case INTEL_PT_BEP:
1797 		case INTEL_PT_BEP_IP:
1798 			intel_pt_log("ERROR: Missing TIP after FUP\n");
1799 			decoder->pkt_state = INTEL_PT_STATE_ERR3;
1800 			decoder->pkt_step = 0;
1801 			return -ENOENT;
1802 
1803 		case INTEL_PT_CBR:
1804 			intel_pt_calc_cbr(decoder);
1805 			break;
1806 
1807 		case INTEL_PT_OVF:
1808 			return intel_pt_overflow(decoder);
1809 
1810 		case INTEL_PT_TIP_PGD:
1811 			decoder->state.from_ip = decoder->ip;
1812 			if (decoder->packet.count == 0) {
1813 				decoder->state.to_ip = 0;
1814 			} else {
1815 				intel_pt_set_ip(decoder);
1816 				decoder->state.to_ip = decoder->ip;
1817 			}
1818 			decoder->pge = false;
1819 			decoder->continuous_period = false;
1820 			decoder->state.type |= INTEL_PT_TRACE_END;
1821 			return 0;
1822 
1823 		case INTEL_PT_TIP_PGE:
1824 			decoder->pge = true;
1825 			intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1826 				     decoder->ip);
1827 			decoder->state.from_ip = 0;
1828 			if (decoder->packet.count == 0) {
1829 				decoder->state.to_ip = 0;
1830 			} else {
1831 				intel_pt_set_ip(decoder);
1832 				decoder->state.to_ip = decoder->ip;
1833 			}
1834 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1835 			intel_pt_mtc_cyc_cnt_pge(decoder);
1836 			return 0;
1837 
1838 		case INTEL_PT_TIP:
1839 			decoder->state.from_ip = decoder->ip;
1840 			if (decoder->packet.count == 0) {
1841 				decoder->state.to_ip = 0;
1842 			} else {
1843 				intel_pt_set_ip(decoder);
1844 				decoder->state.to_ip = decoder->ip;
1845 			}
1846 			return 0;
1847 
1848 		case INTEL_PT_PIP:
1849 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1850 			break;
1851 
1852 		case INTEL_PT_MTC:
1853 			intel_pt_calc_mtc_timestamp(decoder);
1854 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1855 				decoder->state.type |= INTEL_PT_INSTRUCTION;
1856 			break;
1857 
1858 		case INTEL_PT_CYC:
1859 			intel_pt_calc_cyc_timestamp(decoder);
1860 			break;
1861 
1862 		case INTEL_PT_MODE_EXEC:
1863 			decoder->exec_mode = decoder->packet.payload;
1864 			break;
1865 
1866 		case INTEL_PT_VMCS:
1867 		case INTEL_PT_MNT:
1868 		case INTEL_PT_PAD:
1869 			break;
1870 
1871 		default:
1872 			return intel_pt_bug(decoder);
1873 		}
1874 	}
1875 }
1876 
1877 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1878 {
1879 	bool no_tip = false;
1880 	int err;
1881 
1882 	while (1) {
1883 		err = intel_pt_get_next_packet(decoder);
1884 		if (err)
1885 			return err;
1886 next:
1887 		switch (decoder->packet.type) {
1888 		case INTEL_PT_TNT:
1889 			if (!decoder->packet.count)
1890 				break;
1891 			decoder->tnt = decoder->packet;
1892 			decoder->pkt_state = INTEL_PT_STATE_TNT;
1893 			err = intel_pt_walk_tnt(decoder);
1894 			if (err == -EAGAIN)
1895 				break;
1896 			return err;
1897 
1898 		case INTEL_PT_TIP_PGD:
1899 			if (decoder->packet.count != 0)
1900 				intel_pt_set_last_ip(decoder);
1901 			decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1902 			return intel_pt_walk_tip(decoder);
1903 
1904 		case INTEL_PT_TIP_PGE: {
1905 			decoder->pge = true;
1906 			intel_pt_mtc_cyc_cnt_pge(decoder);
1907 			if (decoder->packet.count == 0) {
1908 				intel_pt_log_at("Skipping zero TIP.PGE",
1909 						decoder->pos);
1910 				break;
1911 			}
1912 			intel_pt_set_ip(decoder);
1913 			decoder->state.from_ip = 0;
1914 			decoder->state.to_ip = decoder->ip;
1915 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1916 			return 0;
1917 		}
1918 
1919 		case INTEL_PT_OVF:
1920 			return intel_pt_overflow(decoder);
1921 
1922 		case INTEL_PT_TIP:
1923 			if (decoder->packet.count != 0)
1924 				intel_pt_set_last_ip(decoder);
1925 			decoder->pkt_state = INTEL_PT_STATE_TIP;
1926 			return intel_pt_walk_tip(decoder);
1927 
1928 		case INTEL_PT_FUP:
1929 			if (decoder->packet.count == 0) {
1930 				intel_pt_log_at("Skipping zero FUP",
1931 						decoder->pos);
1932 				no_tip = false;
1933 				break;
1934 			}
1935 			intel_pt_set_last_ip(decoder);
1936 			if (!decoder->branch_enable) {
1937 				decoder->ip = decoder->last_ip;
1938 				if (intel_pt_fup_event(decoder))
1939 					return 0;
1940 				no_tip = false;
1941 				break;
1942 			}
1943 			if (decoder->set_fup_mwait)
1944 				no_tip = true;
1945 			err = intel_pt_walk_fup(decoder);
1946 			if (err != -EAGAIN) {
1947 				if (err)
1948 					return err;
1949 				if (no_tip)
1950 					decoder->pkt_state =
1951 						INTEL_PT_STATE_FUP_NO_TIP;
1952 				else
1953 					decoder->pkt_state = INTEL_PT_STATE_FUP;
1954 				return 0;
1955 			}
1956 			if (no_tip) {
1957 				no_tip = false;
1958 				break;
1959 			}
1960 			return intel_pt_walk_fup_tip(decoder);
1961 
1962 		case INTEL_PT_TRACESTOP:
1963 			decoder->pge = false;
1964 			decoder->continuous_period = false;
1965 			intel_pt_clear_tx_flags(decoder);
1966 			decoder->have_tma = false;
1967 			break;
1968 
1969 		case INTEL_PT_PSB:
1970 			decoder->last_ip = 0;
1971 			decoder->have_last_ip = true;
1972 			intel_pt_clear_stack(&decoder->stack);
1973 			err = intel_pt_walk_psbend(decoder);
1974 			if (err == -EAGAIN)
1975 				goto next;
1976 			if (err)
1977 				return err;
1978 			/*
1979 			 * PSB+ CBR will not have changed but cater for the
1980 			 * possibility of another CBR change that gets caught up
1981 			 * in the PSB+.
1982 			 */
1983 			if (decoder->cbr != decoder->cbr_seen)
1984 				return 0;
1985 			break;
1986 
1987 		case INTEL_PT_PIP:
1988 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1989 			break;
1990 
1991 		case INTEL_PT_MTC:
1992 			intel_pt_calc_mtc_timestamp(decoder);
1993 			if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1994 				break;
1995 			/*
1996 			 * Ensure that there has been an instruction since the
1997 			 * last MTC.
1998 			 */
1999 			if (!decoder->mtc_insn)
2000 				break;
2001 			decoder->mtc_insn = false;
2002 			/* Ensure that there is a timestamp */
2003 			if (!decoder->timestamp)
2004 				break;
2005 			decoder->state.type = INTEL_PT_INSTRUCTION;
2006 			decoder->state.from_ip = decoder->ip;
2007 			decoder->state.to_ip = 0;
2008 			decoder->mtc_insn = false;
2009 			return 0;
2010 
2011 		case INTEL_PT_TSC:
2012 			intel_pt_calc_tsc_timestamp(decoder);
2013 			break;
2014 
2015 		case INTEL_PT_TMA:
2016 			intel_pt_calc_tma(decoder);
2017 			break;
2018 
2019 		case INTEL_PT_CYC:
2020 			intel_pt_calc_cyc_timestamp(decoder);
2021 			break;
2022 
2023 		case INTEL_PT_CBR:
2024 			intel_pt_calc_cbr(decoder);
2025 			if (decoder->cbr != decoder->cbr_seen)
2026 				return 0;
2027 			break;
2028 
2029 		case INTEL_PT_MODE_EXEC:
2030 			decoder->exec_mode = decoder->packet.payload;
2031 			break;
2032 
2033 		case INTEL_PT_MODE_TSX:
2034 			/* MODE_TSX need not be followed by FUP */
2035 			if (!decoder->pge) {
2036 				intel_pt_update_in_tx(decoder);
2037 				break;
2038 			}
2039 			err = intel_pt_mode_tsx(decoder, &no_tip);
2040 			if (err)
2041 				return err;
2042 			goto next;
2043 
2044 		case INTEL_PT_BAD: /* Does not happen */
2045 			return intel_pt_bug(decoder);
2046 
2047 		case INTEL_PT_PSBEND:
2048 		case INTEL_PT_VMCS:
2049 		case INTEL_PT_MNT:
2050 		case INTEL_PT_PAD:
2051 			break;
2052 
2053 		case INTEL_PT_PTWRITE_IP:
2054 			decoder->fup_ptw_payload = decoder->packet.payload;
2055 			err = intel_pt_get_next_packet(decoder);
2056 			if (err)
2057 				return err;
2058 			if (decoder->packet.type == INTEL_PT_FUP) {
2059 				decoder->set_fup_ptw = true;
2060 				no_tip = true;
2061 			} else {
2062 				intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
2063 						decoder->pos);
2064 			}
2065 			goto next;
2066 
2067 		case INTEL_PT_PTWRITE:
2068 			decoder->state.type = INTEL_PT_PTW;
2069 			decoder->state.from_ip = decoder->ip;
2070 			decoder->state.to_ip = 0;
2071 			decoder->state.ptw_payload = decoder->packet.payload;
2072 			return 0;
2073 
2074 		case INTEL_PT_MWAIT:
2075 			decoder->fup_mwait_payload = decoder->packet.payload;
2076 			decoder->set_fup_mwait = true;
2077 			break;
2078 
2079 		case INTEL_PT_PWRE:
2080 			if (decoder->set_fup_mwait) {
2081 				decoder->fup_pwre_payload =
2082 							decoder->packet.payload;
2083 				decoder->set_fup_pwre = true;
2084 				break;
2085 			}
2086 			decoder->state.type = INTEL_PT_PWR_ENTRY;
2087 			decoder->state.from_ip = decoder->ip;
2088 			decoder->state.to_ip = 0;
2089 			decoder->state.pwrx_payload = decoder->packet.payload;
2090 			return 0;
2091 
2092 		case INTEL_PT_EXSTOP_IP:
2093 			err = intel_pt_get_next_packet(decoder);
2094 			if (err)
2095 				return err;
2096 			if (decoder->packet.type == INTEL_PT_FUP) {
2097 				decoder->set_fup_exstop = true;
2098 				no_tip = true;
2099 			} else {
2100 				intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
2101 						decoder->pos);
2102 			}
2103 			goto next;
2104 
2105 		case INTEL_PT_EXSTOP:
2106 			decoder->state.type = INTEL_PT_EX_STOP;
2107 			decoder->state.from_ip = decoder->ip;
2108 			decoder->state.to_ip = 0;
2109 			return 0;
2110 
2111 		case INTEL_PT_PWRX:
2112 			decoder->state.type = INTEL_PT_PWR_EXIT;
2113 			decoder->state.from_ip = decoder->ip;
2114 			decoder->state.to_ip = 0;
2115 			decoder->state.pwrx_payload = decoder->packet.payload;
2116 			return 0;
2117 
2118 		case INTEL_PT_BBP:
2119 			intel_pt_bbp(decoder);
2120 			break;
2121 
2122 		case INTEL_PT_BIP:
2123 			intel_pt_bip(decoder);
2124 			break;
2125 
2126 		case INTEL_PT_BEP:
2127 			decoder->state.type = INTEL_PT_BLK_ITEMS;
2128 			decoder->state.from_ip = decoder->ip;
2129 			decoder->state.to_ip = 0;
2130 			return 0;
2131 
2132 		case INTEL_PT_BEP_IP:
2133 			err = intel_pt_get_next_packet(decoder);
2134 			if (err)
2135 				return err;
2136 			if (decoder->packet.type == INTEL_PT_FUP) {
2137 				decoder->set_fup_bep = true;
2138 				no_tip = true;
2139 			} else {
2140 				intel_pt_log_at("ERROR: Missing FUP after BEP",
2141 						decoder->pos);
2142 			}
2143 			goto next;
2144 
2145 		default:
2146 			return intel_pt_bug(decoder);
2147 		}
2148 	}
2149 }
2150 
2151 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
2152 {
2153 	return decoder->packet.count &&
2154 	       (decoder->have_last_ip || decoder->packet.count == 3 ||
2155 		decoder->packet.count == 6);
2156 }
2157 
2158 /* Walk PSB+ packets to get in sync. */
2159 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
2160 {
2161 	int err;
2162 
2163 	decoder->in_psb = true;
2164 
2165 	while (1) {
2166 		err = intel_pt_get_next_packet(decoder);
2167 		if (err)
2168 			goto out;
2169 
2170 		switch (decoder->packet.type) {
2171 		case INTEL_PT_TIP_PGD:
2172 			decoder->continuous_period = false;
2173 			__fallthrough;
2174 		case INTEL_PT_TIP_PGE:
2175 		case INTEL_PT_TIP:
2176 		case INTEL_PT_PTWRITE:
2177 		case INTEL_PT_PTWRITE_IP:
2178 		case INTEL_PT_EXSTOP:
2179 		case INTEL_PT_EXSTOP_IP:
2180 		case INTEL_PT_MWAIT:
2181 		case INTEL_PT_PWRE:
2182 		case INTEL_PT_PWRX:
2183 		case INTEL_PT_BBP:
2184 		case INTEL_PT_BIP:
2185 		case INTEL_PT_BEP:
2186 		case INTEL_PT_BEP_IP:
2187 			intel_pt_log("ERROR: Unexpected packet\n");
2188 			err = -ENOENT;
2189 			goto out;
2190 
2191 		case INTEL_PT_FUP:
2192 			decoder->pge = true;
2193 			if (intel_pt_have_ip(decoder)) {
2194 				uint64_t current_ip = decoder->ip;
2195 
2196 				intel_pt_set_ip(decoder);
2197 				if (current_ip)
2198 					intel_pt_log_to("Setting IP",
2199 							decoder->ip);
2200 			}
2201 			break;
2202 
2203 		case INTEL_PT_MTC:
2204 			intel_pt_calc_mtc_timestamp(decoder);
2205 			break;
2206 
2207 		case INTEL_PT_TSC:
2208 			intel_pt_calc_tsc_timestamp(decoder);
2209 			break;
2210 
2211 		case INTEL_PT_TMA:
2212 			intel_pt_calc_tma(decoder);
2213 			break;
2214 
2215 		case INTEL_PT_CYC:
2216 			intel_pt_calc_cyc_timestamp(decoder);
2217 			break;
2218 
2219 		case INTEL_PT_CBR:
2220 			intel_pt_calc_cbr(decoder);
2221 			break;
2222 
2223 		case INTEL_PT_PIP:
2224 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2225 			break;
2226 
2227 		case INTEL_PT_MODE_EXEC:
2228 			decoder->exec_mode = decoder->packet.payload;
2229 			break;
2230 
2231 		case INTEL_PT_MODE_TSX:
2232 			intel_pt_update_in_tx(decoder);
2233 			break;
2234 
2235 		case INTEL_PT_TRACESTOP:
2236 			decoder->pge = false;
2237 			decoder->continuous_period = false;
2238 			intel_pt_clear_tx_flags(decoder);
2239 			__fallthrough;
2240 
2241 		case INTEL_PT_TNT:
2242 			decoder->have_tma = false;
2243 			intel_pt_log("ERROR: Unexpected packet\n");
2244 			if (decoder->ip)
2245 				decoder->pkt_state = INTEL_PT_STATE_ERR4;
2246 			else
2247 				decoder->pkt_state = INTEL_PT_STATE_ERR3;
2248 			err = -ENOENT;
2249 			goto out;
2250 
2251 		case INTEL_PT_BAD: /* Does not happen */
2252 			err = intel_pt_bug(decoder);
2253 			goto out;
2254 
2255 		case INTEL_PT_OVF:
2256 			err = intel_pt_overflow(decoder);
2257 			goto out;
2258 
2259 		case INTEL_PT_PSBEND:
2260 			err = 0;
2261 			goto out;
2262 
2263 		case INTEL_PT_PSB:
2264 		case INTEL_PT_VMCS:
2265 		case INTEL_PT_MNT:
2266 		case INTEL_PT_PAD:
2267 		default:
2268 			break;
2269 		}
2270 	}
2271 out:
2272 	decoder->in_psb = false;
2273 
2274 	return err;
2275 }
2276 
2277 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2278 {
2279 	int err;
2280 
2281 	while (1) {
2282 		err = intel_pt_get_next_packet(decoder);
2283 		if (err)
2284 			return err;
2285 
2286 		switch (decoder->packet.type) {
2287 		case INTEL_PT_TIP_PGD:
2288 			decoder->continuous_period = false;
2289 			decoder->pge = false;
2290 			if (intel_pt_have_ip(decoder))
2291 				intel_pt_set_ip(decoder);
2292 			if (!decoder->ip)
2293 				break;
2294 			decoder->state.type |= INTEL_PT_TRACE_END;
2295 			return 0;
2296 
2297 		case INTEL_PT_TIP_PGE:
2298 			decoder->pge = true;
2299 			intel_pt_mtc_cyc_cnt_pge(decoder);
2300 			if (intel_pt_have_ip(decoder))
2301 				intel_pt_set_ip(decoder);
2302 			if (!decoder->ip)
2303 				break;
2304 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2305 			return 0;
2306 
2307 		case INTEL_PT_TIP:
2308 			decoder->pge = true;
2309 			if (intel_pt_have_ip(decoder))
2310 				intel_pt_set_ip(decoder);
2311 			if (!decoder->ip)
2312 				break;
2313 			return 0;
2314 
2315 		case INTEL_PT_FUP:
2316 			if (intel_pt_have_ip(decoder))
2317 				intel_pt_set_ip(decoder);
2318 			if (decoder->ip)
2319 				return 0;
2320 			break;
2321 
2322 		case INTEL_PT_MTC:
2323 			intel_pt_calc_mtc_timestamp(decoder);
2324 			break;
2325 
2326 		case INTEL_PT_TSC:
2327 			intel_pt_calc_tsc_timestamp(decoder);
2328 			break;
2329 
2330 		case INTEL_PT_TMA:
2331 			intel_pt_calc_tma(decoder);
2332 			break;
2333 
2334 		case INTEL_PT_CYC:
2335 			intel_pt_calc_cyc_timestamp(decoder);
2336 			break;
2337 
2338 		case INTEL_PT_CBR:
2339 			intel_pt_calc_cbr(decoder);
2340 			break;
2341 
2342 		case INTEL_PT_PIP:
2343 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2344 			break;
2345 
2346 		case INTEL_PT_MODE_EXEC:
2347 			decoder->exec_mode = decoder->packet.payload;
2348 			break;
2349 
2350 		case INTEL_PT_MODE_TSX:
2351 			intel_pt_update_in_tx(decoder);
2352 			break;
2353 
2354 		case INTEL_PT_OVF:
2355 			return intel_pt_overflow(decoder);
2356 
2357 		case INTEL_PT_BAD: /* Does not happen */
2358 			return intel_pt_bug(decoder);
2359 
2360 		case INTEL_PT_TRACESTOP:
2361 			decoder->pge = false;
2362 			decoder->continuous_period = false;
2363 			intel_pt_clear_tx_flags(decoder);
2364 			decoder->have_tma = false;
2365 			break;
2366 
2367 		case INTEL_PT_PSB:
2368 			decoder->last_ip = 0;
2369 			decoder->have_last_ip = true;
2370 			intel_pt_clear_stack(&decoder->stack);
2371 			err = intel_pt_walk_psb(decoder);
2372 			if (err)
2373 				return err;
2374 			if (decoder->ip) {
2375 				/* Do not have a sample */
2376 				decoder->state.type = 0;
2377 				return 0;
2378 			}
2379 			break;
2380 
2381 		case INTEL_PT_TNT:
2382 		case INTEL_PT_PSBEND:
2383 		case INTEL_PT_VMCS:
2384 		case INTEL_PT_MNT:
2385 		case INTEL_PT_PAD:
2386 		case INTEL_PT_PTWRITE:
2387 		case INTEL_PT_PTWRITE_IP:
2388 		case INTEL_PT_EXSTOP:
2389 		case INTEL_PT_EXSTOP_IP:
2390 		case INTEL_PT_MWAIT:
2391 		case INTEL_PT_PWRE:
2392 		case INTEL_PT_PWRX:
2393 		case INTEL_PT_BBP:
2394 		case INTEL_PT_BIP:
2395 		case INTEL_PT_BEP:
2396 		case INTEL_PT_BEP_IP:
2397 		default:
2398 			break;
2399 		}
2400 	}
2401 }
2402 
2403 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2404 {
2405 	int err;
2406 
2407 	decoder->set_fup_tx_flags = false;
2408 	decoder->set_fup_ptw = false;
2409 	decoder->set_fup_mwait = false;
2410 	decoder->set_fup_pwre = false;
2411 	decoder->set_fup_exstop = false;
2412 	decoder->set_fup_bep = false;
2413 
2414 	if (!decoder->branch_enable) {
2415 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2416 		decoder->overflow = false;
2417 		decoder->state.type = 0; /* Do not have a sample */
2418 		return 0;
2419 	}
2420 
2421 	intel_pt_log("Scanning for full IP\n");
2422 	err = intel_pt_walk_to_ip(decoder);
2423 	if (err)
2424 		return err;
2425 
2426 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2427 	decoder->overflow = false;
2428 
2429 	decoder->state.from_ip = 0;
2430 	decoder->state.to_ip = decoder->ip;
2431 	intel_pt_log_to("Setting IP", decoder->ip);
2432 
2433 	return 0;
2434 }
2435 
2436 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2437 {
2438 	const unsigned char *end = decoder->buf + decoder->len;
2439 	size_t i;
2440 
2441 	for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2442 		if (i > decoder->len)
2443 			continue;
2444 		if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2445 			return i;
2446 	}
2447 	return 0;
2448 }
2449 
2450 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2451 {
2452 	size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2453 	const char *psb = INTEL_PT_PSB_STR;
2454 
2455 	if (rest_psb > decoder->len ||
2456 	    memcmp(decoder->buf, psb + part_psb, rest_psb))
2457 		return 0;
2458 
2459 	return rest_psb;
2460 }
2461 
2462 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2463 				  int part_psb)
2464 {
2465 	int rest_psb, ret;
2466 
2467 	decoder->pos += decoder->len;
2468 	decoder->len = 0;
2469 
2470 	ret = intel_pt_get_next_data(decoder, false);
2471 	if (ret)
2472 		return ret;
2473 
2474 	rest_psb = intel_pt_rest_psb(decoder, part_psb);
2475 	if (!rest_psb)
2476 		return 0;
2477 
2478 	decoder->pos -= part_psb;
2479 	decoder->next_buf = decoder->buf + rest_psb;
2480 	decoder->next_len = decoder->len - rest_psb;
2481 	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2482 	decoder->buf = decoder->temp_buf;
2483 	decoder->len = INTEL_PT_PSB_LEN;
2484 
2485 	return 0;
2486 }
2487 
2488 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2489 {
2490 	unsigned char *next;
2491 	int ret;
2492 
2493 	intel_pt_log("Scanning for PSB\n");
2494 	while (1) {
2495 		if (!decoder->len) {
2496 			ret = intel_pt_get_next_data(decoder, false);
2497 			if (ret)
2498 				return ret;
2499 		}
2500 
2501 		next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2502 			      INTEL_PT_PSB_LEN);
2503 		if (!next) {
2504 			int part_psb;
2505 
2506 			part_psb = intel_pt_part_psb(decoder);
2507 			if (part_psb) {
2508 				ret = intel_pt_get_split_psb(decoder, part_psb);
2509 				if (ret)
2510 					return ret;
2511 			} else {
2512 				decoder->pos += decoder->len;
2513 				decoder->len = 0;
2514 			}
2515 			continue;
2516 		}
2517 
2518 		decoder->pkt_step = next - decoder->buf;
2519 		return intel_pt_get_next_packet(decoder);
2520 	}
2521 }
2522 
2523 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2524 {
2525 	int err;
2526 
2527 	decoder->pge = false;
2528 	decoder->continuous_period = false;
2529 	decoder->have_last_ip = false;
2530 	decoder->last_ip = 0;
2531 	decoder->ip = 0;
2532 	intel_pt_clear_stack(&decoder->stack);
2533 
2534 	err = intel_pt_scan_for_psb(decoder);
2535 	if (err)
2536 		return err;
2537 
2538 	decoder->have_last_ip = true;
2539 	decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2540 
2541 	err = intel_pt_walk_psb(decoder);
2542 	if (err)
2543 		return err;
2544 
2545 	if (decoder->ip) {
2546 		decoder->state.type = 0; /* Do not have a sample */
2547 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2548 	} else {
2549 		return intel_pt_sync_ip(decoder);
2550 	}
2551 
2552 	return 0;
2553 }
2554 
2555 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2556 {
2557 	uint64_t est = decoder->sample_insn_cnt << 1;
2558 
2559 	if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2560 		goto out;
2561 
2562 	est *= decoder->max_non_turbo_ratio;
2563 	est /= decoder->cbr;
2564 out:
2565 	return decoder->sample_timestamp + est;
2566 }
2567 
2568 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2569 {
2570 	int err;
2571 
2572 	do {
2573 		decoder->state.type = INTEL_PT_BRANCH;
2574 		decoder->state.flags = 0;
2575 
2576 		switch (decoder->pkt_state) {
2577 		case INTEL_PT_STATE_NO_PSB:
2578 			err = intel_pt_sync(decoder);
2579 			break;
2580 		case INTEL_PT_STATE_NO_IP:
2581 			decoder->have_last_ip = false;
2582 			decoder->last_ip = 0;
2583 			decoder->ip = 0;
2584 			__fallthrough;
2585 		case INTEL_PT_STATE_ERR_RESYNC:
2586 			err = intel_pt_sync_ip(decoder);
2587 			break;
2588 		case INTEL_PT_STATE_IN_SYNC:
2589 			err = intel_pt_walk_trace(decoder);
2590 			break;
2591 		case INTEL_PT_STATE_TNT:
2592 		case INTEL_PT_STATE_TNT_CONT:
2593 			err = intel_pt_walk_tnt(decoder);
2594 			if (err == -EAGAIN)
2595 				err = intel_pt_walk_trace(decoder);
2596 			break;
2597 		case INTEL_PT_STATE_TIP:
2598 		case INTEL_PT_STATE_TIP_PGD:
2599 			err = intel_pt_walk_tip(decoder);
2600 			break;
2601 		case INTEL_PT_STATE_FUP:
2602 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2603 			err = intel_pt_walk_fup(decoder);
2604 			if (err == -EAGAIN)
2605 				err = intel_pt_walk_fup_tip(decoder);
2606 			else if (!err)
2607 				decoder->pkt_state = INTEL_PT_STATE_FUP;
2608 			break;
2609 		case INTEL_PT_STATE_FUP_NO_TIP:
2610 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2611 			err = intel_pt_walk_fup(decoder);
2612 			if (err == -EAGAIN)
2613 				err = intel_pt_walk_trace(decoder);
2614 			break;
2615 		default:
2616 			err = intel_pt_bug(decoder);
2617 			break;
2618 		}
2619 	} while (err == -ENOLINK);
2620 
2621 	if (err) {
2622 		decoder->state.err = intel_pt_ext_err(err);
2623 		decoder->state.from_ip = decoder->ip;
2624 		intel_pt_update_sample_time(decoder);
2625 		decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2626 	} else {
2627 		decoder->state.err = 0;
2628 		if (decoder->cbr != decoder->cbr_seen) {
2629 			decoder->cbr_seen = decoder->cbr;
2630 			if (!decoder->state.type) {
2631 				decoder->state.from_ip = decoder->ip;
2632 				decoder->state.to_ip = 0;
2633 			}
2634 			decoder->state.type |= INTEL_PT_CBR_CHG;
2635 			decoder->state.cbr_payload = decoder->cbr_payload;
2636 			decoder->state.cbr = decoder->cbr;
2637 		}
2638 		if (intel_pt_sample_time(decoder->pkt_state)) {
2639 			intel_pt_update_sample_time(decoder);
2640 			if (decoder->sample_cyc)
2641 				decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2642 		}
2643 	}
2644 
2645 	decoder->state.timestamp = decoder->sample_timestamp;
2646 	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2647 	decoder->state.cr3 = decoder->cr3;
2648 	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2649 	decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
2650 
2651 	return &decoder->state;
2652 }
2653 
2654 /**
2655  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2656  * @buf: pointer to buffer pointer
2657  * @len: size of buffer
2658  *
2659  * Updates the buffer pointer to point to the start of the next PSB packet if
2660  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2661  * @len is adjusted accordingly.
2662  *
2663  * Return: %true if a PSB packet is found, %false otherwise.
2664  */
2665 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2666 {
2667 	unsigned char *next;
2668 
2669 	next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2670 	if (next) {
2671 		*len -= next - *buf;
2672 		*buf = next;
2673 		return true;
2674 	}
2675 	return false;
2676 }
2677 
2678 /**
2679  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2680  *                     packet.
2681  * @buf: pointer to buffer pointer
2682  * @len: size of buffer
2683  *
2684  * Updates the buffer pointer to point to the start of the following PSB packet
2685  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2686  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2687  *
2688  * Return: %true if a PSB packet is found, %false otherwise.
2689  */
2690 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2691 {
2692 	unsigned char *next;
2693 
2694 	if (!*len)
2695 		return false;
2696 
2697 	next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2698 	if (next) {
2699 		*len -= next - *buf;
2700 		*buf = next;
2701 		return true;
2702 	}
2703 	return false;
2704 }
2705 
2706 /**
2707  * intel_pt_last_psb - find the last PSB packet in a buffer.
2708  * @buf: buffer
2709  * @len: size of buffer
2710  *
2711  * This function finds the last PSB in a buffer.
2712  *
2713  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2714  */
2715 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2716 {
2717 	const char *n = INTEL_PT_PSB_STR;
2718 	unsigned char *p;
2719 	size_t k;
2720 
2721 	if (len < INTEL_PT_PSB_LEN)
2722 		return NULL;
2723 
2724 	k = len - INTEL_PT_PSB_LEN + 1;
2725 	while (1) {
2726 		p = memrchr(buf, n[0], k);
2727 		if (!p)
2728 			return NULL;
2729 		if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2730 			return p;
2731 		k = p - buf;
2732 		if (!k)
2733 			return NULL;
2734 	}
2735 }
2736 
2737 /**
2738  * intel_pt_next_tsc - find and return next TSC.
2739  * @buf: buffer
2740  * @len: size of buffer
2741  * @tsc: TSC value returned
2742  * @rem: returns remaining size when TSC is found
2743  *
2744  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2745  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2746  * PSBEND packet is found.
2747  *
2748  * Return: %true if TSC is found, false otherwise.
2749  */
2750 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2751 			      size_t *rem)
2752 {
2753 	enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
2754 	struct intel_pt_pkt packet;
2755 	int ret;
2756 
2757 	while (len) {
2758 		ret = intel_pt_get_packet(buf, len, &packet, &ctx);
2759 		if (ret <= 0)
2760 			return false;
2761 		if (packet.type == INTEL_PT_TSC) {
2762 			*tsc = packet.payload;
2763 			*rem = len;
2764 			return true;
2765 		}
2766 		if (packet.type == INTEL_PT_PSBEND)
2767 			return false;
2768 		buf += ret;
2769 		len -= ret;
2770 	}
2771 	return false;
2772 }
2773 
2774 /**
2775  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2776  * @tsc1: first TSC to compare
2777  * @tsc2: second TSC to compare
2778  *
2779  * This function compares 7-byte TSC values allowing for the possibility that
2780  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2781  * around so for that purpose this function assumes the absolute difference is
2782  * less than half the maximum difference.
2783  *
2784  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2785  * after @tsc2.
2786  */
2787 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2788 {
2789 	const uint64_t halfway = (1ULL << 55);
2790 
2791 	if (tsc1 == tsc2)
2792 		return 0;
2793 
2794 	if (tsc1 < tsc2) {
2795 		if (tsc2 - tsc1 < halfway)
2796 			return -1;
2797 		else
2798 			return 1;
2799 	} else {
2800 		if (tsc1 - tsc2 < halfway)
2801 			return 1;
2802 		else
2803 			return -1;
2804 	}
2805 }
2806 
2807 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2808 
2809 /**
2810  * adj_for_padding - adjust overlap to account for padding.
2811  * @buf_b: second buffer
2812  * @buf_a: first buffer
2813  * @len_a: size of first buffer
2814  *
2815  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2816  * accordingly.
2817  *
2818  * Return: A pointer into @buf_b from where non-overlapped data starts
2819  */
2820 static unsigned char *adj_for_padding(unsigned char *buf_b,
2821 				      unsigned char *buf_a, size_t len_a)
2822 {
2823 	unsigned char *p = buf_b - MAX_PADDING;
2824 	unsigned char *q = buf_a + len_a - MAX_PADDING;
2825 	int i;
2826 
2827 	for (i = MAX_PADDING; i; i--, p++, q++) {
2828 		if (*p != *q)
2829 			break;
2830 	}
2831 
2832 	return p;
2833 }
2834 
2835 /**
2836  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2837  *                             using TSC.
2838  * @buf_a: first buffer
2839  * @len_a: size of first buffer
2840  * @buf_b: second buffer
2841  * @len_b: size of second buffer
2842  * @consecutive: returns true if there is data in buf_b that is consecutive
2843  *               to buf_a
2844  *
2845  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2846  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2847  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2848  * @buf_a and @buf_b are positioned at a PSB.
2849  *
2850  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2851  * @buf_b + @len_b if there is no non-overlapped data.
2852  */
2853 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2854 						size_t len_a,
2855 						unsigned char *buf_b,
2856 						size_t len_b, bool *consecutive)
2857 {
2858 	uint64_t tsc_a, tsc_b;
2859 	unsigned char *p;
2860 	size_t len, rem_a, rem_b;
2861 
2862 	p = intel_pt_last_psb(buf_a, len_a);
2863 	if (!p)
2864 		return buf_b; /* No PSB in buf_a => no overlap */
2865 
2866 	len = len_a - (p - buf_a);
2867 	if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2868 		/* The last PSB+ in buf_a is incomplete, so go back one more */
2869 		len_a -= len;
2870 		p = intel_pt_last_psb(buf_a, len_a);
2871 		if (!p)
2872 			return buf_b; /* No full PSB+ => assume no overlap */
2873 		len = len_a - (p - buf_a);
2874 		if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2875 			return buf_b; /* No TSC in buf_a => assume no overlap */
2876 	}
2877 
2878 	while (1) {
2879 		/* Ignore PSB+ with no TSC */
2880 		if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2881 			int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2882 
2883 			/* Same TSC, so buffers are consecutive */
2884 			if (!cmp && rem_b >= rem_a) {
2885 				unsigned char *start;
2886 
2887 				*consecutive = true;
2888 				start = buf_b + len_b - (rem_b - rem_a);
2889 				return adj_for_padding(start, buf_a, len_a);
2890 			}
2891 			if (cmp < 0)
2892 				return buf_b; /* tsc_a < tsc_b => no overlap */
2893 		}
2894 
2895 		if (!intel_pt_step_psb(&buf_b, &len_b))
2896 			return buf_b + len_b; /* No PSB in buf_b => no data */
2897 	}
2898 }
2899 
2900 /**
2901  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2902  * @buf_a: first buffer
2903  * @len_a: size of first buffer
2904  * @buf_b: second buffer
2905  * @len_b: size of second buffer
2906  * @have_tsc: can use TSC packets to detect overlap
2907  * @consecutive: returns true if there is data in buf_b that is consecutive
2908  *               to buf_a
2909  *
2910  * When trace samples or snapshots are recorded there is the possibility that
2911  * the data overlaps.  Note that, for the purposes of decoding, data is only
2912  * useful if it begins with a PSB packet.
2913  *
2914  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2915  * @buf_b + @len_b if there is no non-overlapped data.
2916  */
2917 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2918 				     unsigned char *buf_b, size_t len_b,
2919 				     bool have_tsc, bool *consecutive)
2920 {
2921 	unsigned char *found;
2922 
2923 	/* Buffer 'b' must start at PSB so throw away everything before that */
2924 	if (!intel_pt_next_psb(&buf_b, &len_b))
2925 		return buf_b + len_b; /* No PSB */
2926 
2927 	if (!intel_pt_next_psb(&buf_a, &len_a))
2928 		return buf_b; /* No overlap */
2929 
2930 	if (have_tsc) {
2931 		found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2932 						  consecutive);
2933 		if (found)
2934 			return found;
2935 	}
2936 
2937 	/*
2938 	 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2939 	 * we can ignore the first part of buffer 'a'.
2940 	 */
2941 	while (len_b < len_a) {
2942 		if (!intel_pt_step_psb(&buf_a, &len_a))
2943 			return buf_b; /* No overlap */
2944 	}
2945 
2946 	/* Now len_b >= len_a */
2947 	while (1) {
2948 		/* Potential overlap so check the bytes */
2949 		found = memmem(buf_a, len_a, buf_b, len_a);
2950 		if (found) {
2951 			*consecutive = true;
2952 			return adj_for_padding(buf_b + len_a, buf_a, len_a);
2953 		}
2954 
2955 		/* Try again at next PSB in buffer 'a' */
2956 		if (!intel_pt_step_psb(&buf_a, &len_a))
2957 			return buf_b; /* No overlap */
2958 	}
2959 }
2960 
2961 /**
2962  * struct fast_forward_data - data used by intel_pt_ff_cb().
2963  * @timestamp: timestamp to fast forward towards
2964  * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
2965  *                 the fast forward timestamp.
2966  */
2967 struct fast_forward_data {
2968 	uint64_t timestamp;
2969 	uint64_t buf_timestamp;
2970 };
2971 
2972 /**
2973  * intel_pt_ff_cb - fast forward lookahead callback.
2974  * @buffer: Intel PT trace buffer
2975  * @data: opaque pointer to fast forward data (struct fast_forward_data)
2976  *
2977  * Determine if @buffer trace is past the fast forward timestamp.
2978  *
2979  * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
2980  *         timestamp, and 0 otherwise.
2981  */
2982 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
2983 {
2984 	struct fast_forward_data *d = data;
2985 	unsigned char *buf;
2986 	uint64_t tsc;
2987 	size_t rem;
2988 	size_t len;
2989 
2990 	buf = (unsigned char *)buffer->buf;
2991 	len = buffer->len;
2992 
2993 	if (!intel_pt_next_psb(&buf, &len) ||
2994 	    !intel_pt_next_tsc(buf, len, &tsc, &rem))
2995 		return 0;
2996 
2997 	tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
2998 
2999 	intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
3000 		     tsc, buffer->ref_timestamp);
3001 
3002 	/*
3003 	 * If the buffer contains a timestamp earlier that the fast forward
3004 	 * timestamp, then record it, else stop.
3005 	 */
3006 	if (tsc < d->timestamp)
3007 		d->buf_timestamp = buffer->ref_timestamp;
3008 	else
3009 		return 1;
3010 
3011 	return 0;
3012 }
3013 
3014 /**
3015  * intel_pt_fast_forward - reposition decoder forwards.
3016  * @decoder: Intel PT decoder
3017  * @timestamp: timestamp to fast forward towards
3018  *
3019  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
3020  *
3021  * Return: 0 on success or negative error code on failure.
3022  */
3023 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
3024 {
3025 	struct fast_forward_data d = { .timestamp = timestamp };
3026 	unsigned char *buf;
3027 	size_t len;
3028 	int err;
3029 
3030 	intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
3031 
3032 	/* Find buffer timestamp of buffer to fast forward to */
3033 	err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
3034 	if (err < 0)
3035 		return err;
3036 
3037 	/* Walk to buffer with same buffer timestamp */
3038 	if (d.buf_timestamp) {
3039 		do {
3040 			decoder->pos += decoder->len;
3041 			decoder->len = 0;
3042 			err = intel_pt_get_next_data(decoder, true);
3043 			/* -ENOLINK means non-consecutive trace */
3044 			if (err && err != -ENOLINK)
3045 				return err;
3046 		} while (decoder->buf_timestamp != d.buf_timestamp);
3047 	}
3048 
3049 	if (!decoder->buf)
3050 		return 0;
3051 
3052 	buf = (unsigned char *)decoder->buf;
3053 	len = decoder->len;
3054 
3055 	if (!intel_pt_next_psb(&buf, &len))
3056 		return 0;
3057 
3058 	/*
3059 	 * Walk PSBs while the PSB timestamp is less than the fast forward
3060 	 * timestamp.
3061 	 */
3062 	do {
3063 		uint64_t tsc;
3064 		size_t rem;
3065 
3066 		if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
3067 			break;
3068 		tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
3069 		/*
3070 		 * A TSC packet can slip past MTC packets but, after fast
3071 		 * forward, decoding starts at the TSC timestamp. That means
3072 		 * the timestamps may not be exactly the same as the timestamps
3073 		 * that would have been decoded without fast forward.
3074 		 */
3075 		if (tsc < timestamp) {
3076 			intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
3077 			decoder->pos += decoder->len - len;
3078 			decoder->buf = buf;
3079 			decoder->len = len;
3080 			intel_pt_reposition(decoder);
3081 		} else {
3082 			break;
3083 		}
3084 	} while (intel_pt_step_psb(&buf, &len));
3085 
3086 	return 0;
3087 }
3088