1 // SPDX-License-Identifier: GPL-2.0
3 * Arm Statistical Profiling Extensions (SPE) support
4 * Copyright (c) 2017-2018, Arm Ltd.
11 #include <linux/bitops.h>
12 #include <linux/kernel.h>
13 #include <linux/log2.h>
14 #include <linux/types.h>
15 #include <linux/zalloc.h>
28 #include "thread-stack.h"
30 #include "util/synthetic-events.h"
33 #include "arm-spe-decoder/arm-spe-decoder.h"
34 #include "arm-spe-decoder/arm-spe-pkt-decoder.h"
36 #define MAX_TIMESTAMP (~0ULL)
39 struct auxtrace auxtrace;
40 struct auxtrace_queues queues;
41 struct auxtrace_heap heap;
42 struct itrace_synth_opts synth_opts;
44 struct perf_session *session;
45 struct machine *machine;
55 u8 sample_remote_access;
70 unsigned long num_events;
73 struct arm_spe_queue {
75 unsigned int queue_nr;
76 struct auxtrace_buffer *buffer;
77 struct auxtrace_buffer *old_buffer;
78 union perf_event *event_buf;
84 struct arm_spe_decoder *decoder;
87 struct thread *thread;
90 static void arm_spe_dump(struct arm_spe *spe __maybe_unused,
91 unsigned char *buf, size_t len)
93 struct arm_spe_pkt packet;
96 char desc[ARM_SPE_PKT_DESC_MAX];
97 const char *color = PERF_COLOR_BLUE;
99 color_fprintf(stdout, color,
100 ". ... ARM SPE data: size %zu bytes\n",
104 ret = arm_spe_get_packet(buf, len, &packet);
110 color_fprintf(stdout, color, " %08x: ", pos);
111 for (i = 0; i < pkt_len; i++)
112 color_fprintf(stdout, color, " %02x", buf[i]);
114 color_fprintf(stdout, color, " ");
116 ret = arm_spe_pkt_desc(&packet, desc,
117 ARM_SPE_PKT_DESC_MAX);
119 color_fprintf(stdout, color, " %s\n", desc);
121 color_fprintf(stdout, color, " Bad packet!\n");
129 static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf,
133 arm_spe_dump(spe, buf, len);
136 static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data)
138 struct arm_spe_queue *speq = data;
139 struct auxtrace_buffer *buffer = speq->buffer;
140 struct auxtrace_buffer *old_buffer = speq->old_buffer;
141 struct auxtrace_queue *queue;
143 queue = &speq->spe->queues.queue_array[speq->queue_nr];
145 buffer = auxtrace_buffer__next(queue, buffer);
146 /* If no more data, drop the previous auxtrace_buffer and return */
149 auxtrace_buffer__drop_data(old_buffer);
154 speq->buffer = buffer;
156 /* If the aux_buffer doesn't have data associated, try to load it */
158 /* get the file desc associated with the perf data file */
159 int fd = perf_data__fd(speq->spe->session->data);
161 buffer->data = auxtrace_buffer__get_data(buffer, fd);
166 b->len = buffer->size;
167 b->buf = buffer->data;
171 auxtrace_buffer__drop_data(old_buffer);
172 speq->old_buffer = buffer;
174 auxtrace_buffer__drop_data(buffer);
175 return arm_spe_get_trace(b, data);
181 static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe,
182 unsigned int queue_nr)
184 struct arm_spe_params params = { .get_trace = 0, };
185 struct arm_spe_queue *speq;
187 speq = zalloc(sizeof(*speq));
191 speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
192 if (!speq->event_buf)
196 speq->queue_nr = queue_nr;
202 params.get_trace = arm_spe_get_trace;
205 /* create new decoder */
206 speq->decoder = arm_spe_decoder_new(¶ms);
213 zfree(&speq->event_buf);
219 static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip)
221 return ip >= spe->kernel_start ?
222 PERF_RECORD_MISC_KERNEL :
223 PERF_RECORD_MISC_USER;
226 static void arm_spe_prep_sample(struct arm_spe *spe,
227 struct arm_spe_queue *speq,
228 union perf_event *event,
229 struct perf_sample *sample)
231 struct arm_spe_record *record = &speq->decoder->record;
233 if (!spe->timeless_decoding)
234 sample->time = speq->timestamp;
236 sample->ip = record->from_ip;
237 sample->cpumode = arm_spe_cpumode(spe, sample->ip);
238 sample->pid = speq->pid;
239 sample->tid = speq->tid;
241 sample->cpu = speq->cpu;
243 event->sample.header.type = PERF_RECORD_SAMPLE;
244 event->sample.header.misc = sample->cpumode;
245 event->sample.header.size = sizeof(struct perf_event_header);
249 arm_spe_deliver_synth_event(struct arm_spe *spe,
250 struct arm_spe_queue *speq __maybe_unused,
251 union perf_event *event,
252 struct perf_sample *sample)
256 ret = perf_session__deliver_synth_event(spe->session, event, sample);
258 pr_err("ARM SPE: failed to deliver event, error %d\n", ret);
263 static int arm_spe__synth_mem_sample(struct arm_spe_queue *speq,
264 u64 spe_events_id, u64 data_src)
266 struct arm_spe *spe = speq->spe;
267 struct arm_spe_record *record = &speq->decoder->record;
268 union perf_event *event = speq->event_buf;
269 struct perf_sample sample = { .ip = 0, };
271 arm_spe_prep_sample(spe, speq, event, &sample);
273 sample.id = spe_events_id;
274 sample.stream_id = spe_events_id;
275 sample.addr = record->virt_addr;
276 sample.phys_addr = record->phys_addr;
277 sample.data_src = data_src;
279 return arm_spe_deliver_synth_event(spe, speq, event, &sample);
282 static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq,
285 struct arm_spe *spe = speq->spe;
286 struct arm_spe_record *record = &speq->decoder->record;
287 union perf_event *event = speq->event_buf;
288 struct perf_sample sample = { .ip = 0, };
290 arm_spe_prep_sample(spe, speq, event, &sample);
292 sample.id = spe_events_id;
293 sample.stream_id = spe_events_id;
294 sample.addr = record->to_ip;
296 return arm_spe_deliver_synth_event(spe, speq, event, &sample);
299 #define SPE_MEM_TYPE (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS | \
300 ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS | \
301 ARM_SPE_REMOTE_ACCESS)
303 static bool arm_spe__is_memory_event(enum arm_spe_sample_type type)
305 if (type & SPE_MEM_TYPE)
311 static u64 arm_spe__synth_data_source(const struct arm_spe_record *record)
313 union perf_mem_data_src data_src = { 0 };
315 if (record->op == ARM_SPE_LD)
316 data_src.mem_op = PERF_MEM_OP_LOAD;
318 data_src.mem_op = PERF_MEM_OP_STORE;
320 if (record->type & (ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS)) {
321 data_src.mem_lvl = PERF_MEM_LVL_L3;
323 if (record->type & ARM_SPE_LLC_MISS)
324 data_src.mem_lvl |= PERF_MEM_LVL_MISS;
326 data_src.mem_lvl |= PERF_MEM_LVL_HIT;
327 } else if (record->type & (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS)) {
328 data_src.mem_lvl = PERF_MEM_LVL_L1;
330 if (record->type & ARM_SPE_L1D_MISS)
331 data_src.mem_lvl |= PERF_MEM_LVL_MISS;
333 data_src.mem_lvl |= PERF_MEM_LVL_HIT;
336 if (record->type & ARM_SPE_REMOTE_ACCESS)
337 data_src.mem_lvl |= PERF_MEM_LVL_REM_CCE1;
339 if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) {
340 data_src.mem_dtlb = PERF_MEM_TLB_WK;
342 if (record->type & ARM_SPE_TLB_MISS)
343 data_src.mem_dtlb |= PERF_MEM_TLB_MISS;
345 data_src.mem_dtlb |= PERF_MEM_TLB_HIT;
351 static int arm_spe_sample(struct arm_spe_queue *speq)
353 const struct arm_spe_record *record = &speq->decoder->record;
354 struct arm_spe *spe = speq->spe;
358 data_src = arm_spe__synth_data_source(record);
360 if (spe->sample_flc) {
361 if (record->type & ARM_SPE_L1D_MISS) {
362 err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id,
368 if (record->type & ARM_SPE_L1D_ACCESS) {
369 err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id,
376 if (spe->sample_llc) {
377 if (record->type & ARM_SPE_LLC_MISS) {
378 err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id,
384 if (record->type & ARM_SPE_LLC_ACCESS) {
385 err = arm_spe__synth_mem_sample(speq, spe->llc_access_id,
392 if (spe->sample_tlb) {
393 if (record->type & ARM_SPE_TLB_MISS) {
394 err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id,
400 if (record->type & ARM_SPE_TLB_ACCESS) {
401 err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id,
408 if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) {
409 err = arm_spe__synth_branch_sample(speq, spe->branch_miss_id);
414 if (spe->sample_remote_access &&
415 (record->type & ARM_SPE_REMOTE_ACCESS)) {
416 err = arm_spe__synth_mem_sample(speq, spe->remote_access_id,
422 if (spe->sample_memory && arm_spe__is_memory_event(record->type)) {
423 err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src);
431 static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp)
433 struct arm_spe *spe = speq->spe;
436 if (!spe->kernel_start)
437 spe->kernel_start = machine__kernel_start(spe->machine);
440 ret = arm_spe_decode(speq->decoder);
442 pr_debug("No data or all data has been processed.\n");
447 * Error is detected when decode SPE trace data, continue to
448 * the next trace data and find out more records.
453 ret = arm_spe_sample(speq);
457 if (!spe->timeless_decoding && speq->timestamp >= *timestamp) {
458 *timestamp = speq->timestamp;
466 static int arm_spe__setup_queue(struct arm_spe *spe,
467 struct auxtrace_queue *queue,
468 unsigned int queue_nr)
470 struct arm_spe_queue *speq = queue->priv;
471 struct arm_spe_record *record;
473 if (list_empty(&queue->head) || speq)
476 speq = arm_spe__alloc_queue(spe, queue_nr);
483 if (queue->cpu != -1)
484 speq->cpu = queue->cpu;
486 if (!speq->on_heap) {
489 if (spe->timeless_decoding)
493 ret = arm_spe_decode(speq->decoder);
501 record = &speq->decoder->record;
503 speq->timestamp = record->timestamp;
504 ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp);
507 speq->on_heap = true;
513 static int arm_spe__setup_queues(struct arm_spe *spe)
518 for (i = 0; i < spe->queues.nr_queues; i++) {
519 ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i);
527 static int arm_spe__update_queues(struct arm_spe *spe)
529 if (spe->queues.new_data) {
530 spe->queues.new_data = false;
531 return arm_spe__setup_queues(spe);
537 static bool arm_spe__is_timeless_decoding(struct arm_spe *spe)
540 struct evlist *evlist = spe->session->evlist;
541 bool timeless_decoding = true;
544 * Circle through the list of event and complain if we find one
545 * with the time bit set.
547 evlist__for_each_entry(evlist, evsel) {
548 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
549 timeless_decoding = false;
552 return timeless_decoding;
555 static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe,
556 struct auxtrace_queue *queue)
558 struct arm_spe_queue *speq = queue->priv;
561 tid = machine__get_current_tid(spe->machine, speq->cpu);
564 thread__zput(speq->thread);
566 speq->tid = queue->tid;
568 if ((!speq->thread) && (speq->tid != -1)) {
569 speq->thread = machine__find_thread(spe->machine, -1,
574 speq->pid = speq->thread->pid_;
575 if (queue->cpu == -1)
576 speq->cpu = speq->thread->cpu;
580 static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp)
582 unsigned int queue_nr;
587 struct auxtrace_queue *queue;
588 struct arm_spe_queue *speq;
590 if (!spe->heap.heap_cnt)
593 if (spe->heap.heap_array[0].ordinal >= timestamp)
596 queue_nr = spe->heap.heap_array[0].queue_nr;
597 queue = &spe->queues.queue_array[queue_nr];
600 auxtrace_heap__pop(&spe->heap);
602 if (spe->heap.heap_cnt) {
603 ts = spe->heap.heap_array[0].ordinal + 1;
610 arm_spe_set_pid_tid_cpu(spe, queue);
612 ret = arm_spe_run_decoder(speq, &ts);
614 auxtrace_heap__add(&spe->heap, queue_nr, ts);
619 ret = auxtrace_heap__add(&spe->heap, queue_nr, ts);
623 speq->on_heap = false;
630 static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid,
633 struct auxtrace_queues *queues = &spe->queues;
637 for (i = 0; i < queues->nr_queues; i++) {
638 struct auxtrace_queue *queue = &spe->queues.queue_array[i];
639 struct arm_spe_queue *speq = queue->priv;
641 if (speq && (tid == -1 || speq->tid == tid)) {
643 arm_spe_set_pid_tid_cpu(spe, queue);
644 arm_spe_run_decoder(speq, &ts);
650 static int arm_spe_process_event(struct perf_session *session,
651 union perf_event *event,
652 struct perf_sample *sample,
653 struct perf_tool *tool)
657 struct arm_spe *spe = container_of(session->auxtrace,
658 struct arm_spe, auxtrace);
663 if (!tool->ordered_events) {
664 pr_err("SPE trace requires ordered events\n");
668 if (sample->time && (sample->time != (u64) -1))
669 timestamp = sample->time;
673 if (timestamp || spe->timeless_decoding) {
674 err = arm_spe__update_queues(spe);
679 if (spe->timeless_decoding) {
680 if (event->header.type == PERF_RECORD_EXIT) {
681 err = arm_spe_process_timeless_queues(spe,
685 } else if (timestamp) {
686 if (event->header.type == PERF_RECORD_EXIT) {
687 err = arm_spe_process_queues(spe, timestamp);
696 static int arm_spe_process_auxtrace_event(struct perf_session *session,
697 union perf_event *event,
698 struct perf_tool *tool __maybe_unused)
700 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
703 if (!spe->data_queued) {
704 struct auxtrace_buffer *buffer;
706 int fd = perf_data__fd(session->data);
709 if (perf_data__is_pipe(session->data)) {
712 data_offset = lseek(fd, 0, SEEK_CUR);
713 if (data_offset == -1)
717 err = auxtrace_queues__add_event(&spe->queues, session, event,
718 data_offset, &buffer);
722 /* Dump here now we have copied a piped trace out of the pipe */
724 if (auxtrace_buffer__get_data(buffer, fd)) {
725 arm_spe_dump_event(spe, buffer->data,
727 auxtrace_buffer__put_data(buffer);
735 static int arm_spe_flush(struct perf_session *session __maybe_unused,
736 struct perf_tool *tool __maybe_unused)
738 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
745 if (!tool->ordered_events)
748 ret = arm_spe__update_queues(spe);
752 if (spe->timeless_decoding)
753 return arm_spe_process_timeless_queues(spe, -1,
756 return arm_spe_process_queues(spe, MAX_TIMESTAMP);
759 static void arm_spe_free_queue(void *priv)
761 struct arm_spe_queue *speq = priv;
765 thread__zput(speq->thread);
766 arm_spe_decoder_free(speq->decoder);
767 zfree(&speq->event_buf);
771 static void arm_spe_free_events(struct perf_session *session)
773 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
775 struct auxtrace_queues *queues = &spe->queues;
778 for (i = 0; i < queues->nr_queues; i++) {
779 arm_spe_free_queue(queues->queue_array[i].priv);
780 queues->queue_array[i].priv = NULL;
782 auxtrace_queues__free(queues);
785 static void arm_spe_free(struct perf_session *session)
787 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
790 auxtrace_heap__free(&spe->heap);
791 arm_spe_free_events(session);
792 session->auxtrace = NULL;
796 static bool arm_spe_evsel_is_auxtrace(struct perf_session *session,
799 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace);
801 return evsel->core.attr.type == spe->pmu_type;
804 static const char * const arm_spe_info_fmts[] = {
805 [ARM_SPE_PMU_TYPE] = " PMU Type %"PRId64"\n",
808 static void arm_spe_print_info(__u64 *arr)
813 fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]);
816 struct arm_spe_synth {
817 struct perf_tool dummy_tool;
818 struct perf_session *session;
821 static int arm_spe_event_synth(struct perf_tool *tool,
822 union perf_event *event,
823 struct perf_sample *sample __maybe_unused,
824 struct machine *machine __maybe_unused)
826 struct arm_spe_synth *arm_spe_synth =
827 container_of(tool, struct arm_spe_synth, dummy_tool);
829 return perf_session__deliver_synth_event(arm_spe_synth->session,
833 static int arm_spe_synth_event(struct perf_session *session,
834 struct perf_event_attr *attr, u64 id)
836 struct arm_spe_synth arm_spe_synth;
838 memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth));
839 arm_spe_synth.session = session;
841 return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1,
842 &id, arm_spe_event_synth);
845 static void arm_spe_set_event_name(struct evlist *evlist, u64 id,
850 evlist__for_each_entry(evlist, evsel) {
851 if (evsel->core.id && evsel->core.id[0] == id) {
854 evsel->name = strdup(name);
861 arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)
863 struct evlist *evlist = session->evlist;
865 struct perf_event_attr attr;
870 evlist__for_each_entry(evlist, evsel) {
871 if (evsel->core.attr.type == spe->pmu_type) {
878 pr_debug("No selected events with SPE trace data\n");
882 memset(&attr, 0, sizeof(struct perf_event_attr));
883 attr.size = sizeof(struct perf_event_attr);
884 attr.type = PERF_TYPE_HARDWARE;
885 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
886 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
887 PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC;
888 if (spe->timeless_decoding)
889 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
891 attr.sample_type |= PERF_SAMPLE_TIME;
893 attr.exclude_user = evsel->core.attr.exclude_user;
894 attr.exclude_kernel = evsel->core.attr.exclude_kernel;
895 attr.exclude_hv = evsel->core.attr.exclude_hv;
896 attr.exclude_host = evsel->core.attr.exclude_host;
897 attr.exclude_guest = evsel->core.attr.exclude_guest;
898 attr.sample_id_all = evsel->core.attr.sample_id_all;
899 attr.read_format = evsel->core.attr.read_format;
901 /* create new id val to be a fixed offset from evsel id */
902 id = evsel->core.id[0] + 1000000000;
907 if (spe->synth_opts.flc) {
908 spe->sample_flc = true;
910 /* Level 1 data cache miss */
911 err = arm_spe_synth_event(session, &attr, id);
914 spe->l1d_miss_id = id;
915 arm_spe_set_event_name(evlist, id, "l1d-miss");
918 /* Level 1 data cache access */
919 err = arm_spe_synth_event(session, &attr, id);
922 spe->l1d_access_id = id;
923 arm_spe_set_event_name(evlist, id, "l1d-access");
927 if (spe->synth_opts.llc) {
928 spe->sample_llc = true;
930 /* Last level cache miss */
931 err = arm_spe_synth_event(session, &attr, id);
934 spe->llc_miss_id = id;
935 arm_spe_set_event_name(evlist, id, "llc-miss");
938 /* Last level cache access */
939 err = arm_spe_synth_event(session, &attr, id);
942 spe->llc_access_id = id;
943 arm_spe_set_event_name(evlist, id, "llc-access");
947 if (spe->synth_opts.tlb) {
948 spe->sample_tlb = true;
951 err = arm_spe_synth_event(session, &attr, id);
954 spe->tlb_miss_id = id;
955 arm_spe_set_event_name(evlist, id, "tlb-miss");
959 err = arm_spe_synth_event(session, &attr, id);
962 spe->tlb_access_id = id;
963 arm_spe_set_event_name(evlist, id, "tlb-access");
967 if (spe->synth_opts.branches) {
968 spe->sample_branch = true;
971 err = arm_spe_synth_event(session, &attr, id);
974 spe->branch_miss_id = id;
975 arm_spe_set_event_name(evlist, id, "branch-miss");
979 if (spe->synth_opts.remote_access) {
980 spe->sample_remote_access = true;
983 err = arm_spe_synth_event(session, &attr, id);
986 spe->remote_access_id = id;
987 arm_spe_set_event_name(evlist, id, "remote-access");
991 if (spe->synth_opts.mem) {
992 spe->sample_memory = true;
994 err = arm_spe_synth_event(session, &attr, id);
998 arm_spe_set_event_name(evlist, id, "memory");
1004 int arm_spe_process_auxtrace_info(union perf_event *event,
1005 struct perf_session *session)
1007 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
1008 size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX;
1009 struct arm_spe *spe;
1012 if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
1016 spe = zalloc(sizeof(struct arm_spe));
1020 err = auxtrace_queues__init(&spe->queues);
1024 spe->session = session;
1025 spe->machine = &session->machines.host; /* No kvm support */
1026 spe->auxtrace_type = auxtrace_info->type;
1027 spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE];
1029 spe->timeless_decoding = arm_spe__is_timeless_decoding(spe);
1030 spe->auxtrace.process_event = arm_spe_process_event;
1031 spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event;
1032 spe->auxtrace.flush_events = arm_spe_flush;
1033 spe->auxtrace.free_events = arm_spe_free_events;
1034 spe->auxtrace.free = arm_spe_free;
1035 spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace;
1036 session->auxtrace = &spe->auxtrace;
1038 arm_spe_print_info(&auxtrace_info->priv[0]);
1043 if (session->itrace_synth_opts && session->itrace_synth_opts->set)
1044 spe->synth_opts = *session->itrace_synth_opts;
1046 itrace_synth_opts__set_default(&spe->synth_opts, false);
1048 err = arm_spe_synth_events(spe, session);
1050 goto err_free_queues;
1052 err = auxtrace_queues__process_index(&spe->queues, session);
1054 goto err_free_queues;
1056 if (spe->queues.populated)
1057 spe->data_queued = true;
1062 auxtrace_queues__free(&spe->queues);
1063 session->auxtrace = NULL;