1 // SPDX-License-Identifier: GPL-2.0
3 * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
4 * sink device could then pair with an appropriate per-cpu coresight source
5 * device (ETE) thus generating required trace data. Trace can be enabled
6 * via the perf framework.
8 * The AUX buffer handling is inspired from Arm SPE PMU driver.
10 * Copyright (C) 2020 ARM Ltd.
12 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
14 #define DRVNAME "arm_trbe"
16 #define pr_fmt(fmt) DRVNAME ": " fmt
18 #include <asm/barrier.h>
19 #include <asm/cpufeature.h>
21 #include "coresight-self-hosted-trace.h"
22 #include "coresight-trbe.h"
24 #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
27 * A padding packet that will help the user space tools
28 * in skipping relevant sections in the captured trace
29 * data which could not be decoded. TRBE doesn't support
30 * formatting the trace data, unlike the legacy CoreSight
31 * sinks and thus we use ETE trace packets to pad the
32 * sections of the buffer.
34 #define ETE_IGNORE_PACKET 0x70
37 * Minimum amount of meaningful trace will contain:
38 * A-Sync, Trace Info, Trace On, Address, Atom.
39 * This is about 44bytes of ETE trace. To be on
40 * the safer side, we assume 64bytes is the minimum
41 * space required for a meaningful session, before
42 * we hit a "WRAP" event.
44 #define TRBE_TRACE_MIN_BUF_SIZE 64
46 enum trbe_fault_action {
48 TRBE_FAULT_ACT_SPURIOUS,
54 * Even though trbe_base represents vmap()
55 * mapped allocated buffer's start address,
56 * it's being as unsigned long for various
57 * arithmetic and comparision operations &
58 * also to be consistent with trbe_write &
59 * trbe_limit sibling pointers.
61 unsigned long trbe_base;
62 /* The base programmed into the TRBE */
63 unsigned long trbe_hw_base;
64 unsigned long trbe_limit;
65 unsigned long trbe_write;
69 struct trbe_cpudata *cpudata;
75 * The errata are defined in arm64 generic cpu_errata framework.
76 * Since the errata work arounds could be applied individually
77 * to the affected CPUs inside the TRBE driver, we need to know if
78 * a given CPU is affected by the erratum. Unlike the other erratum
79 * work arounds, TRBE driver needs to check multiple times during
80 * a trace session. Thus we need a quicker access to per-CPU
81 * errata and not issue costly this_cpu_has_cap() everytime.
82 * We keep a set of the affected errata in trbe_cpudata, per TRBE.
84 * We rely on the corresponding cpucaps to be defined for a given
85 * TRBE erratum. We map the given cpucap into a TRBE internal number
86 * to make the tracking of the errata lean.
89 * - Not duplicating the detection logic
90 * - Streamlined detection of erratum across the system
93 static int trbe_errata_cpucaps[] = {
94 -1, /* Sentinel, must be the last entry */
97 /* The total number of listed errata in trbe_errata_cpucaps */
98 #define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1)
101 * struct trbe_cpudata: TRBE instance specific data
102 * @trbe_flag - TRBE dirty/access flag support
103 * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1.
104 * @trbe_align - Software alignment used for the TRBPTR_EL1.
105 * @cpu - CPU this TRBE belongs to.
106 * @mode - Mode of current operation. (perf/disabled)
107 * @drvdata - TRBE specific drvdata
108 * @errata - Bit map for the errata on this TRBE.
110 struct trbe_cpudata {
116 struct trbe_buf *buf;
117 struct trbe_drvdata *drvdata;
118 DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
121 struct trbe_drvdata {
122 struct trbe_cpudata __percpu *cpudata;
123 struct perf_output_handle * __percpu *handle;
124 struct hlist_node hotplug_node;
126 cpumask_t supported_cpus;
127 enum cpuhp_state trbe_online;
128 struct platform_device *pdev;
131 static void trbe_check_errata(struct trbe_cpudata *cpudata)
135 for (i = 0; i < TRBE_ERRATA_MAX; i++) {
136 int cap = trbe_errata_cpucaps[i];
138 if (WARN_ON_ONCE(cap < 0))
140 if (this_cpu_has_cap(cap))
141 set_bit(i, cpudata->errata);
145 static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
147 return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
150 static int trbe_alloc_node(struct perf_event *event)
152 if (event->cpu == -1)
154 return cpu_to_node(event->cpu);
157 static void trbe_drain_buffer(void)
163 static void trbe_drain_and_disable_local(void)
165 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
170 * Disable the TRBE without clearing LIMITPTR which
171 * might be required for fetching the buffer limits.
173 trblimitr &= ~TRBLIMITR_ENABLE;
174 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
178 static void trbe_reset_local(void)
180 trbe_drain_and_disable_local();
181 write_sysreg_s(0, SYS_TRBLIMITR_EL1);
182 write_sysreg_s(0, SYS_TRBPTR_EL1);
183 write_sysreg_s(0, SYS_TRBBASER_EL1);
184 write_sysreg_s(0, SYS_TRBSR_EL1);
187 static void trbe_report_wrap_event(struct perf_output_handle *handle)
190 * Mark the buffer to indicate that there was a WRAP event by
191 * setting the COLLISION flag. This indicates to the user that
192 * the TRBE trace collection was stopped without stopping the
193 * ETE and thus there might be some amount of trace that was
194 * lost between the time the WRAP was detected and the IRQ
195 * was consumed by the CPU.
197 * Setting the TRUNCATED flag would move the event to STOPPED
198 * state unnecessarily, even when there is space left in the
199 * ring buffer. Using the COLLISION flag doesn't have this side
200 * effect. We only set TRUNCATED flag when there is no space
201 * left in the ring buffer.
203 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
206 static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
208 struct trbe_buf *buf = etm_perf_sink_config(handle);
211 * We cannot proceed with the buffer collection and we
212 * do not have any data for the current session. The
213 * etm_perf driver expects to close out the aux_buffer
214 * at event_stop(). So disable the TRBE here and leave
215 * the update_buffer() to return a 0 size.
217 trbe_drain_and_disable_local();
218 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
219 perf_aux_output_end(handle, 0);
220 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
224 * TRBE Buffer Management
226 * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
227 * it starts writing trace data from the write pointer onward till the limit pointer.
228 * When the write pointer reaches the address just before the limit pointer, it gets
229 * wrapped around again to the base pointer. This is called a TRBE wrap event, which
230 * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
231 * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
232 * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
235 * Wrap around with an IRQ
236 * ------ < ------ < ------- < ----- < -----
238 * ------ > ------ > ------- > ----- > -----
240 * +---------------+-----------------------+
242 * +---------------+-----------------------+
243 * Base Pointer Write Pointer Limit Pointer
245 * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
246 * pointer can be aligned to the implementation defined TRBE trace buffer alignment
247 * as captured in trbe_cpudata->trbe_align.
251 * +---------------------------------------+----- ~ ~ ------
252 * |$$$$$$$|################|$$$$$$$$$$$$$$| |
253 * +---------------------------------------+----- ~ ~ ------
254 * Base Pointer Write Pointer Limit Pointer
256 * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
257 * values which tracks all the driver writes and user reads from the perf auxiliary
258 * buffer. Generally [head..tail] is the area where the driver can write into unless
259 * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
260 * configured depending on the perf_output_handle indices, so that the driver does
261 * not override into areas in the perf auxiliary buffer which is being or yet to be
262 * consumed from the user space. The enabled TRBE buffer area is a moving subset of
263 * the allocated perf auxiliary buffer.
266 static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
268 memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
271 static void trbe_pad_buf(struct perf_output_handle *handle, int len)
273 struct trbe_buf *buf = etm_perf_sink_config(handle);
274 u64 head = PERF_IDX2OFF(handle->head, buf);
276 __trbe_pad_buf(buf, head, len);
278 perf_aux_output_skip(handle, len);
281 static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
283 struct trbe_buf *buf = etm_perf_sink_config(handle);
286 * The ETE trace has alignment synchronization packets allowing
287 * the decoder to reset in case of an overflow or corruption.
288 * So we can use the entire buffer for the snapshot mode.
290 return buf->nr_pages * PAGE_SIZE;
294 * TRBE Limit Calculation
296 * The following markers are used to illustrate various TRBE buffer situations.
298 * $$$$ - Data area, unconsumed captured trace data, not to be overridden
299 * #### - Free area, enabled, trace will be written
300 * %%%% - Free area, disabled, trace will not be written
301 * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
303 static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
305 struct trbe_buf *buf = etm_perf_sink_config(handle);
306 struct trbe_cpudata *cpudata = buf->cpudata;
307 const u64 bufsize = buf->nr_pages * PAGE_SIZE;
309 u64 head, tail, wakeup;
311 head = PERF_IDX2OFF(handle->head, buf);
317 * head TRBE align tail
318 * +----|-------|---------------|-------+
319 * |$$$$|=======|###############|$$$$$$$|
320 * +----|-------|---------------|-------+
321 * trbe_base trbe_base + nr_pages
323 * Perf aux buffer output head position can be misaligned depending on
324 * various factors including user space reads. In case misaligned, head
325 * needs to be aligned before TRBE can be configured. Pad the alignment
326 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
327 * and skip this section thus advancing the head.
329 if (!IS_ALIGNED(head, cpudata->trbe_align)) {
330 unsigned long delta = roundup(head, cpudata->trbe_align) - head;
332 delta = min(delta, handle->size);
333 trbe_pad_buf(handle, delta);
334 head = PERF_IDX2OFF(handle->head, buf);
338 * head = tail (size = 0)
339 * +----|-------------------------------+
340 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ |
341 * +----|-------------------------------+
342 * trbe_base trbe_base + nr_pages
344 * Perf aux buffer does not have any space for the driver to write into.
349 /* Compute the tail and wakeup indices now that we've aligned head */
350 tail = PERF_IDX2OFF(handle->head + handle->size, buf);
351 wakeup = PERF_IDX2OFF(handle->wakeup, buf);
354 * Lets calculate the buffer area which TRBE could write into. There
355 * are three possible scenarios here. Limit needs to be aligned with
356 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
362 * +----|-----------------------|-------+
363 * |$$$$|#######################|$$$$$$$|
364 * +----|-----------------------|-------+
365 * trbe_base limit trbe_base + nr_pages
367 * TRBE could write into [head..tail] area. Unless the tail is right at
368 * the end of the buffer, neither an wrap around nor an IRQ is expected
369 * while being enabled.
373 * head = tail (size > 0)
374 * +----|-------------------------------+
375 * |%%%%|###############################|
376 * +----|-------------------------------+
377 * trbe_base limit = trbe_base + nr_pages
379 * TRBE should just write into [head..base + nr_pages] area even though
380 * the entire buffer is empty. Reason being, when the trace reaches the
381 * end of the buffer, it will just wrap around with an IRQ giving an
382 * opportunity to reconfigure the buffer.
387 * +----|-----------------------|-------+
388 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
389 * +----|-----------------------|-------+
390 * trbe_base limit = trbe_base + nr_pages
392 * TRBE should just write into [head..base + nr_pages] area even though
393 * the [trbe_base..tail] is also empty. Reason being, when the trace
394 * reaches the end of the buffer, it will just wrap around with an IRQ
395 * giving an opportunity to reconfigure the buffer.
398 limit = round_down(tail, PAGE_SIZE);
401 * Wakeup may be arbitrarily far into the future. If it's not in the
402 * current generation, either we'll wrap before hitting it, or it's
403 * in the past and has been handled already.
405 * If there's a wakeup before we wrap, arrange to be woken up by the
406 * page boundary following it. Keep the tail boundary if that's lower.
409 * +----|---------------|-------|-------+
410 * |$$$$|###############|%%%%%%%|$$$$$$$|
411 * +----|---------------|-------|-------+
412 * trbe_base limit trbe_base + nr_pages
414 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
415 limit = min(limit, round_up(wakeup, PAGE_SIZE));
418 * There are two situation when this can happen i.e limit is before
419 * the head and hence TRBE cannot be configured.
421 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
422 * within the same PAGE size range.
425 * |----------------------|
428 * +------------|------|--------|-------+
429 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
430 * +------------|------|--------|-------+
431 * trbe_base trbe_base + nr_pages
433 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
434 * head and wakeup are within same PAGE size range.
437 * |----------------------|
439 * limit head wakeup tail
440 * +----|------|-------|--------|-------+
441 * |$$$$$$$$$$$|=======|========|$$$$$$$|
442 * +----|------|-------|--------|-------+
443 * trbe_base trbe_base + nr_pages
448 trbe_pad_buf(handle, handle->size);
452 static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
454 struct trbe_buf *buf = etm_perf_sink_config(handle);
455 u64 limit = __trbe_normal_offset(handle);
456 u64 head = PERF_IDX2OFF(handle->head, buf);
459 * If the head is too close to the limit and we don't
460 * have space for a meaningful run, we rather pad it
463 if (limit && (limit - head < TRBE_TRACE_MIN_BUF_SIZE)) {
464 trbe_pad_buf(handle, limit - head);
465 limit = __trbe_normal_offset(handle);
470 static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
472 struct trbe_buf *buf = etm_perf_sink_config(handle);
473 unsigned long offset;
476 offset = trbe_snapshot_offset(handle);
478 offset = trbe_normal_offset(handle);
479 return buf->trbe_base + offset;
482 static void clr_trbe_status(void)
484 u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
486 WARN_ON(is_trbe_enabled());
489 trbsr &= ~TRBSR_WRAP;
490 trbsr &= ~(TRBSR_EC_MASK << TRBSR_EC_SHIFT);
491 trbsr &= ~(TRBSR_BSC_MASK << TRBSR_BSC_SHIFT);
492 trbsr &= ~TRBSR_STOP;
493 write_sysreg_s(trbsr, SYS_TRBSR_EL1);
496 static void set_trbe_limit_pointer_enabled(unsigned long addr)
498 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
500 WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_LIMIT_SHIFT)));
501 WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
503 trblimitr &= ~TRBLIMITR_NVM;
504 trblimitr &= ~(TRBLIMITR_FILL_MODE_MASK << TRBLIMITR_FILL_MODE_SHIFT);
505 trblimitr &= ~(TRBLIMITR_TRIG_MODE_MASK << TRBLIMITR_TRIG_MODE_SHIFT);
506 trblimitr &= ~(TRBLIMITR_LIMIT_MASK << TRBLIMITR_LIMIT_SHIFT);
509 * Fill trace buffer mode is used here while configuring the
510 * TRBE for trace capture. In this particular mode, the trace
511 * collection is stopped and a maintenance interrupt is raised
512 * when the current write pointer wraps. This pause in trace
513 * collection gives the software an opportunity to capture the
514 * trace data in the interrupt handler, before reconfiguring
517 trblimitr |= (TRBE_FILL_MODE_FILL & TRBLIMITR_FILL_MODE_MASK) << TRBLIMITR_FILL_MODE_SHIFT;
520 * Trigger mode is not used here while configuring the TRBE for
521 * the trace capture. Hence just keep this in the ignore mode.
523 trblimitr |= (TRBE_TRIG_MODE_IGNORE & TRBLIMITR_TRIG_MODE_MASK) <<
524 TRBLIMITR_TRIG_MODE_SHIFT;
525 trblimitr |= (addr & PAGE_MASK);
527 trblimitr |= TRBLIMITR_ENABLE;
528 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
530 /* Synchronize the TRBE enable event */
534 static void trbe_enable_hw(struct trbe_buf *buf)
536 WARN_ON(buf->trbe_hw_base < buf->trbe_base);
537 WARN_ON(buf->trbe_write < buf->trbe_hw_base);
538 WARN_ON(buf->trbe_write >= buf->trbe_limit);
542 set_trbe_base_pointer(buf->trbe_hw_base);
543 set_trbe_write_pointer(buf->trbe_write);
546 * Synchronize all the register updates
547 * till now before enabling the TRBE.
550 set_trbe_limit_pointer_enabled(buf->trbe_limit);
553 static enum trbe_fault_action trbe_get_fault_act(u64 trbsr)
555 int ec = get_trbe_ec(trbsr);
556 int bsc = get_trbe_bsc(trbsr);
558 WARN_ON(is_trbe_running(trbsr));
559 if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
560 return TRBE_FAULT_ACT_FATAL;
562 if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
563 return TRBE_FAULT_ACT_FATAL;
565 if (is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) {
566 if (get_trbe_write_pointer() == get_trbe_base_pointer())
567 return TRBE_FAULT_ACT_WRAP;
569 return TRBE_FAULT_ACT_SPURIOUS;
572 static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
573 struct trbe_buf *buf, bool wrap)
576 u64 start_off, end_off;
579 * If the TRBE has wrapped around the write pointer has
580 * wrapped and should be treated as limit.
583 write = get_trbe_limit_pointer();
585 write = get_trbe_write_pointer();
588 * TRBE may use a different base address than the base
589 * of the ring buffer. Thus use the beginning of the ring
590 * buffer to compute the offsets.
592 end_off = write - buf->trbe_base;
593 start_off = PERF_IDX2OFF(handle->head, buf);
595 if (WARN_ON_ONCE(end_off < start_off))
597 return (end_off - start_off);
600 static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
601 struct perf_event *event, void **pages,
602 int nr_pages, bool snapshot)
604 struct trbe_buf *buf;
605 struct page **pglist;
609 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
610 * just a single page, there would not be any room left while writing
611 * into a partially filled TRBE buffer after the page size alignment.
612 * Hence restrict the minimum buffer size as two pages.
617 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));
619 return ERR_PTR(-ENOMEM);
621 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
624 return ERR_PTR(-ENOMEM);
627 for (i = 0; i < nr_pages; i++)
628 pglist[i] = virt_to_page(pages[i]);
630 buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
631 if (!buf->trbe_base) {
634 return ERR_PTR(-ENOMEM);
636 buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
637 buf->trbe_write = buf->trbe_base;
638 buf->snapshot = snapshot;
639 buf->nr_pages = nr_pages;
645 static void arm_trbe_free_buffer(void *config)
647 struct trbe_buf *buf = config;
649 vunmap((void *)buf->trbe_base);
653 static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
654 struct perf_output_handle *handle,
657 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
658 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
659 struct trbe_buf *buf = config;
660 enum trbe_fault_action act;
661 unsigned long size, status;
665 WARN_ON(buf->cpudata != cpudata);
666 WARN_ON(cpudata->cpu != smp_processor_id());
667 WARN_ON(cpudata->drvdata != drvdata);
668 if (cpudata->mode != CS_MODE_PERF)
672 * We are about to disable the TRBE. And this could in turn
673 * fill up the buffer triggering, an IRQ. This could be consumed
674 * by the PE asynchronously, causing a race here against
675 * the IRQ handler in closing out the handle. So, let us
676 * make sure the IRQ can't trigger while we are collecting
677 * the buffer. We also make sure that a WRAP event is handled
680 local_irq_save(flags);
683 * If the TRBE was disabled due to lack of space in the AUX buffer or a
684 * spurious fault, the driver leaves it disabled, truncating the buffer.
685 * Since the etm_perf driver expects to close out the AUX buffer, the
686 * driver skips it. Thus, just pass in 0 size here to indicate that the
687 * buffer was truncated.
689 if (!is_trbe_enabled()) {
694 * perf handle structure needs to be shared with the TRBE IRQ handler for
695 * capturing trace data and restarting the handle. There is a probability
696 * of an undefined reference based crash when etm event is being stopped
697 * while a TRBE IRQ also getting processed. This happens due the release
698 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
699 * the TRBE here will ensure that no IRQ could be generated when the perf
700 * handle gets freed in etm_event_stop().
702 trbe_drain_and_disable_local();
704 /* Check if there is a pending interrupt and handle it here */
705 status = read_sysreg_s(SYS_TRBSR_EL1);
706 if (is_trbe_irq(status)) {
709 * Now that we are handling the IRQ here, clear the IRQ
710 * from the status, to let the irq handler know that it
716 act = trbe_get_fault_act(status);
718 * If this was not due to a WRAP event, we have some
719 * errors and as such buffer is empty.
721 if (act != TRBE_FAULT_ACT_WRAP) {
726 trbe_report_wrap_event(handle);
730 size = trbe_get_trace_size(handle, buf, wrap);
733 local_irq_restore(flags);
736 handle->head += size;
740 static int __arm_trbe_enable(struct trbe_buf *buf,
741 struct perf_output_handle *handle)
743 perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
744 buf->trbe_limit = compute_trbe_buffer_limit(handle);
745 buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
746 if (buf->trbe_limit == buf->trbe_base) {
747 trbe_stop_and_truncate_event(handle);
750 /* Set the base of the TRBE to the buffer base */
751 buf->trbe_hw_base = buf->trbe_base;
752 *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
757 static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data)
759 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
760 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
761 struct perf_output_handle *handle = data;
762 struct trbe_buf *buf = etm_perf_sink_config(handle);
764 WARN_ON(cpudata->cpu != smp_processor_id());
765 WARN_ON(cpudata->drvdata != drvdata);
766 if (mode != CS_MODE_PERF)
770 cpudata->mode = mode;
771 buf->cpudata = cpudata;
773 return __arm_trbe_enable(buf, handle);
776 static int arm_trbe_disable(struct coresight_device *csdev)
778 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
779 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
780 struct trbe_buf *buf = cpudata->buf;
782 WARN_ON(buf->cpudata != cpudata);
783 WARN_ON(cpudata->cpu != smp_processor_id());
784 WARN_ON(cpudata->drvdata != drvdata);
785 if (cpudata->mode != CS_MODE_PERF)
788 trbe_drain_and_disable_local();
791 cpudata->mode = CS_MODE_DISABLED;
795 static void trbe_handle_spurious(struct perf_output_handle *handle)
797 u64 limitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
800 * If the IRQ was spurious, simply re-enable the TRBE
801 * back without modifying the buffer parameters to
802 * retain the trace collected so far.
804 limitr |= TRBLIMITR_ENABLE;
805 write_sysreg_s(limitr, SYS_TRBLIMITR_EL1);
809 static int trbe_handle_overflow(struct perf_output_handle *handle)
811 struct perf_event *event = handle->event;
812 struct trbe_buf *buf = etm_perf_sink_config(handle);
814 struct etm_event_data *event_data;
816 size = trbe_get_trace_size(handle, buf, true);
818 handle->head += size;
820 trbe_report_wrap_event(handle);
821 perf_aux_output_end(handle, size);
822 event_data = perf_aux_output_begin(handle, event);
825 * We are unable to restart the trace collection,
826 * thus leave the TRBE disabled. The etm-perf driver
827 * is able to detect this with a disconnected handle
828 * (handle->event = NULL).
830 trbe_drain_and_disable_local();
831 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
835 return __arm_trbe_enable(buf, handle);
838 static bool is_perf_trbe(struct perf_output_handle *handle)
840 struct trbe_buf *buf = etm_perf_sink_config(handle);
841 struct trbe_cpudata *cpudata = buf->cpudata;
842 struct trbe_drvdata *drvdata = cpudata->drvdata;
843 int cpu = smp_processor_id();
845 WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
846 WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
848 if (cpudata->mode != CS_MODE_PERF)
851 if (cpudata->cpu != cpu)
854 if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))
860 static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
862 struct perf_output_handle **handle_ptr = dev;
863 struct perf_output_handle *handle = *handle_ptr;
864 enum trbe_fault_action act;
866 bool truncated = false;
869 /* Reads to TRBSR_EL1 is fine when TRBE is active */
870 status = read_sysreg_s(SYS_TRBSR_EL1);
872 * If the pending IRQ was handled by update_buffer callback
873 * we have nothing to do here.
875 if (!is_trbe_irq(status))
878 /* Prohibit the CPU from tracing before we disable the TRBE */
879 trfcr = cpu_prohibit_trace();
881 * Ensure the trace is visible to the CPUs and
882 * any external aborts have been resolved.
884 trbe_drain_and_disable_local();
888 if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
891 if (!is_perf_trbe(handle))
894 act = trbe_get_fault_act(status);
896 case TRBE_FAULT_ACT_WRAP:
897 truncated = !!trbe_handle_overflow(handle);
899 case TRBE_FAULT_ACT_SPURIOUS:
900 trbe_handle_spurious(handle);
902 case TRBE_FAULT_ACT_FATAL:
903 trbe_stop_and_truncate_event(handle);
909 * If the buffer was truncated, ensure perf callbacks
910 * have completed, which will disable the event.
912 * Otherwise, restore the trace filter controls to
923 static const struct coresight_ops_sink arm_trbe_sink_ops = {
924 .enable = arm_trbe_enable,
925 .disable = arm_trbe_disable,
926 .alloc_buffer = arm_trbe_alloc_buffer,
927 .free_buffer = arm_trbe_free_buffer,
928 .update_buffer = arm_trbe_update_buffer,
931 static const struct coresight_ops arm_trbe_cs_ops = {
932 .sink_ops = &arm_trbe_sink_ops,
935 static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
937 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
939 return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);
941 static DEVICE_ATTR_RO(align);
943 static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
945 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
947 return sprintf(buf, "%d\n", cpudata->trbe_flag);
949 static DEVICE_ATTR_RO(flag);
951 static struct attribute *arm_trbe_attrs[] = {
952 &dev_attr_align.attr,
957 static const struct attribute_group arm_trbe_group = {
958 .attrs = arm_trbe_attrs,
961 static const struct attribute_group *arm_trbe_groups[] = {
966 static void arm_trbe_enable_cpu(void *info)
968 struct trbe_drvdata *drvdata = info;
971 enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);
974 static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
976 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
977 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
978 struct coresight_desc desc = { 0 };
981 if (WARN_ON(trbe_csdev))
984 /* If the TRBE was not probed on the CPU, we shouldn't be here */
985 if (WARN_ON(!cpudata->drvdata))
988 dev = &cpudata->drvdata->pdev->dev;
989 desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
993 desc.type = CORESIGHT_DEV_TYPE_SINK;
994 desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
995 desc.ops = &arm_trbe_cs_ops;
996 desc.pdata = dev_get_platdata(dev);
997 desc.groups = arm_trbe_groups;
999 trbe_csdev = coresight_register(&desc);
1000 if (IS_ERR(trbe_csdev))
1003 dev_set_drvdata(&trbe_csdev->dev, cpudata);
1004 coresight_set_percpu_sink(cpu, trbe_csdev);
1007 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1011 * Must be called with preemption disabled, for trbe_check_errata().
1013 static void arm_trbe_probe_cpu(void *info)
1015 struct trbe_drvdata *drvdata = info;
1016 int cpu = smp_processor_id();
1017 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1020 if (WARN_ON(!cpudata))
1023 if (!is_trbe_available()) {
1024 pr_err("TRBE is not implemented on cpu %d\n", cpu);
1028 trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
1029 if (!is_trbe_programmable(trbidr)) {
1030 pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
1034 cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
1035 if (cpudata->trbe_hw_align > SZ_2K) {
1036 pr_err("Unsupported alignment on cpu %d\n", cpu);
1041 * Run the TRBE erratum checks, now that we know
1042 * this instance is about to be registered.
1044 trbe_check_errata(cpudata);
1046 cpudata->trbe_align = cpudata->trbe_hw_align;
1047 cpudata->trbe_flag = get_trbe_flag_update(trbidr);
1049 cpudata->drvdata = drvdata;
1052 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1055 static void arm_trbe_remove_coresight_cpu(void *info)
1057 int cpu = smp_processor_id();
1058 struct trbe_drvdata *drvdata = info;
1059 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1060 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1062 disable_percpu_irq(drvdata->irq);
1065 coresight_unregister(trbe_csdev);
1066 cpudata->drvdata = NULL;
1067 coresight_set_percpu_sink(cpu, NULL);
1071 static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
1075 drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
1076 if (!drvdata->cpudata)
1079 for_each_cpu(cpu, &drvdata->supported_cpus) {
1080 /* If we fail to probe the CPU, let us defer it to hotplug callbacks */
1081 if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
1083 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1084 arm_trbe_register_coresight_cpu(drvdata, cpu);
1085 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1086 smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);
1091 static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
1095 for_each_cpu(cpu, &drvdata->supported_cpus)
1096 smp_call_function_single(cpu, arm_trbe_remove_coresight_cpu, drvdata, 1);
1097 free_percpu(drvdata->cpudata);
1101 static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
1104 arm_trbe_probe_cpu(drvdata);
1108 static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
1110 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1112 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1115 * If this CPU was not probed for TRBE,
1116 * initialize it now.
1118 if (!coresight_get_percpu_sink(cpu)) {
1119 arm_trbe_probe_hotplugged_cpu(drvdata);
1120 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1121 arm_trbe_register_coresight_cpu(drvdata, cpu);
1122 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1123 arm_trbe_enable_cpu(drvdata);
1125 arm_trbe_enable_cpu(drvdata);
1131 static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1133 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1135 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1136 disable_percpu_irq(drvdata->irq);
1142 static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
1144 enum cpuhp_state trbe_online;
1147 trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
1148 arm_trbe_cpu_startup, arm_trbe_cpu_teardown);
1149 if (trbe_online < 0)
1152 ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);
1154 cpuhp_remove_multi_state(trbe_online);
1157 drvdata->trbe_online = trbe_online;
1161 static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
1163 cpuhp_remove_multi_state(drvdata->trbe_online);
1166 static int arm_trbe_probe_irq(struct platform_device *pdev,
1167 struct trbe_drvdata *drvdata)
1171 drvdata->irq = platform_get_irq(pdev, 0);
1172 if (drvdata->irq < 0) {
1173 pr_err("IRQ not found for the platform device\n");
1174 return drvdata->irq;
1177 if (!irq_is_percpu(drvdata->irq)) {
1178 pr_err("IRQ is not a PPI\n");
1182 if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))
1185 drvdata->handle = alloc_percpu(struct perf_output_handle *);
1186 if (!drvdata->handle)
1189 ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);
1191 free_percpu(drvdata->handle);
1197 static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
1199 free_percpu_irq(drvdata->irq, drvdata->handle);
1200 free_percpu(drvdata->handle);
1203 static int arm_trbe_device_probe(struct platform_device *pdev)
1205 struct coresight_platform_data *pdata;
1206 struct trbe_drvdata *drvdata;
1207 struct device *dev = &pdev->dev;
1210 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1214 pdata = coresight_get_platform_data(dev);
1216 return PTR_ERR(pdata);
1218 dev_set_drvdata(dev, drvdata);
1219 dev->platform_data = pdata;
1220 drvdata->pdev = pdev;
1221 ret = arm_trbe_probe_irq(pdev, drvdata);
1225 ret = arm_trbe_probe_coresight(drvdata);
1229 ret = arm_trbe_probe_cpuhp(drvdata);
1235 arm_trbe_remove_coresight(drvdata);
1237 arm_trbe_remove_irq(drvdata);
1241 static int arm_trbe_device_remove(struct platform_device *pdev)
1243 struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
1245 arm_trbe_remove_cpuhp(drvdata);
1246 arm_trbe_remove_coresight(drvdata);
1247 arm_trbe_remove_irq(drvdata);
1251 static const struct of_device_id arm_trbe_of_match[] = {
1252 { .compatible = "arm,trace-buffer-extension"},
1255 MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
1257 static struct platform_driver arm_trbe_driver = {
1260 .of_match_table = of_match_ptr(arm_trbe_of_match),
1261 .suppress_bind_attrs = true,
1263 .probe = arm_trbe_device_probe,
1264 .remove = arm_trbe_device_remove,
1267 static int __init arm_trbe_init(void)
1271 if (arm64_kernel_unmapped_at_el0()) {
1272 pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
1276 ret = platform_driver_register(&arm_trbe_driver);
1280 pr_err("Error registering %s platform driver\n", DRVNAME);
1284 static void __exit arm_trbe_exit(void)
1286 platform_driver_unregister(&arm_trbe_driver);
1288 module_init(arm_trbe_init);
1289 module_exit(arm_trbe_exit);
1291 MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
1292 MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
1293 MODULE_LICENSE("GPL v2");