1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7 #include <linux/atomic.h>
8 #include <linux/coresight.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/iommu.h>
11 #include <linux/idr.h>
12 #include <linux/mutex.h>
13 #include <linux/refcount.h>
14 #include <linux/slab.h>
15 #include <linux/types.h>
16 #include <linux/vmalloc.h>
17 #include "coresight-catu.h"
18 #include "coresight-etm-perf.h"
19 #include "coresight-priv.h"
20 #include "coresight-tmc.h"
30 * etr_perf_buffer - Perf buffer used for ETR
31 * @drvdata - The ETR drvdaga this buffer has been allocated for.
32 * @etr_buf - Actual buffer used by the ETR
33 * @pid - The PID this etr_perf_buffer belongs to.
34 * @snaphost - Perf session mode
35 * @head - handle->head at the beginning of the session.
36 * @nr_pages - Number of pages in the ring buffer.
37 * @pages - Array of Pages in the ring buffer.
39 struct etr_perf_buffer {
40 struct tmc_drvdata *drvdata;
41 struct etr_buf *etr_buf;
49 /* Convert the perf index to an offset within the ETR buffer */
50 #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
52 /* Lower limit for ETR hardware buffer */
53 #define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
56 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
57 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
58 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
59 * contain more than one SG buffer and tables.
61 * A table entry has the following format:
63 * ---Bit31------------Bit4-------Bit1-----Bit0--
64 * | Address[39:12] | SBZ | Entry Type |
65 * ----------------------------------------------
67 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
72 * b01 - Last entry in the tables, points to 4K page buffer.
73 * b10 - Normal entry, points to 4K page buffer.
74 * b11 - Link. The address points to the base of next table.
79 #define ETR_SG_PAGE_SHIFT 12
80 #define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
81 #define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
82 #define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
83 #define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
85 #define ETR_SG_ET_MASK 0x3
86 #define ETR_SG_ET_LAST 0x1
87 #define ETR_SG_ET_NORMAL 0x2
88 #define ETR_SG_ET_LINK 0x3
90 #define ETR_SG_ADDR_SHIFT 4
92 #define ETR_SG_ENTRY(addr, type) \
93 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
94 (type & ETR_SG_ET_MASK))
96 #define ETR_SG_ADDR(entry) \
97 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
98 #define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
101 * struct etr_sg_table : ETR SG Table
102 * @sg_table: Generic SG Table holding the data/table pages.
103 * @hwaddr: hwaddress used by the TMC, which is the base
104 * address of the table.
106 struct etr_sg_table {
107 struct tmc_sg_table *sg_table;
112 * tmc_etr_sg_table_entries: Total number of table entries required to map
113 * @nr_pages system pages.
115 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
116 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
117 * with the last entry pointing to another page of table entries.
118 * If we spill over to a new page for mapping 1 entry, we could as
119 * well replace the link entry of the previous page with the last entry.
121 static inline unsigned long __attribute_const__
122 tmc_etr_sg_table_entries(int nr_pages)
124 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
125 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
127 * If we spill over to a new page for 1 entry, we could as well
128 * make it the LAST entry in the previous page, skipping the Link
131 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
133 return nr_sgpages + nr_sglinks;
137 * tmc_pages_get_offset: Go through all the pages in the tmc_pages
138 * and map the device address @addr to an offset within the virtual
142 tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
145 dma_addr_t page_start;
147 for (i = 0; i < tmc_pages->nr_pages; i++) {
148 page_start = tmc_pages->daddrs[i];
149 if (addr >= page_start && addr < (page_start + PAGE_SIZE))
150 return i * PAGE_SIZE + (addr - page_start);
157 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
158 * If the pages were not allocated in tmc_pages_alloc(), we would
159 * simply drop the refcount.
161 static void tmc_pages_free(struct tmc_pages *tmc_pages,
162 struct device *dev, enum dma_data_direction dir)
165 struct device *real_dev = dev->parent;
167 for (i = 0; i < tmc_pages->nr_pages; i++) {
168 if (tmc_pages->daddrs && tmc_pages->daddrs[i])
169 dma_unmap_page(real_dev, tmc_pages->daddrs[i],
171 if (tmc_pages->pages && tmc_pages->pages[i])
172 __free_page(tmc_pages->pages[i]);
175 kfree(tmc_pages->pages);
176 kfree(tmc_pages->daddrs);
177 tmc_pages->pages = NULL;
178 tmc_pages->daddrs = NULL;
179 tmc_pages->nr_pages = 0;
183 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
184 * If @pages is not NULL, the list of page virtual addresses are
185 * used as the data pages. The pages are then dma_map'ed for @dev
186 * with dma_direction @dir.
188 * Returns 0 upon success, else the error number.
190 static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
191 struct device *dev, int node,
192 enum dma_data_direction dir, void **pages)
197 struct device *real_dev = dev->parent;
199 nr_pages = tmc_pages->nr_pages;
200 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
202 if (!tmc_pages->daddrs)
204 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
206 if (!tmc_pages->pages) {
207 kfree(tmc_pages->daddrs);
208 tmc_pages->daddrs = NULL;
212 for (i = 0; i < nr_pages; i++) {
213 if (pages && pages[i]) {
214 page = virt_to_page(pages[i]);
215 /* Hold a refcount on the page */
218 page = alloc_pages_node(node,
219 GFP_KERNEL | __GFP_ZERO, 0);
221 paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
222 if (dma_mapping_error(real_dev, paddr))
224 tmc_pages->daddrs[i] = paddr;
225 tmc_pages->pages[i] = page;
229 tmc_pages_free(tmc_pages, dev, dir);
234 tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
236 return tmc_pages_get_offset(&sg_table->data_pages, addr);
239 static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
241 if (sg_table->table_vaddr)
242 vunmap(sg_table->table_vaddr);
243 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
246 static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
248 if (sg_table->data_vaddr)
249 vunmap(sg_table->data_vaddr);
250 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
253 void tmc_free_sg_table(struct tmc_sg_table *sg_table)
255 tmc_free_table_pages(sg_table);
256 tmc_free_data_pages(sg_table);
260 * Alloc pages for the table. Since this will be used by the device,
261 * allocate the pages closer to the device (i.e, dev_to_node(dev)
262 * rather than the CPU node).
264 static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
267 struct tmc_pages *table_pages = &sg_table->table_pages;
269 rc = tmc_pages_alloc(table_pages, sg_table->dev,
270 dev_to_node(sg_table->dev),
271 DMA_TO_DEVICE, NULL);
274 sg_table->table_vaddr = vmap(table_pages->pages,
275 table_pages->nr_pages,
278 if (!sg_table->table_vaddr)
281 sg_table->table_daddr = table_pages->daddrs[0];
285 static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
289 /* Allocate data pages on the node requested by the caller */
290 rc = tmc_pages_alloc(&sg_table->data_pages,
291 sg_table->dev, sg_table->node,
292 DMA_FROM_DEVICE, pages);
294 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
295 sg_table->data_pages.nr_pages,
298 if (!sg_table->data_vaddr)
305 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
306 * and data buffers. TMC writes to the data buffers and reads from the SG
309 * @dev - Coresight device to which page should be DMA mapped.
310 * @node - Numa node for mem allocations
311 * @nr_tpages - Number of pages for the table entries.
312 * @nr_dpages - Number of pages for Data buffer.
313 * @pages - Optional list of virtual address of pages.
315 struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
322 struct tmc_sg_table *sg_table;
324 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
326 return ERR_PTR(-ENOMEM);
327 sg_table->data_pages.nr_pages = nr_dpages;
328 sg_table->table_pages.nr_pages = nr_tpages;
329 sg_table->node = node;
332 rc = tmc_alloc_data_pages(sg_table, pages);
334 rc = tmc_alloc_table_pages(sg_table);
336 tmc_free_sg_table(sg_table);
345 * tmc_sg_table_sync_data_range: Sync the data buffer written
346 * by the device from @offset upto a @size bytes.
348 void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
349 u64 offset, u64 size)
352 int npages = DIV_ROUND_UP(size, PAGE_SIZE);
353 struct device *real_dev = table->dev->parent;
354 struct tmc_pages *data = &table->data_pages;
356 start = offset >> PAGE_SHIFT;
357 for (i = start; i < (start + npages); i++) {
358 index = i % data->nr_pages;
359 dma_sync_single_for_cpu(real_dev, data->daddrs[index],
360 PAGE_SIZE, DMA_FROM_DEVICE);
364 /* tmc_sg_sync_table: Sync the page table */
365 void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
368 struct device *real_dev = sg_table->dev->parent;
369 struct tmc_pages *table_pages = &sg_table->table_pages;
371 for (i = 0; i < table_pages->nr_pages; i++)
372 dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
373 PAGE_SIZE, DMA_TO_DEVICE);
377 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
378 * in the SG buffer. The @bufpp is updated to point to the buffer.
380 * the length of linear data available at @offset.
382 * <= 0 if no data is available.
384 ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
385 u64 offset, size_t len, char **bufpp)
388 int pg_idx = offset >> PAGE_SHIFT;
389 int pg_offset = offset & (PAGE_SIZE - 1);
390 struct tmc_pages *data_pages = &sg_table->data_pages;
392 size = tmc_sg_table_buf_size(sg_table);
396 /* Make sure we don't go beyond the end */
397 len = (len < (size - offset)) ? len : size - offset;
398 /* Respect the page boundaries */
399 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
401 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
406 /* Map a dma address to virtual address */
408 tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
409 dma_addr_t addr, bool table)
413 struct tmc_pages *tmc_pages;
416 tmc_pages = &sg_table->table_pages;
417 base = (unsigned long)sg_table->table_vaddr;
419 tmc_pages = &sg_table->data_pages;
420 base = (unsigned long)sg_table->data_vaddr;
423 offset = tmc_pages_get_offset(tmc_pages, addr);
426 return base + offset;
429 /* Dump the given sg_table */
430 static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
435 struct tmc_sg_table *sg_table = etr_table->sg_table;
437 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
438 etr_table->hwaddr, true);
440 addr = ETR_SG_ADDR(*ptr);
441 switch (ETR_SG_ET(*ptr)) {
442 case ETR_SG_ET_NORMAL:
443 dev_dbg(sg_table->dev,
444 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
448 dev_dbg(sg_table->dev,
449 "%05d: *** %p\t:{L} 0x%llx ***\n",
451 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
455 dev_dbg(sg_table->dev,
456 "%05d: ### %p\t:[L] 0x%llx ###\n",
460 dev_dbg(sg_table->dev,
461 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
467 dev_dbg(sg_table->dev, "******* End of Table *****\n");
470 static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
474 * Populate the SG Table page table entries from table/data
475 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
476 * So does a Table page. So we keep track of indices of the tables
477 * in each system page and move the pointers accordingly.
479 #define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
480 static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
483 int i, type, nr_entries;
484 int tpidx = 0; /* index to the current system table_page */
485 int sgtidx = 0; /* index to the sg_table within the current syspage */
486 int sgtentry = 0; /* the entry within the sg_table */
487 int dpidx = 0; /* index to the current system data_page */
488 int spidx = 0; /* index to the SG page within the current data page */
489 sgte_t *ptr; /* pointer to the table entry to fill */
490 struct tmc_sg_table *sg_table = etr_table->sg_table;
491 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
492 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
494 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
496 * Use the contiguous virtual address of the table to update entries.
498 ptr = sg_table->table_vaddr;
500 * Fill all the entries, except the last entry to avoid special
501 * checks within the loop.
503 for (i = 0; i < nr_entries - 1; i++) {
504 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
506 * Last entry in a sg_table page is a link address to
507 * the next table page. If this sg_table is the last
508 * one in the system page, it links to the first
509 * sg_table in the next system page. Otherwise, it
510 * links to the next sg_table page within the system
513 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
514 paddr = table_daddrs[tpidx + 1];
516 paddr = table_daddrs[tpidx] +
517 (ETR_SG_PAGE_SIZE * (sgtidx + 1));
519 type = ETR_SG_ET_LINK;
522 * Update the indices to the data_pages to point to the
523 * next sg_page in the data buffer.
525 type = ETR_SG_ET_NORMAL;
526 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
527 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
530 *ptr++ = ETR_SG_ENTRY(paddr, type);
532 * Move to the next table pointer, moving the table page index
535 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
536 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
541 /* Set up the last entry, which is always a data pointer */
542 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
543 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
547 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
548 * populate the table.
550 * @dev - Device pointer for the TMC
551 * @node - NUMA node where the memory should be allocated
552 * @size - Total size of the data buffer
553 * @pages - Optional list of page virtual address
555 static struct etr_sg_table *
556 tmc_init_etr_sg_table(struct device *dev, int node,
557 unsigned long size, void **pages)
559 int nr_entries, nr_tpages;
560 int nr_dpages = size >> PAGE_SHIFT;
561 struct tmc_sg_table *sg_table;
562 struct etr_sg_table *etr_table;
564 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
566 return ERR_PTR(-ENOMEM);
567 nr_entries = tmc_etr_sg_table_entries(nr_dpages);
568 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
570 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
571 if (IS_ERR(sg_table)) {
573 return ERR_CAST(sg_table);
576 etr_table->sg_table = sg_table;
577 /* TMC should use table base address for DBA */
578 etr_table->hwaddr = sg_table->table_daddr;
579 tmc_etr_sg_table_populate(etr_table);
580 /* Sync the table pages for the HW */
581 tmc_sg_table_sync_table(sg_table);
582 tmc_etr_sg_table_dump(etr_table);
588 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
590 static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
591 struct etr_buf *etr_buf, int node,
594 struct etr_flat_buf *flat_buf;
595 struct device *real_dev = drvdata->csdev->dev.parent;
597 /* We cannot reuse existing pages for flat buf */
601 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
605 flat_buf->vaddr = dma_alloc_coherent(real_dev, etr_buf->size,
606 &flat_buf->daddr, GFP_KERNEL);
607 if (!flat_buf->vaddr) {
612 flat_buf->size = etr_buf->size;
613 flat_buf->dev = &drvdata->csdev->dev;
614 etr_buf->hwaddr = flat_buf->daddr;
615 etr_buf->mode = ETR_MODE_FLAT;
616 etr_buf->private = flat_buf;
620 static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
622 struct etr_flat_buf *flat_buf = etr_buf->private;
624 if (flat_buf && flat_buf->daddr) {
625 struct device *real_dev = flat_buf->dev->parent;
627 dma_free_coherent(real_dev, flat_buf->size,
628 flat_buf->vaddr, flat_buf->daddr);
633 static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
636 * Adjust the buffer to point to the beginning of the trace data
637 * and update the available trace data.
639 etr_buf->offset = rrp - etr_buf->hwaddr;
641 etr_buf->len = etr_buf->size;
643 etr_buf->len = rwp - rrp;
646 static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
647 u64 offset, size_t len, char **bufpp)
649 struct etr_flat_buf *flat_buf = etr_buf->private;
651 *bufpp = (char *)flat_buf->vaddr + offset;
653 * tmc_etr_buf_get_data already adjusts the length to handle
654 * buffer wrapping around.
659 static const struct etr_buf_operations etr_flat_buf_ops = {
660 .alloc = tmc_etr_alloc_flat_buf,
661 .free = tmc_etr_free_flat_buf,
662 .sync = tmc_etr_sync_flat_buf,
663 .get_data = tmc_etr_get_data_flat_buf,
667 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
670 static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
671 struct etr_buf *etr_buf, int node,
674 struct etr_sg_table *etr_table;
675 struct device *dev = &drvdata->csdev->dev;
677 etr_table = tmc_init_etr_sg_table(dev, node,
678 etr_buf->size, pages);
679 if (IS_ERR(etr_table))
681 etr_buf->hwaddr = etr_table->hwaddr;
682 etr_buf->mode = ETR_MODE_ETR_SG;
683 etr_buf->private = etr_table;
687 static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
689 struct etr_sg_table *etr_table = etr_buf->private;
692 tmc_free_sg_table(etr_table->sg_table);
697 static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
698 size_t len, char **bufpp)
700 struct etr_sg_table *etr_table = etr_buf->private;
702 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
705 static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
707 long r_offset, w_offset;
708 struct etr_sg_table *etr_table = etr_buf->private;
709 struct tmc_sg_table *table = etr_table->sg_table;
711 /* Convert hw address to offset in the buffer */
712 r_offset = tmc_sg_get_data_page_offset(table, rrp);
715 "Unable to map RRP %llx to offset\n", rrp);
720 w_offset = tmc_sg_get_data_page_offset(table, rwp);
723 "Unable to map RWP %llx to offset\n", rwp);
728 etr_buf->offset = r_offset;
730 etr_buf->len = etr_buf->size;
732 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
734 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
737 static const struct etr_buf_operations etr_sg_buf_ops = {
738 .alloc = tmc_etr_alloc_sg_buf,
739 .free = tmc_etr_free_sg_buf,
740 .sync = tmc_etr_sync_sg_buf,
741 .get_data = tmc_etr_get_data_sg_buf,
745 * TMC ETR could be connected to a CATU device, which can provide address
746 * translation service. This is represented by the Output port of the TMC
747 * (ETR) connected to the input port of the CATU.
749 * Returns : coresight_device ptr for the CATU device if a CATU is found.
752 struct coresight_device *
753 tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
756 struct coresight_device *tmp, *etr = drvdata->csdev;
758 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
761 for (i = 0; i < etr->pdata->nr_outport; i++) {
762 tmp = etr->pdata->conns[i].child_dev;
763 if (tmp && coresight_is_catu_device(tmp))
770 static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata,
771 struct etr_buf *etr_buf)
773 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
775 if (catu && helper_ops(catu)->enable)
776 return helper_ops(catu)->enable(catu, etr_buf);
780 static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata)
782 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
784 if (catu && helper_ops(catu)->disable)
785 helper_ops(catu)->disable(catu, drvdata->etr_buf);
788 static const struct etr_buf_operations *etr_buf_ops[] = {
789 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
790 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
791 [ETR_MODE_CATU] = IS_ENABLED(CONFIG_CORESIGHT_CATU)
792 ? &etr_catu_buf_ops : NULL,
795 static inline int tmc_etr_mode_alloc_buf(int mode,
796 struct tmc_drvdata *drvdata,
797 struct etr_buf *etr_buf, int node,
804 case ETR_MODE_ETR_SG:
806 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
807 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
810 etr_buf->ops = etr_buf_ops[mode];
818 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
819 * @drvdata : ETR device details.
820 * @size : size of the requested buffer.
821 * @flags : Required properties for the buffer.
822 * @node : Node for memory allocations.
823 * @pages : An optional list of pages.
825 static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
826 ssize_t size, int flags,
827 int node, void **pages)
830 bool has_etr_sg, has_iommu;
831 bool has_sg, has_catu;
832 struct etr_buf *etr_buf;
833 struct device *dev = &drvdata->csdev->dev;
835 has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
836 has_iommu = iommu_get_domain_for_dev(dev->parent);
837 has_catu = !!tmc_etr_get_catu_device(drvdata);
839 has_sg = has_catu || has_etr_sg;
841 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
843 return ERR_PTR(-ENOMEM);
845 etr_buf->size = size;
848 * If we have to use an existing list of pages, we cannot reliably
849 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
850 * we use the contiguous DMA memory if at least one of the following
851 * conditions is true:
852 * a) The ETR cannot use Scatter-Gather.
853 * b) we have a backing IOMMU
854 * c) The requested memory size is smaller (< 1M).
856 * Fallback to available mechanisms.
860 (!has_sg || has_iommu || size < SZ_1M))
861 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
862 etr_buf, node, pages);
863 if (rc && has_etr_sg)
864 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
865 etr_buf, node, pages);
867 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
868 etr_buf, node, pages);
874 refcount_set(&etr_buf->refcount, 1);
875 dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
876 (unsigned long)size >> 10, etr_buf->mode);
880 static void tmc_free_etr_buf(struct etr_buf *etr_buf)
882 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
883 etr_buf->ops->free(etr_buf);
888 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
889 * with a maximum of @len bytes.
890 * Returns: The size of the linear data available @pos, with *bufpp
891 * updated to point to the buffer.
893 static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
894 u64 offset, size_t len, char **bufpp)
896 /* Adjust the length to limit this transaction to end of buffer */
897 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
899 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
903 tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
908 len = tmc_etr_buf_get_data(etr_buf, offset,
909 CORESIGHT_BARRIER_PKT_SIZE, &bufp);
910 if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE))
912 coresight_insert_barrier_packet(bufp);
913 return offset + CORESIGHT_BARRIER_PKT_SIZE;
917 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
918 * Makes sure the trace data is synced to the memory for consumption.
919 * @etr_buf->offset will hold the offset to the beginning of the trace data
920 * within the buffer, with @etr_buf->len bytes to consume.
922 static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
924 struct etr_buf *etr_buf = drvdata->etr_buf;
928 rrp = tmc_read_rrp(drvdata);
929 rwp = tmc_read_rwp(drvdata);
930 status = readl_relaxed(drvdata->base + TMC_STS);
933 * If there were memory errors in the session, truncate the
936 if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
937 dev_dbg(&drvdata->csdev->dev,
938 "tmc memory error detected, truncating buffer\n");
944 etr_buf->full = status & TMC_STS_FULL;
946 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
948 etr_buf->ops->sync(etr_buf, rrp, rwp);
950 /* Insert barrier packets at the beginning, if there was an overflow */
952 tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset);
955 static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
958 struct etr_buf *etr_buf = drvdata->etr_buf;
960 CS_UNLOCK(drvdata->base);
962 /* Wait for TMCSReady bit to be set */
963 tmc_wait_for_tmcready(drvdata);
965 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
966 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
968 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
969 axictl &= ~TMC_AXICTL_CLEAR_MASK;
970 axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16);
971 axictl |= TMC_AXICTL_AXCACHE_OS;
973 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
974 axictl &= ~TMC_AXICTL_ARCACHE_MASK;
975 axictl |= TMC_AXICTL_ARCACHE_OS;
978 if (etr_buf->mode == ETR_MODE_ETR_SG)
979 axictl |= TMC_AXICTL_SCT_GAT_MODE;
981 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
982 tmc_write_dba(drvdata, etr_buf->hwaddr);
984 * If the TMC pointers must be programmed before the session,
985 * we have to set it properly (i.e, RRP/RWP to base address and
986 * STS to "not full").
988 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
989 tmc_write_rrp(drvdata, etr_buf->hwaddr);
990 tmc_write_rwp(drvdata, etr_buf->hwaddr);
991 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
992 writel_relaxed(sts, drvdata->base + TMC_STS);
995 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
996 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
997 TMC_FFCR_TRIGON_TRIGIN,
998 drvdata->base + TMC_FFCR);
999 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1000 tmc_enable_hw(drvdata);
1002 CS_LOCK(drvdata->base);
1005 static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1006 struct etr_buf *etr_buf)
1010 /* Callers should provide an appropriate buffer for use */
1011 if (WARN_ON(!etr_buf))
1014 if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1015 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1018 if (WARN_ON(drvdata->etr_buf))
1022 * If this ETR is connected to a CATU, enable it before we turn
1025 rc = tmc_etr_enable_catu(drvdata, etr_buf);
1028 rc = coresight_claim_device(drvdata->base);
1030 drvdata->etr_buf = etr_buf;
1031 __tmc_etr_enable_hw(drvdata);
1038 * Return the available trace data in the buffer (starts at etr_buf->offset,
1039 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1040 * also updating the @bufpp on where to find it. Since the trace data
1041 * starts at anywhere in the buffer, depending on the RRP, we adjust the
1042 * @len returned to handle buffer wrapping around.
1044 * We are protected here by drvdata->reading != 0, which ensures the
1045 * sysfs_buf stays alive.
1047 ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1048 loff_t pos, size_t len, char **bufpp)
1051 ssize_t actual = len;
1052 struct etr_buf *etr_buf = drvdata->sysfs_buf;
1054 if (pos + actual > etr_buf->len)
1055 actual = etr_buf->len - pos;
1059 /* Compute the offset from which we read the data */
1060 offset = etr_buf->offset + pos;
1061 if (offset >= etr_buf->size)
1062 offset -= etr_buf->size;
1063 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1066 static struct etr_buf *
1067 tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1069 return tmc_alloc_etr_buf(drvdata, drvdata->size,
1070 0, cpu_to_node(0), NULL);
1074 tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1077 tmc_free_etr_buf(buf);
1080 static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1082 struct etr_buf *etr_buf = drvdata->etr_buf;
1084 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1085 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1086 drvdata->sysfs_buf = NULL;
1088 tmc_sync_etr_buf(drvdata);
1092 static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1094 CS_UNLOCK(drvdata->base);
1096 tmc_flush_and_stop(drvdata);
1098 * When operating in sysFS mode the content of the buffer needs to be
1099 * read before the TMC is disabled.
1101 if (drvdata->mode == CS_MODE_SYSFS)
1102 tmc_etr_sync_sysfs_buf(drvdata);
1104 tmc_disable_hw(drvdata);
1106 CS_LOCK(drvdata->base);
1110 static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1112 __tmc_etr_disable_hw(drvdata);
1113 /* Disable CATU device if this ETR is connected to one */
1114 tmc_etr_disable_catu(drvdata);
1115 coresight_disclaim_device(drvdata->base);
1116 /* Reset the ETR buf used by hardware */
1117 drvdata->etr_buf = NULL;
1120 static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1123 unsigned long flags;
1124 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1125 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1128 * If we are enabling the ETR from disabled state, we need to make
1129 * sure we have a buffer with the right size. The etr_buf is not reset
1130 * immediately after we stop the tracing in SYSFS mode as we wait for
1131 * the user to collect the data. We may be able to reuse the existing
1132 * buffer, provided the size matches. Any allocation has to be done
1133 * with the lock released.
1135 spin_lock_irqsave(&drvdata->spinlock, flags);
1136 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1137 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1138 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1140 /* Allocate memory with the locks released */
1141 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1142 if (IS_ERR(new_buf))
1143 return PTR_ERR(new_buf);
1145 /* Let's try again */
1146 spin_lock_irqsave(&drvdata->spinlock, flags);
1149 if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1155 * In sysFS mode we can have multiple writers per sink. Since this
1156 * sink is already enabled no memory is needed and the HW need not be
1157 * touched, even if the buffer size has changed.
1159 if (drvdata->mode == CS_MODE_SYSFS) {
1160 atomic_inc(csdev->refcnt);
1165 * If we don't have a buffer or it doesn't match the requested size,
1166 * use the buffer allocated above. Otherwise reuse the existing buffer.
1168 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1169 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1170 free_buf = sysfs_buf;
1171 drvdata->sysfs_buf = new_buf;
1174 ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf);
1176 drvdata->mode = CS_MODE_SYSFS;
1177 atomic_inc(csdev->refcnt);
1180 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1182 /* Free memory outside the spinlock if need be */
1184 tmc_etr_free_sysfs_buf(free_buf);
1187 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1193 * alloc_etr_buf: Allocate ETR buffer for use by perf.
1194 * The size of the hardware buffer is dependent on the size configured
1195 * via sysfs and the perf ring buffer size. We prefer to allocate the
1196 * largest possible size, scaling down the size by half until it
1197 * reaches a minimum limit (1M), beyond which we give up.
1199 static struct etr_buf *
1200 alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1201 int nr_pages, void **pages, bool snapshot)
1204 struct etr_buf *etr_buf;
1207 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1209 * Try to match the perf ring buffer size if it is larger
1210 * than the size requested via sysfs.
1212 if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1213 etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT),
1215 if (!IS_ERR(etr_buf))
1220 * Else switch to configured size for this ETR
1221 * and scale down until we hit the minimum limit.
1223 size = drvdata->size;
1225 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1226 if (!IS_ERR(etr_buf))
1229 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1231 return ERR_PTR(-ENOMEM);
1237 static struct etr_buf *
1238 get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1239 struct perf_event *event, int nr_pages,
1240 void **pages, bool snapshot)
1243 pid_t pid = task_pid_nr(event->owner);
1244 struct etr_buf *etr_buf;
1248 * An etr_perf_buffer is associated with an event and holds a reference
1249 * to the AUX ring buffer that was created for that event. In CPU-wide
1250 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1251 * buffer, share a sink. As such an etr_perf_buffer is created for each
1252 * event but a single etr_buf associated with the ETR is shared between
1253 * them. The last event in a trace session will copy the content of the
1254 * etr_buf to its AUX ring buffer. Ring buffer associated to other
1255 * events are simply not used an freed as events are destoyed. We still
1256 * need to allocate a ring buffer for each event since we don't know
1257 * which event will be last.
1261 * The first thing to do here is check if an etr_buf has already been
1262 * allocated for this session. If so it is shared with this event,
1263 * otherwise it is created.
1265 mutex_lock(&drvdata->idr_mutex);
1266 etr_buf = idr_find(&drvdata->idr, pid);
1268 refcount_inc(&etr_buf->refcount);
1269 mutex_unlock(&drvdata->idr_mutex);
1273 /* If we made it here no buffer has been allocated, do so now. */
1274 mutex_unlock(&drvdata->idr_mutex);
1276 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1277 if (IS_ERR(etr_buf))
1280 /* Now that we have a buffer, add it to the IDR. */
1281 mutex_lock(&drvdata->idr_mutex);
1282 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1283 mutex_unlock(&drvdata->idr_mutex);
1285 /* Another event with this session ID has allocated this buffer. */
1286 if (ret == -ENOSPC) {
1287 tmc_free_etr_buf(etr_buf);
1291 /* The IDR can't allocate room for a new session, abandon ship. */
1292 if (ret == -ENOMEM) {
1293 tmc_free_etr_buf(etr_buf);
1294 return ERR_PTR(ret);
1301 static struct etr_buf *
1302 get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1303 struct perf_event *event, int nr_pages,
1304 void **pages, bool snapshot)
1307 * In per-thread mode the etr_buf isn't shared, so just go ahead
1308 * with memory allocation.
1310 return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1313 static struct etr_buf *
1314 get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1315 int nr_pages, void **pages, bool snapshot)
1317 if (event->cpu == -1)
1318 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1321 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1325 static struct etr_perf_buffer *
1326 tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1327 int nr_pages, void **pages, bool snapshot)
1330 struct etr_buf *etr_buf;
1331 struct etr_perf_buffer *etr_perf;
1333 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1335 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1337 return ERR_PTR(-ENOMEM);
1339 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1340 if (!IS_ERR(etr_buf))
1344 return ERR_PTR(-ENOMEM);
1348 * Keep a reference to the ETR this buffer has been allocated for
1349 * in order to have access to the IDR in tmc_free_etr_buffer().
1351 etr_perf->drvdata = drvdata;
1352 etr_perf->etr_buf = etr_buf;
1358 static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1359 struct perf_event *event, void **pages,
1360 int nr_pages, bool snapshot)
1362 struct etr_perf_buffer *etr_perf;
1363 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1365 etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1366 nr_pages, pages, snapshot);
1367 if (IS_ERR(etr_perf)) {
1368 dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1372 etr_perf->pid = task_pid_nr(event->owner);
1373 etr_perf->snapshot = snapshot;
1374 etr_perf->nr_pages = nr_pages;
1375 etr_perf->pages = pages;
1380 static void tmc_free_etr_buffer(void *config)
1382 struct etr_perf_buffer *etr_perf = config;
1383 struct tmc_drvdata *drvdata = etr_perf->drvdata;
1384 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1387 goto free_etr_perf_buffer;
1389 mutex_lock(&drvdata->idr_mutex);
1390 /* If we are not the last one to use the buffer, don't touch it. */
1391 if (!refcount_dec_and_test(&etr_buf->refcount)) {
1392 mutex_unlock(&drvdata->idr_mutex);
1393 goto free_etr_perf_buffer;
1396 /* We are the last one, remove from the IDR and free the buffer. */
1397 buf = idr_remove(&drvdata->idr, etr_perf->pid);
1398 mutex_unlock(&drvdata->idr_mutex);
1401 * Something went very wrong if the buffer associated with this ID
1402 * is not the same in the IDR. Leak to avoid use after free.
1404 if (buf && WARN_ON(buf != etr_buf))
1405 goto free_etr_perf_buffer;
1407 tmc_free_etr_buf(etr_perf->etr_buf);
1409 free_etr_perf_buffer:
1414 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1415 * buffer to the perf ring buffer.
1417 static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1418 unsigned long to_copy)
1421 long pg_idx, pg_offset, src_offset;
1422 unsigned long head = etr_perf->head;
1423 char **dst_pages, *src_buf;
1424 struct etr_buf *etr_buf = etr_perf->etr_buf;
1426 head = etr_perf->head;
1427 pg_idx = head >> PAGE_SHIFT;
1428 pg_offset = head & (PAGE_SIZE - 1);
1429 dst_pages = (char **)etr_perf->pages;
1430 src_offset = etr_buf->offset + etr_buf->len - to_copy;
1432 while (to_copy > 0) {
1434 * In one iteration, we can copy minimum of :
1435 * 1) what is available in the source buffer,
1436 * 2) what is available in the source buffer, before it
1438 * 3) what is available in the destination page.
1441 if (src_offset >= etr_buf->size)
1442 src_offset -= etr_buf->size;
1443 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1445 if (WARN_ON_ONCE(bytes <= 0))
1447 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1449 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1453 /* Move destination pointers */
1455 if (pg_offset == PAGE_SIZE) {
1457 if (++pg_idx == etr_perf->nr_pages)
1461 /* Move source pointers */
1462 src_offset += bytes;
1467 * tmc_update_etr_buffer : Update the perf ring buffer with the
1468 * available trace data. We use software double buffering at the moment.
1470 * TODO: Add support for reusing the perf ring buffer.
1472 static unsigned long
1473 tmc_update_etr_buffer(struct coresight_device *csdev,
1474 struct perf_output_handle *handle,
1478 unsigned long flags, size = 0;
1479 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1480 struct etr_perf_buffer *etr_perf = config;
1481 struct etr_buf *etr_buf = etr_perf->etr_buf;
1483 spin_lock_irqsave(&drvdata->spinlock, flags);
1485 /* Don't do anything if another tracer is using this sink */
1486 if (atomic_read(csdev->refcnt) != 1) {
1487 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1491 if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1493 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1497 CS_UNLOCK(drvdata->base);
1499 tmc_flush_and_stop(drvdata);
1500 tmc_sync_etr_buf(drvdata);
1502 CS_LOCK(drvdata->base);
1503 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1505 size = etr_buf->len;
1506 if (!etr_perf->snapshot && size > handle->size) {
1507 size = handle->size;
1510 tmc_etr_sync_perf_buffer(etr_perf, size);
1513 * In snapshot mode we simply increment the head by the number of byte
1514 * that were written. User space function cs_etm_find_snapshot() will
1515 * figure out how many bytes to get from the AUX buffer based on the
1516 * position of the head.
1518 if (etr_perf->snapshot)
1519 handle->head += size;
1521 lost |= etr_buf->full;
1524 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1525 * captured buffer is expected to be truncated and 2) a full buffer
1526 * prevents the event from being re-enabled by the perf core,
1527 * resulting in stale data being send to user space.
1529 if (!etr_perf->snapshot && lost)
1530 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1534 static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1538 unsigned long flags;
1539 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1540 struct perf_output_handle *handle = data;
1541 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1543 spin_lock_irqsave(&drvdata->spinlock, flags);
1544 /* Don't use this sink if it is already claimed by sysFS */
1545 if (drvdata->mode == CS_MODE_SYSFS) {
1550 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1555 /* Get a handle on the pid of the process to monitor */
1556 pid = etr_perf->pid;
1558 /* Do not proceed if this device is associated with another session */
1559 if (drvdata->pid != -1 && drvdata->pid != pid) {
1564 etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf);
1567 * No HW configuration is needed if the sink is already in
1568 * use for this session.
1570 if (drvdata->pid == pid) {
1571 atomic_inc(csdev->refcnt);
1575 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1577 /* Associate with monitored process. */
1579 drvdata->mode = CS_MODE_PERF;
1580 drvdata->perf_buf = etr_perf->etr_buf;
1581 atomic_inc(csdev->refcnt);
1585 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1589 static int tmc_enable_etr_sink(struct coresight_device *csdev,
1590 u32 mode, void *data)
1594 return tmc_enable_etr_sink_sysfs(csdev);
1596 return tmc_enable_etr_sink_perf(csdev, data);
1599 /* We shouldn't be here */
1603 static int tmc_disable_etr_sink(struct coresight_device *csdev)
1605 unsigned long flags;
1606 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1608 spin_lock_irqsave(&drvdata->spinlock, flags);
1610 if (drvdata->reading) {
1611 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1615 if (atomic_dec_return(csdev->refcnt)) {
1616 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1620 /* Complain if we (somehow) got out of sync */
1621 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1622 tmc_etr_disable_hw(drvdata);
1623 /* Dissociate from monitored process. */
1625 drvdata->mode = CS_MODE_DISABLED;
1626 /* Reset perf specific data */
1627 drvdata->perf_buf = NULL;
1629 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1631 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1635 static const struct coresight_ops_sink tmc_etr_sink_ops = {
1636 .enable = tmc_enable_etr_sink,
1637 .disable = tmc_disable_etr_sink,
1638 .alloc_buffer = tmc_alloc_etr_buffer,
1639 .update_buffer = tmc_update_etr_buffer,
1640 .free_buffer = tmc_free_etr_buffer,
1643 const struct coresight_ops tmc_etr_cs_ops = {
1644 .sink_ops = &tmc_etr_sink_ops,
1647 int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1650 unsigned long flags;
1652 /* config types are set a boot time and never change */
1653 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1656 spin_lock_irqsave(&drvdata->spinlock, flags);
1657 if (drvdata->reading) {
1663 * We can safely allow reads even if the ETR is operating in PERF mode,
1664 * since the sysfs session is captured in mode specific data.
1665 * If drvdata::sysfs_data is NULL the trace data has been read already.
1667 if (!drvdata->sysfs_buf) {
1672 /* Disable the TMC if we are trying to read from a running session. */
1673 if (drvdata->mode == CS_MODE_SYSFS)
1674 __tmc_etr_disable_hw(drvdata);
1676 drvdata->reading = true;
1678 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1683 int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1685 unsigned long flags;
1686 struct etr_buf *sysfs_buf = NULL;
1688 /* config types are set a boot time and never change */
1689 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1692 spin_lock_irqsave(&drvdata->spinlock, flags);
1694 /* RE-enable the TMC if need be */
1695 if (drvdata->mode == CS_MODE_SYSFS) {
1697 * The trace run will continue with the same allocated trace
1698 * buffer. Since the tracer is still enabled drvdata::buf can't
1701 __tmc_etr_enable_hw(drvdata);
1704 * The ETR is not tracing and the buffer was just read.
1705 * As such prepare to free the trace buffer.
1707 sysfs_buf = drvdata->sysfs_buf;
1708 drvdata->sysfs_buf = NULL;
1711 drvdata->reading = false;
1712 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1714 /* Free allocated memory out side of the spinlock */
1716 tmc_etr_free_sysfs_buf(sysfs_buf);