drivers/hwtracing/coresight/coresight-tmc-etf.c

   1 /*
   2  * Copyright(C) 2016 Linaro Limited. All rights reserved.
   3  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/circ_buf.h>
  19 #include <linux/coresight.h>
  20 #include <linux/perf_event.h>
  21 #include <linux/slab.h>
  22 #include "coresight-priv.h"
  23 #include "coresight-tmc.h"
  24
  25 static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
  26 {
  27         CS_UNLOCK(drvdata->base);
  28
  29         /* Wait for TMCSReady bit to be set */
  30         tmc_wait_for_tmcready(drvdata);
  31
  32         writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
  33         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
  34                        TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
  35                        TMC_FFCR_TRIGON_TRIGIN,
  36                        drvdata->base + TMC_FFCR);
  37
  38         writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
  39         tmc_enable_hw(drvdata);
  40
  41         CS_LOCK(drvdata->base);
  42 }
  43
  44 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
  45 {
  46         bool lost = false;
  47         char *bufp;
  48         const u32 *barrier;
  49         u32 read_data, status;
  50         int i;
  51
  52         /*
  53          * Get a hold of the status register and see if a wrap around
  54          * has occurred.
  55          */
  56         status = readl_relaxed(drvdata->base + TMC_STS);
  57         if (status & TMC_STS_FULL)
  58                 lost = true;
  59
  60         bufp = drvdata->buf;
  61         drvdata->len = 0;
  62         barrier = barrier_pkt;
  63         while (1) {
  64                 for (i = 0; i < drvdata->memwidth; i++) {
  65                         read_data = readl_relaxed(drvdata->base + TMC_RRD);
  66                         if (read_data == 0xFFFFFFFF)
  67                                 return;
  68
  69                         if (lost && *barrier) {
  70                                 read_data = *barrier;
  71                                 barrier++;
  72                         }
  73
  74                         memcpy(bufp, &read_data, 4);
  75                         bufp += 4;
  76                         drvdata->len += 4;
  77                 }
  78         }
  79 }
  80
  81 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
  82 {
  83         CS_UNLOCK(drvdata->base);
  84
  85         tmc_flush_and_stop(drvdata);
  86         /*
  87          * When operating in sysFS mode the content of the buffer needs to be
  88          * read before the TMC is disabled.
  89          */
  90         if (drvdata->mode == CS_MODE_SYSFS)
  91                 tmc_etb_dump_hw(drvdata);
  92         tmc_disable_hw(drvdata);
  93
  94         CS_LOCK(drvdata->base);
  95 }
  96
  97 static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
  98 {
  99         CS_UNLOCK(drvdata->base);
 100
 101         /* Wait for TMCSReady bit to be set */
 102         tmc_wait_for_tmcready(drvdata);
 103
 104         writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
 105         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
 106                        drvdata->base + TMC_FFCR);
 107         writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
 108         tmc_enable_hw(drvdata);
 109
 110         CS_LOCK(drvdata->base);
 111 }
 112
 113 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
 114 {
 115         CS_UNLOCK(drvdata->base);
 116
 117         tmc_flush_and_stop(drvdata);
 118         tmc_disable_hw(drvdata);
 119
 120         CS_LOCK(drvdata->base);
 121 }
 122
 123 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
 124 {
 125         int ret = 0;
 126         bool used = false;
 127         char *buf = NULL;
 128         unsigned long flags;
 129         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 130
 131         /*
 132          * If we don't have a buffer release the lock and allocate memory.
 133          * Otherwise keep the lock and move along.
 134          */
 135         spin_lock_irqsave(&drvdata->spinlock, flags);
 136         if (!drvdata->buf) {
 137                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 138
 139                 /* Allocating the memory here while outside of the spinlock */
 140                 buf = kzalloc(drvdata->size, GFP_KERNEL);
 141                 if (!buf)
 142                         return -ENOMEM;
 143
 144                 /* Let's try again */
 145                 spin_lock_irqsave(&drvdata->spinlock, flags);
 146         }
 147
 148         if (drvdata->reading) {
 149                 ret = -EBUSY;
 150                 goto out;
 151         }
 152
 153         /*
 154          * In sysFS mode we can have multiple writers per sink.  Since this
 155          * sink is already enabled no memory is needed and the HW need not be
 156          * touched.
 157          */
 158         if (drvdata->mode == CS_MODE_SYSFS)
 159                 goto out;
 160
 161         /*
 162          * If drvdata::buf isn't NULL, memory was allocated for a previous
 163          * trace run but wasn't read.  If so simply zero-out the memory.
 164          * Otherwise use the memory allocated above.
 165          *
 166          * The memory is freed when users read the buffer using the
 167          * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
 168          * details.
 169          */
 170         if (drvdata->buf) {
 171                 memset(drvdata->buf, 0, drvdata->size);
 172         } else {
 173                 used = true;
 174                 drvdata->buf = buf;
 175         }
 176
 177         drvdata->mode = CS_MODE_SYSFS;
 178         tmc_etb_enable_hw(drvdata);
 179 out:
 180         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 181
 182         /* Free memory outside the spinlock if need be */
 183         if (!used)
 184                 kfree(buf);
 185
 186         return ret;
 187 }
 188
 189 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev)
 190 {
 191         int ret = 0;
 192         unsigned long flags;
 193         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 194
 195         spin_lock_irqsave(&drvdata->spinlock, flags);
 196         if (drvdata->reading) {
 197                 ret = -EINVAL;
 198                 goto out;
 199         }
 200
 201         /*
 202          * In Perf mode there can be only one writer per sink.  There
 203          * is also no need to continue if the ETB/ETR is already operated
 204          * from sysFS.
 205          */
 206         if (drvdata->mode != CS_MODE_DISABLED) {
 207                 ret = -EINVAL;
 208                 goto out;
 209         }
 210
 211         drvdata->mode = CS_MODE_PERF;
 212         tmc_etb_enable_hw(drvdata);
 213 out:
 214         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 215
 216         return ret;
 217 }
 218
 219 static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode)
 220 {
 221         int ret;
 222         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 223
 224         switch (mode) {
 225         case CS_MODE_SYSFS:
 226                 ret = tmc_enable_etf_sink_sysfs(csdev);
 227                 break;
 228         case CS_MODE_PERF:
 229                 ret = tmc_enable_etf_sink_perf(csdev);
 230                 break;
 231         /* We shouldn't be here */
 232         default:
 233                 ret = -EINVAL;
 234                 break;
 235         }
 236
 237         if (ret)
 238                 return ret;
 239
 240         dev_info(drvdata->dev, "TMC-ETB/ETF enabled\n");
 241         return 0;
 242 }
 243
 244 static void tmc_disable_etf_sink(struct coresight_device *csdev)
 245 {
 246         unsigned long flags;
 247         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 248
 249         spin_lock_irqsave(&drvdata->spinlock, flags);
 250         if (drvdata->reading) {
 251                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 252                 return;
 253         }
 254
 255         /* Disable the TMC only if it needs to */
 256         if (drvdata->mode != CS_MODE_DISABLED) {
 257                 tmc_etb_disable_hw(drvdata);
 258                 drvdata->mode = CS_MODE_DISABLED;
 259         }
 260
 261         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 262
 263         dev_info(drvdata->dev, "TMC-ETB/ETF disabled\n");
 264 }
 265
 266 static int tmc_enable_etf_link(struct coresight_device *csdev,
 267                                int inport, int outport)
 268 {
 269         unsigned long flags;
 270         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 271
 272         spin_lock_irqsave(&drvdata->spinlock, flags);
 273         if (drvdata->reading) {
 274                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 275                 return -EBUSY;
 276         }
 277
 278         tmc_etf_enable_hw(drvdata);
 279         drvdata->mode = CS_MODE_SYSFS;
 280         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 281
 282         dev_info(drvdata->dev, "TMC-ETF enabled\n");
 283         return 0;
 284 }
 285
 286 static void tmc_disable_etf_link(struct coresight_device *csdev,
 287                                  int inport, int outport)
 288 {
 289         unsigned long flags;
 290         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 291
 292         spin_lock_irqsave(&drvdata->spinlock, flags);
 293         if (drvdata->reading) {
 294                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 295                 return;
 296         }
 297
 298         tmc_etf_disable_hw(drvdata);
 299         drvdata->mode = CS_MODE_DISABLED;
 300         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 301
 302         dev_info(drvdata->dev, "TMC-ETF disabled\n");
 303 }
 304
 305 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, int cpu,
 306                                   void **pages, int nr_pages, bool overwrite)
 307 {
 308         int node;
 309         struct cs_buffers *buf;
 310
 311         if (cpu == -1)
 312                 cpu = smp_processor_id();
 313         node = cpu_to_node(cpu);
 314
 315         /* Allocate memory structure for interaction with Perf */
 316         buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
 317         if (!buf)
 318                 return NULL;
 319
 320         buf->snapshot = overwrite;
 321         buf->nr_pages = nr_pages;
 322         buf->data_pages = pages;
 323
 324         return buf;
 325 }
 326
 327 static void tmc_free_etf_buffer(void *config)
 328 {
 329         struct cs_buffers *buf = config;
 330
 331         kfree(buf);
 332 }
 333
 334 static int tmc_set_etf_buffer(struct coresight_device *csdev,
 335                               struct perf_output_handle *handle,
 336                               void *sink_config)
 337 {
 338         int ret = 0;
 339         unsigned long head;
 340         struct cs_buffers *buf = sink_config;
 341
 342         /* wrap head around to the amount of space we have */
 343         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 344
 345         /* find the page to write to */
 346         buf->cur = head / PAGE_SIZE;
 347
 348         /* and offset within that page */
 349         buf->offset = head % PAGE_SIZE;
 350
 351         local_set(&buf->data_size, 0);
 352
 353         return ret;
 354 }
 355
 356 static unsigned long tmc_reset_etf_buffer(struct coresight_device *csdev,
 357                                           struct perf_output_handle *handle,
 358                                           void *sink_config)
 359 {
 360         long size = 0;
 361         struct cs_buffers *buf = sink_config;
 362
 363         if (buf) {
 364                 /*
 365                  * In snapshot mode ->data_size holds the new address of the
 366                  * ring buffer's head.  The size itself is the whole address
 367                  * range since we want the latest information.
 368                  */
 369                 if (buf->snapshot)
 370                         handle->head = local_xchg(&buf->data_size,
 371                                                   buf->nr_pages << PAGE_SHIFT);
 372                 /*
 373                  * Tell the tracer PMU how much we got in this run and if
 374                  * something went wrong along the way.  Nobody else can use
 375                  * this cs_buffers instance until we are done.  As such
 376                  * resetting parameters here and squaring off with the ring
 377                  * buffer API in the tracer PMU is fine.
 378                  */
 379                 size = local_xchg(&buf->data_size, 0);
 380         }
 381
 382         return size;
 383 }
 384
 385 static void tmc_update_etf_buffer(struct coresight_device *csdev,
 386                                   struct perf_output_handle *handle,
 387                                   void *sink_config)
 388 {
 389         bool lost = false;
 390         int i, cur;
 391         const u32 *barrier;
 392         u32 *buf_ptr;
 393         u64 read_ptr, write_ptr;
 394         u32 status, to_read;
 395         unsigned long offset;
 396         struct cs_buffers *buf = sink_config;
 397         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 398
 399         if (!buf)
 400                 return;
 401
 402         /* This shouldn't happen */
 403         if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
 404                 return;
 405
 406         CS_UNLOCK(drvdata->base);
 407
 408         tmc_flush_and_stop(drvdata);
 409
 410         read_ptr = tmc_read_rrp(drvdata);
 411         write_ptr = tmc_read_rwp(drvdata);
 412
 413         /*
 414          * Get a hold of the status register and see if a wrap around
 415          * has occurred.  If so adjust things accordingly.
 416          */
 417         status = readl_relaxed(drvdata->base + TMC_STS);
 418         if (status & TMC_STS_FULL) {
 419                 lost = true;
 420                 to_read = drvdata->size;
 421         } else {
 422                 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
 423         }
 424
 425         /*
 426          * The TMC RAM buffer may be bigger than the space available in the
 427          * perf ring buffer (handle->size).  If so advance the RRP so that we
 428          * get the latest trace data.
 429          */
 430         if (to_read > handle->size) {
 431                 u32 mask = 0;
 432
 433                 /*
 434                  * The value written to RRP must be byte-address aligned to
 435                  * the width of the trace memory databus _and_ to a frame
 436                  * boundary (16 byte), whichever is the biggest. For example,
 437                  * for 32-bit, 64-bit and 128-bit wide trace memory, the four
 438                  * LSBs must be 0s. For 256-bit wide trace memory, the five
 439                  * LSBs must be 0s.
 440                  */
 441                 switch (drvdata->memwidth) {
 442                 case TMC_MEM_INTF_WIDTH_32BITS:
 443                 case TMC_MEM_INTF_WIDTH_64BITS:
 444                 case TMC_MEM_INTF_WIDTH_128BITS:
 445                         mask = GENMASK(31, 5);
 446                         break;
 447                 case TMC_MEM_INTF_WIDTH_256BITS:
 448                         mask = GENMASK(31, 6);
 449                         break;
 450                 }
 451
 452                 /*
 453                  * Make sure the new size is aligned in accordance with the
 454                  * requirement explained above.
 455                  */
 456                 to_read = handle->size & mask;
 457                 /* Move the RAM read pointer up */
 458                 read_ptr = (write_ptr + drvdata->size) - to_read;
 459                 /* Make sure we are still within our limits */
 460                 if (read_ptr > (drvdata->size - 1))
 461                         read_ptr -= drvdata->size;
 462                 /* Tell the HW */
 463                 tmc_write_rrp(drvdata, read_ptr);
 464                 lost = true;
 465         }
 466
 467         if (lost)
 468                 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 469
 470         cur = buf->cur;
 471         offset = buf->offset;
 472         barrier = barrier_pkt;
 473
 474         /* for every byte to read */
 475         for (i = 0; i < to_read; i += 4) {
 476                 buf_ptr = buf->data_pages[cur] + offset;
 477                 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
 478
 479                 if (lost && *barrier) {
 480                         *buf_ptr = *barrier;
 481                         barrier++;
 482                 }
 483
 484                 offset += 4;
 485                 if (offset >= PAGE_SIZE) {
 486                         offset = 0;
 487                         cur++;
 488                         /* wrap around at the end of the buffer */
 489                         cur &= buf->nr_pages - 1;
 490                 }
 491         }
 492
 493         /*
 494          * In snapshot mode all we have to do is communicate to
 495          * perf_aux_output_end() the address of the current head.  In full
 496          * trace mode the same function expects a size to move rb->aux_head
 497          * forward.
 498          */
 499         if (buf->snapshot)
 500                 local_set(&buf->data_size, (cur * PAGE_SIZE) + offset);
 501         else
 502                 local_add(to_read, &buf->data_size);
 503
 504         CS_LOCK(drvdata->base);
 505 }
 506
 507 static const struct coresight_ops_sink tmc_etf_sink_ops = {
 508         .enable         = tmc_enable_etf_sink,
 509         .disable        = tmc_disable_etf_sink,
 510         .alloc_buffer   = tmc_alloc_etf_buffer,
 511         .free_buffer    = tmc_free_etf_buffer,
 512         .set_buffer     = tmc_set_etf_buffer,
 513         .reset_buffer   = tmc_reset_etf_buffer,
 514         .update_buffer  = tmc_update_etf_buffer,
 515 };
 516
 517 static const struct coresight_ops_link tmc_etf_link_ops = {
 518         .enable         = tmc_enable_etf_link,
 519         .disable        = tmc_disable_etf_link,
 520 };
 521
 522 const struct coresight_ops tmc_etb_cs_ops = {
 523         .sink_ops       = &tmc_etf_sink_ops,
 524 };
 525
 526 const struct coresight_ops tmc_etf_cs_ops = {
 527         .sink_ops       = &tmc_etf_sink_ops,
 528         .link_ops       = &tmc_etf_link_ops,
 529 };
 530
 531 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 532 {
 533         enum tmc_mode mode;
 534         int ret = 0;
 535         unsigned long flags;
 536
 537         /* config types are set a boot time and never change */
 538         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 539                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 540                 return -EINVAL;
 541
 542         spin_lock_irqsave(&drvdata->spinlock, flags);
 543
 544         if (drvdata->reading) {
 545                 ret = -EBUSY;
 546                 goto out;
 547         }
 548
 549         /* There is no point in reading a TMC in HW FIFO mode */
 550         mode = readl_relaxed(drvdata->base + TMC_MODE);
 551         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 552                 ret = -EINVAL;
 553                 goto out;
 554         }
 555
 556         /* Don't interfere if operated from Perf */
 557         if (drvdata->mode == CS_MODE_PERF) {
 558                 ret = -EINVAL;
 559                 goto out;
 560         }
 561
 562         /* If drvdata::buf is NULL the trace data has been read already */
 563         if (drvdata->buf == NULL) {
 564                 ret = -EINVAL;
 565                 goto out;
 566         }
 567
 568         /* Disable the TMC if need be */
 569         if (drvdata->mode == CS_MODE_SYSFS)
 570                 tmc_etb_disable_hw(drvdata);
 571
 572         drvdata->reading = true;
 573 out:
 574         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 575
 576         return ret;
 577 }
 578
 579 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
 580 {
 581         char *buf = NULL;
 582         enum tmc_mode mode;
 583         unsigned long flags;
 584
 585         /* config types are set a boot time and never change */
 586         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 587                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 588                 return -EINVAL;
 589
 590         spin_lock_irqsave(&drvdata->spinlock, flags);
 591
 592         /* There is no point in reading a TMC in HW FIFO mode */
 593         mode = readl_relaxed(drvdata->base + TMC_MODE);
 594         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 595                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 596                 return -EINVAL;
 597         }
 598
 599         /* Re-enable the TMC if need be */
 600         if (drvdata->mode == CS_MODE_SYSFS) {
 601                 /*
 602                  * The trace run will continue with the same allocated trace
 603                  * buffer. As such zero-out the buffer so that we don't end
 604                  * up with stale data.
 605                  *
 606                  * Since the tracer is still enabled drvdata::buf
 607                  * can't be NULL.
 608                  */
 609                 memset(drvdata->buf, 0, drvdata->size);
 610                 tmc_etb_enable_hw(drvdata);
 611         } else {
 612                 /*
 613                  * The ETB/ETF is not tracing and the buffer was just read.
 614                  * As such prepare to free the trace buffer.
 615                  */
 616                 buf = drvdata->buf;
 617                 drvdata->buf = NULL;
 618         }
 619
 620         drvdata->reading = false;
 621         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 622
 623         /*
 624          * Free allocated memory outside of the spinlock.  There is no need
 625          * to assert the validity of 'buf' since calling kfree(NULL) is safe.
 626          */
 627         kfree(buf);
 628
 629         return 0;
 630 }