drivers/soc/ti/k3-ringacc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * TI K3 NAVSS Ring Accelerator subsystem driver
   4  *
   5  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   6  */
   7
   8 #include <linux/dma-mapping.h>
   9 #include <linux/io.h>
  10 #include <linux/init.h>
  11 #include <linux/of.h>
  12 #include <linux/of_device.h>
  13 #include <linux/platform_device.h>
  14 #include <linux/sys_soc.h>
  15 #include <linux/dma/ti-cppi5.h>
  16 #include <linux/soc/ti/k3-ringacc.h>
  17 #include <linux/soc/ti/ti_sci_protocol.h>
  18 #include <linux/soc/ti/ti_sci_inta_msi.h>
  19 #include <linux/of_irq.h>
  20 #include <linux/irqdomain.h>
  21
  22 static LIST_HEAD(k3_ringacc_list);
  23 static DEFINE_MUTEX(k3_ringacc_list_lock);
  24
  25 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK             GENMASK(19, 0)
  26 #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK             GENMASK(15, 0)
  27
  28 /**
  29  * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
  30  *
  31  * @resv_16: Reserved
  32  * @db: Ring Doorbell Register
  33  * @resv_4: Reserved
  34  * @occ: Ring Occupancy Register
  35  * @indx: Ring Current Index Register
  36  * @hwocc: Ring Hardware Occupancy Register
  37  * @hwindx: Ring Hardware Current Index Register
  38  */
  39 struct k3_ring_rt_regs {
  40         u32     resv_16[4];
  41         u32     db;
  42         u32     resv_4[1];
  43         u32     occ;
  44         u32     indx;
  45         u32     hwocc;
  46         u32     hwindx;
  47 };
  48
  49 #define K3_RINGACC_RT_REGS_STEP                 0x1000
  50 #define K3_DMARING_RT_REGS_STEP                 0x2000
  51 #define K3_DMARING_RT_REGS_REVERSE_OFS          0x1000
  52 #define K3_RINGACC_RT_OCC_MASK                  GENMASK(20, 0)
  53 #define K3_DMARING_RT_OCC_TDOWN_COMPLETE        BIT(31)
  54 #define K3_DMARING_RT_DB_ENTRY_MASK             GENMASK(7, 0)
  55 #define K3_DMARING_RT_DB_TDOWN_ACK              BIT(31)
  56
  57 /**
  58  * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
  59  *
  60  * @head_data: Ring Head Entry Data Registers
  61  * @tail_data: Ring Tail Entry Data Registers
  62  * @peek_head_data: Ring Peek Head Entry Data Regs
  63  * @peek_tail_data: Ring Peek Tail Entry Data Regs
  64  */
  65 struct k3_ring_fifo_regs {
  66         u32     head_data[128];
  67         u32     tail_data[128];
  68         u32     peek_head_data[128];
  69         u32     peek_tail_data[128];
  70 };
  71
  72 /**
  73  * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
  74  *
  75  * @revision: Revision Register
  76  * @config: Config Register
  77  */
  78 struct k3_ringacc_proxy_gcfg_regs {
  79         u32     revision;
  80         u32     config;
  81 };
  82
  83 #define K3_RINGACC_PROXY_CFG_THREADS_MASK               GENMASK(15, 0)
  84
  85 /**
  86  * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region
  87  *
  88  * @control: Proxy Control Register
  89  * @status: Proxy Status Register
  90  * @resv_512: Reserved
  91  * @data: Proxy Data Register
  92  */
  93 struct k3_ringacc_proxy_target_regs {
  94         u32     control;
  95         u32     status;
  96         u8      resv_512[504];
  97         u32     data[128];
  98 };
  99
 100 #define K3_RINGACC_PROXY_TARGET_STEP    0x1000
 101 #define K3_RINGACC_PROXY_NOT_USED       (-1)
 102
 103 enum k3_ringacc_proxy_access_mode {
 104         PROXY_ACCESS_MODE_HEAD = 0,
 105         PROXY_ACCESS_MODE_TAIL = 1,
 106         PROXY_ACCESS_MODE_PEEK_HEAD = 2,
 107         PROXY_ACCESS_MODE_PEEK_TAIL = 3,
 108 };
 109
 110 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
 111 #define K3_RINGACC_FIFO_REGS_STEP       0x1000
 112 #define K3_RINGACC_MAX_DB_RING_CNT    (127U)
 113
 114 struct k3_ring_ops {
 115         int (*push_tail)(struct k3_ring *ring, void *elm);
 116         int (*push_head)(struct k3_ring *ring, void *elm);
 117         int (*pop_tail)(struct k3_ring *ring, void *elm);
 118         int (*pop_head)(struct k3_ring *ring, void *elm);
 119 };
 120
 121 /**
 122  * struct k3_ring_state - Internal state tracking structure
 123  *
 124  * @free: Number of free entries
 125  * @occ: Occupancy
 126  * @windex: Write index
 127  * @rindex: Read index
 128  */
 129 struct k3_ring_state {
 130         u32 free;
 131         u32 occ;
 132         u32 windex;
 133         u32 rindex;
 134         u32 tdown_complete:1;
 135 };
 136
 137 /**
 138  * struct k3_ring - RA Ring descriptor
 139  *
 140  * @rt: Ring control/status registers
 141  * @fifos: Ring queues registers
 142  * @proxy: Ring Proxy Datapath registers
 143  * @ring_mem_dma: Ring buffer dma address
 144  * @ring_mem_virt: Ring buffer virt address
 145  * @ops: Ring operations
 146  * @size: Ring size in elements
 147  * @elm_size: Size of the ring element
 148  * @mode: Ring mode
 149  * @flags: flags
 150  * @state: Ring state
 151  * @ring_id: Ring Id
 152  * @parent: Pointer on struct @k3_ringacc
 153  * @use_count: Use count for shared rings
 154  * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
 155  * @dma_dev: device to be used for DMA API (allocation, mapping)
 156  * @asel: Address Space Select value for physical addresses
 157  */
 158 struct k3_ring {
 159         struct k3_ring_rt_regs __iomem *rt;
 160         struct k3_ring_fifo_regs __iomem *fifos;
 161         struct k3_ringacc_proxy_target_regs  __iomem *proxy;
 162         dma_addr_t      ring_mem_dma;
 163         void            *ring_mem_virt;
 164         struct k3_ring_ops *ops;
 165         u32             size;
 166         enum k3_ring_size elm_size;
 167         enum k3_ring_mode mode;
 168         u32             flags;
 169 #define K3_RING_FLAG_BUSY       BIT(1)
 170 #define K3_RING_FLAG_SHARED     BIT(2)
 171 #define K3_RING_FLAG_REVERSE    BIT(3)
 172         struct k3_ring_state state;
 173         u32             ring_id;
 174         struct k3_ringacc       *parent;
 175         u32             use_count;
 176         int             proxy_id;
 177         struct device   *dma_dev;
 178         u32             asel;
 179 #define K3_ADDRESS_ASEL_SHIFT   48
 180 };
 181
 182 struct k3_ringacc_ops {
 183         int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc);
 184 };
 185
 186 /**
 187  * struct k3_ringacc - Rings accelerator descriptor
 188  *
 189  * @dev: pointer on RA device
 190  * @proxy_gcfg: RA proxy global config registers
 191  * @proxy_target_base: RA proxy datapath region
 192  * @num_rings: number of ring in RA
 193  * @rings_inuse: bitfield for ring usage tracking
 194  * @rm_gp_range: general purpose rings range from tisci
 195  * @dma_ring_reset_quirk: DMA reset w/a enable
 196  * @num_proxies: number of RA proxies
 197  * @proxy_inuse: bitfield for proxy usage tracking
 198  * @rings: array of rings descriptors (struct @k3_ring)
 199  * @list: list of RAs in the system
 200  * @req_lock: protect rings allocation
 201  * @tisci: pointer ti-sci handle
 202  * @tisci_ring_ops: ti-sci rings ops
 203  * @tisci_dev_id: ti-sci device id
 204  * @ops: SoC specific ringacc operation
 205  * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA)
 206  */
 207 struct k3_ringacc {
 208         struct device *dev;
 209         struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
 210         void __iomem *proxy_target_base;
 211         u32 num_rings; /* number of rings in Ringacc module */
 212         unsigned long *rings_inuse;
 213         struct ti_sci_resource *rm_gp_range;
 214
 215         bool dma_ring_reset_quirk;
 216         u32 num_proxies;
 217         unsigned long *proxy_inuse;
 218
 219         struct k3_ring *rings;
 220         struct list_head list;
 221         struct mutex req_lock; /* protect rings allocation */
 222
 223         const struct ti_sci_handle *tisci;
 224         const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
 225         u32 tisci_dev_id;
 226
 227         const struct k3_ringacc_ops *ops;
 228         bool dma_rings;
 229 };
 230
 231 /**
 232  * struct k3_ringacc - Rings accelerator SoC data
 233  *
 234  * @dma_ring_reset_quirk:  DMA reset w/a enable
 235  */
 236 struct k3_ringacc_soc_data {
 237         unsigned dma_ring_reset_quirk:1;
 238 };
 239
 240 static int k3_ringacc_ring_read_occ(struct k3_ring *ring)
 241 {
 242         return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK;
 243 }
 244
 245 static void k3_ringacc_ring_update_occ(struct k3_ring *ring)
 246 {
 247         u32 val;
 248
 249         val = readl(&ring->rt->occ);
 250
 251         ring->state.occ = val & K3_RINGACC_RT_OCC_MASK;
 252         ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE);
 253 }
 254
 255 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
 256 {
 257         return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
 258                (4 << ring->elm_size);
 259 }
 260
 261 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
 262 {
 263         return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
 264 }
 265
 266 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
 267 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
 268 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
 269 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
 270
 271 static struct k3_ring_ops k3_ring_mode_ring_ops = {
 272                 .push_tail = k3_ringacc_ring_push_mem,
 273                 .pop_head = k3_ringacc_ring_pop_mem,
 274 };
 275
 276 static struct k3_ring_ops k3_dmaring_fwd_ops = {
 277                 .push_tail = k3_ringacc_ring_push_mem,
 278                 .pop_head = k3_dmaring_fwd_pop,
 279 };
 280
 281 static struct k3_ring_ops k3_dmaring_reverse_ops = {
 282                 /* Reverse side of the DMA ring can only be popped by SW */
 283                 .pop_head = k3_dmaring_reverse_pop,
 284 };
 285
 286 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
 287 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
 288 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
 289 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
 290
 291 static struct k3_ring_ops k3_ring_mode_msg_ops = {
 292                 .push_tail = k3_ringacc_ring_push_io,
 293                 .push_head = k3_ringacc_ring_push_head_io,
 294                 .pop_tail = k3_ringacc_ring_pop_tail_io,
 295                 .pop_head = k3_ringacc_ring_pop_io,
 296 };
 297
 298 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
 299 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
 300 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
 301 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
 302
 303 static struct k3_ring_ops k3_ring_mode_proxy_ops = {
 304                 .push_tail = k3_ringacc_ring_push_tail_proxy,
 305                 .push_head = k3_ringacc_ring_push_head_proxy,
 306                 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
 307                 .pop_head = k3_ringacc_ring_pop_head_proxy,
 308 };
 309
 310 static void k3_ringacc_ring_dump(struct k3_ring *ring)
 311 {
 312         struct device *dev = ring->parent->dev;
 313
 314         dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
 315         dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
 316                 &ring->ring_mem_dma);
 317         dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
 318                 ring->elm_size, ring->size, ring->mode, ring->proxy_id);
 319         dev_dbg(dev, "dump flags %08X\n", ring->flags);
 320
 321         dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
 322         dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
 323         dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
 324         dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
 325         dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
 326
 327         if (ring->ring_mem_virt)
 328                 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE,
 329                                      16, 1, ring->ring_mem_virt, 16 * 8, false);
 330 }
 331
 332 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
 333                                         int id, u32 flags)
 334 {
 335         int proxy_id = K3_RINGACC_PROXY_NOT_USED;
 336
 337         mutex_lock(&ringacc->req_lock);
 338
 339         if (id == K3_RINGACC_RING_ID_ANY) {
 340                 /* Request for any general purpose ring */
 341                 struct ti_sci_resource_desc *gp_rings =
 342                                                 &ringacc->rm_gp_range->desc[0];
 343                 unsigned long size;
 344
 345                 size = gp_rings->start + gp_rings->num;
 346                 id = find_next_zero_bit(ringacc->rings_inuse, size,
 347                                         gp_rings->start);
 348                 if (id == size)
 349                         goto error;
 350         } else if (id < 0) {
 351                 goto error;
 352         }
 353
 354         if (test_bit(id, ringacc->rings_inuse) &&
 355             !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
 356                 goto error;
 357         else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
 358                 goto out;
 359
 360         if (flags & K3_RINGACC_RING_USE_PROXY) {
 361                 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
 362                                               ringacc->num_proxies, 0);
 363                 if (proxy_id == ringacc->num_proxies)
 364                         goto error;
 365         }
 366
 367         if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 368                 set_bit(proxy_id, ringacc->proxy_inuse);
 369                 ringacc->rings[id].proxy_id = proxy_id;
 370                 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
 371                         proxy_id);
 372         } else {
 373                 dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
 374         }
 375
 376         set_bit(id, ringacc->rings_inuse);
 377 out:
 378         ringacc->rings[id].use_count++;
 379         mutex_unlock(&ringacc->req_lock);
 380         return &ringacc->rings[id];
 381
 382 error:
 383         mutex_unlock(&ringacc->req_lock);
 384         return NULL;
 385 }
 386 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
 387
 388 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
 389                                         struct k3_ring **fwd_ring,
 390                                         struct k3_ring **compl_ring)
 391 {
 392         int ret = 0;
 393
 394         /*
 395          * DMA rings must be requested by ID, completion ring is the reverse
 396          * side of the forward ring
 397          */
 398         if (fwd_id < 0)
 399                 return -EINVAL;
 400
 401         mutex_lock(&ringacc->req_lock);
 402
 403         if (test_bit(fwd_id, ringacc->rings_inuse)) {
 404                 ret = -EBUSY;
 405                 goto error;
 406         }
 407
 408         *fwd_ring = &ringacc->rings[fwd_id];
 409         *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
 410         set_bit(fwd_id, ringacc->rings_inuse);
 411         ringacc->rings[fwd_id].use_count++;
 412         dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
 413
 414         mutex_unlock(&ringacc->req_lock);
 415         return 0;
 416
 417 error:
 418         mutex_unlock(&ringacc->req_lock);
 419         return ret;
 420 }
 421
 422 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
 423                                   int fwd_id, int compl_id,
 424                                   struct k3_ring **fwd_ring,
 425                                   struct k3_ring **compl_ring)
 426 {
 427         int ret = 0;
 428
 429         if (!fwd_ring || !compl_ring)
 430                 return -EINVAL;
 431
 432         if (ringacc->dma_rings)
 433                 return k3_dmaring_request_dual_ring(ringacc, fwd_id,
 434                                                     fwd_ring, compl_ring);
 435
 436         *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
 437         if (!(*fwd_ring))
 438                 return -ENODEV;
 439
 440         *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0);
 441         if (!(*compl_ring)) {
 442                 k3_ringacc_ring_free(*fwd_ring);
 443                 ret = -ENODEV;
 444         }
 445
 446         return ret;
 447 }
 448 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair);
 449
 450 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
 451 {
 452         struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
 453         struct k3_ringacc *ringacc = ring->parent;
 454         int ret;
 455
 456         ring_cfg.nav_id = ringacc->tisci_dev_id;
 457         ring_cfg.index = ring->ring_id;
 458         ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID;
 459         ring_cfg.count = ring->size;
 460
 461         ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
 462         if (ret)
 463                 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
 464                         ret, ring->ring_id);
 465 }
 466
 467 void k3_ringacc_ring_reset(struct k3_ring *ring)
 468 {
 469         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 470                 return;
 471
 472         memset(&ring->state, 0, sizeof(ring->state));
 473
 474         k3_ringacc_ring_reset_sci(ring);
 475 }
 476 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
 477
 478 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
 479                                                enum k3_ring_mode mode)
 480 {
 481         struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
 482         struct k3_ringacc *ringacc = ring->parent;
 483         int ret;
 484
 485         ring_cfg.nav_id = ringacc->tisci_dev_id;
 486         ring_cfg.index = ring->ring_id;
 487         ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID;
 488         ring_cfg.mode = mode;
 489
 490         ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
 491         if (ret)
 492                 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
 493                         ret, ring->ring_id);
 494 }
 495
 496 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
 497 {
 498         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 499                 return;
 500
 501         if (!ring->parent->dma_ring_reset_quirk)
 502                 goto reset;
 503
 504         if (!occ)
 505                 occ = k3_ringacc_ring_read_occ(ring);
 506
 507         if (occ) {
 508                 u32 db_ring_cnt, db_ring_cnt_cur;
 509
 510                 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
 511                         ring->ring_id, occ);
 512                 /* TI-SCI ring reset */
 513                 k3_ringacc_ring_reset_sci(ring);
 514
 515                 /*
 516                  * Setup the ring in ring/doorbell mode (if not already in this
 517                  * mode)
 518                  */
 519                 if (ring->mode != K3_RINGACC_RING_MODE_RING)
 520                         k3_ringacc_ring_reconfig_qmode_sci(
 521                                         ring, K3_RINGACC_RING_MODE_RING);
 522                 /*
 523                  * Ring the doorbell 2**22 – ringOcc times.
 524                  * This will wrap the internal UDMAP ring state occupancy
 525                  * counter (which is 21-bits wide) to 0.
 526                  */
 527                 db_ring_cnt = (1U << 22) - occ;
 528
 529                 while (db_ring_cnt != 0) {
 530                         /*
 531                          * Ring the doorbell with the maximum count each
 532                          * iteration if possible to minimize the total
 533                          * of writes
 534                          */
 535                         if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
 536                                 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
 537                         else
 538                                 db_ring_cnt_cur = db_ring_cnt;
 539
 540                         writel(db_ring_cnt_cur, &ring->rt->db);
 541                         db_ring_cnt -= db_ring_cnt_cur;
 542                 }
 543
 544                 /* Restore the original ring mode (if not ring mode) */
 545                 if (ring->mode != K3_RINGACC_RING_MODE_RING)
 546                         k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
 547         }
 548
 549 reset:
 550         /* Reset the ring */
 551         k3_ringacc_ring_reset(ring);
 552 }
 553 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
 554
 555 static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
 556 {
 557         struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
 558         struct k3_ringacc *ringacc = ring->parent;
 559         int ret;
 560
 561         ring_cfg.nav_id = ringacc->tisci_dev_id;
 562         ring_cfg.index = ring->ring_id;
 563         ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
 564
 565         ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
 566         if (ret)
 567                 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
 568                         ret, ring->ring_id);
 569 }
 570
 571 int k3_ringacc_ring_free(struct k3_ring *ring)
 572 {
 573         struct k3_ringacc *ringacc;
 574
 575         if (!ring)
 576                 return -EINVAL;
 577
 578         ringacc = ring->parent;
 579
 580         /*
 581          * DMA rings: rings shared memory and configuration, only forward ring
 582          * is configured and reverse ring considered as slave.
 583          */
 584         if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
 585                 return 0;
 586
 587         dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
 588
 589         if (!test_bit(ring->ring_id, ringacc->rings_inuse))
 590                 return -EINVAL;
 591
 592         mutex_lock(&ringacc->req_lock);
 593
 594         if (--ring->use_count)
 595                 goto out;
 596
 597         if (!(ring->flags & K3_RING_FLAG_BUSY))
 598                 goto no_init;
 599
 600         k3_ringacc_ring_free_sci(ring);
 601
 602         dma_free_coherent(ring->dma_dev,
 603                           ring->size * (4 << ring->elm_size),
 604                           ring->ring_mem_virt, ring->ring_mem_dma);
 605         ring->flags = 0;
 606         ring->ops = NULL;
 607         ring->dma_dev = NULL;
 608         ring->asel = 0;
 609
 610         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 611                 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
 612                 ring->proxy = NULL;
 613                 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
 614         }
 615
 616 no_init:
 617         clear_bit(ring->ring_id, ringacc->rings_inuse);
 618
 619 out:
 620         mutex_unlock(&ringacc->req_lock);
 621         return 0;
 622 }
 623 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
 624
 625 u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
 626 {
 627         if (!ring)
 628                 return -EINVAL;
 629
 630         return ring->ring_id;
 631 }
 632 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
 633
 634 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
 635 {
 636         if (!ring)
 637                 return -EINVAL;
 638
 639         return ring->parent->tisci_dev_id;
 640 }
 641 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
 642
 643 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
 644 {
 645         int irq_num;
 646
 647         if (!ring)
 648                 return -EINVAL;
 649
 650         irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id);
 651         if (irq_num <= 0)
 652                 irq_num = -EINVAL;
 653         return irq_num;
 654 }
 655 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num);
 656
 657 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
 658 {
 659         struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
 660         struct k3_ringacc *ringacc = ring->parent;
 661         int ret;
 662
 663         if (!ringacc->tisci)
 664                 return -EINVAL;
 665
 666         ring_cfg.nav_id = ringacc->tisci_dev_id;
 667         ring_cfg.index = ring->ring_id;
 668         ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
 669         ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma);
 670         ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma);
 671         ring_cfg.count = ring->size;
 672         ring_cfg.mode = ring->mode;
 673         ring_cfg.size = ring->elm_size;
 674         ring_cfg.asel = ring->asel;
 675
 676         ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
 677         if (ret)
 678                 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
 679                         ret, ring->ring_id);
 680
 681         return ret;
 682 }
 683
 684 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
 685 {
 686         struct k3_ringacc *ringacc;
 687         struct k3_ring *reverse_ring;
 688         int ret = 0;
 689
 690         if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 ||
 691             cfg->mode != K3_RINGACC_RING_MODE_RING ||
 692             cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK)
 693                 return -EINVAL;
 694
 695         ringacc = ring->parent;
 696
 697         /*
 698          * DMA rings: rings shared memory and configuration, only forward ring
 699          * is configured and reverse ring considered as slave.
 700          */
 701         if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
 702                 return 0;
 703
 704         if (!test_bit(ring->ring_id, ringacc->rings_inuse))
 705                 return -EINVAL;
 706
 707         ring->size = cfg->size;
 708         ring->elm_size = cfg->elm_size;
 709         ring->mode = cfg->mode;
 710         ring->asel = cfg->asel;
 711         ring->dma_dev = cfg->dma_dev;
 712         if (!ring->dma_dev) {
 713                 dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",
 714                          ring->ring_id);
 715                 ring->dma_dev = ringacc->dev;
 716         }
 717
 718         memset(&ring->state, 0, sizeof(ring->state));
 719
 720         ring->ops = &k3_dmaring_fwd_ops;
 721
 722         ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
 723                                                  ring->size * (4 << ring->elm_size),
 724                                                  &ring->ring_mem_dma, GFP_KERNEL);
 725         if (!ring->ring_mem_virt) {
 726                 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
 727                 ret = -ENOMEM;
 728                 goto err_free_ops;
 729         }
 730
 731         ret = k3_ringacc_ring_cfg_sci(ring);
 732         if (ret)
 733                 goto err_free_mem;
 734
 735         ring->flags |= K3_RING_FLAG_BUSY;
 736
 737         k3_ringacc_ring_dump(ring);
 738
 739         /* DMA rings: configure reverse ring */
 740         reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
 741         reverse_ring->size = cfg->size;
 742         reverse_ring->elm_size = cfg->elm_size;
 743         reverse_ring->mode = cfg->mode;
 744         reverse_ring->asel = cfg->asel;
 745         memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
 746         reverse_ring->ops = &k3_dmaring_reverse_ops;
 747
 748         reverse_ring->ring_mem_virt = ring->ring_mem_virt;
 749         reverse_ring->ring_mem_dma = ring->ring_mem_dma;
 750         reverse_ring->flags |= K3_RING_FLAG_BUSY;
 751         k3_ringacc_ring_dump(reverse_ring);
 752
 753         return 0;
 754
 755 err_free_mem:
 756         dma_free_coherent(ring->dma_dev,
 757                           ring->size * (4 << ring->elm_size),
 758                           ring->ring_mem_virt,
 759                           ring->ring_mem_dma);
 760 err_free_ops:
 761         ring->ops = NULL;
 762         ring->proxy = NULL;
 763         ring->dma_dev = NULL;
 764         ring->asel = 0;
 765         return ret;
 766 }
 767
 768 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
 769 {
 770         struct k3_ringacc *ringacc;
 771         int ret = 0;
 772
 773         if (!ring || !cfg)
 774                 return -EINVAL;
 775
 776         ringacc = ring->parent;
 777
 778         if (ringacc->dma_rings)
 779                 return k3_dmaring_cfg(ring, cfg);
 780
 781         if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
 782             cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
 783             cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
 784             !test_bit(ring->ring_id, ringacc->rings_inuse))
 785                 return -EINVAL;
 786
 787         if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE &&
 788             ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
 789             cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) {
 790                 dev_err(ringacc->dev,
 791                         "Message mode must use proxy for %u element size\n",
 792                         4 << ring->elm_size);
 793                 return -EINVAL;
 794         }
 795
 796         /*
 797          * In case of shared ring only the first user (master user) can
 798          * configure the ring. The sequence should be by the client:
 799          * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user
 800          * k3_ringacc_ring_cfg(ring, cfg); # master configuration
 801          * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
 802          * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
 803          */
 804         if (ring->use_count != 1)
 805                 return 0;
 806
 807         ring->size = cfg->size;
 808         ring->elm_size = cfg->elm_size;
 809         ring->mode = cfg->mode;
 810         memset(&ring->state, 0, sizeof(ring->state));
 811
 812         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
 813                 ring->proxy = ringacc->proxy_target_base +
 814                               ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
 815
 816         switch (ring->mode) {
 817         case K3_RINGACC_RING_MODE_RING:
 818                 ring->ops = &k3_ring_mode_ring_ops;
 819                 ring->dma_dev = cfg->dma_dev;
 820                 if (!ring->dma_dev)
 821                         ring->dma_dev = ringacc->dev;
 822                 break;
 823         case K3_RINGACC_RING_MODE_MESSAGE:
 824                 ring->dma_dev = ringacc->dev;
 825                 if (ring->proxy)
 826                         ring->ops = &k3_ring_mode_proxy_ops;
 827                 else
 828                         ring->ops = &k3_ring_mode_msg_ops;
 829                 break;
 830         default:
 831                 ring->ops = NULL;
 832                 ret = -EINVAL;
 833                 goto err_free_proxy;
 834         }
 835
 836         ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
 837                                                  ring->size * (4 << ring->elm_size),
 838                                                  &ring->ring_mem_dma, GFP_KERNEL);
 839         if (!ring->ring_mem_virt) {
 840                 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
 841                 ret = -ENOMEM;
 842                 goto err_free_ops;
 843         }
 844
 845         ret = k3_ringacc_ring_cfg_sci(ring);
 846
 847         if (ret)
 848                 goto err_free_mem;
 849
 850         ring->flags |= K3_RING_FLAG_BUSY;
 851         ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
 852                         K3_RING_FLAG_SHARED : 0;
 853
 854         k3_ringacc_ring_dump(ring);
 855
 856         return 0;
 857
 858 err_free_mem:
 859         dma_free_coherent(ring->dma_dev,
 860                           ring->size * (4 << ring->elm_size),
 861                           ring->ring_mem_virt,
 862                           ring->ring_mem_dma);
 863 err_free_ops:
 864         ring->ops = NULL;
 865         ring->dma_dev = NULL;
 866 err_free_proxy:
 867         ring->proxy = NULL;
 868         return ret;
 869 }
 870 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
 871
 872 u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
 873 {
 874         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 875                 return -EINVAL;
 876
 877         return ring->size;
 878 }
 879 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
 880
 881 u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
 882 {
 883         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 884                 return -EINVAL;
 885
 886         if (!ring->state.free)
 887                 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring);
 888
 889         return ring->state.free;
 890 }
 891 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
 892
 893 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
 894 {
 895         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 896                 return -EINVAL;
 897
 898         return k3_ringacc_ring_read_occ(ring);
 899 }
 900 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
 901
 902 u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
 903 {
 904         return !k3_ringacc_ring_get_free(ring);
 905 }
 906 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
 907
 908 enum k3_ringacc_access_mode {
 909         K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
 910         K3_RINGACC_ACCESS_MODE_POP_HEAD,
 911         K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
 912         K3_RINGACC_ACCESS_MODE_POP_TAIL,
 913         K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
 914         K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
 915 };
 916
 917 #define K3_RINGACC_PROXY_MODE(x)        (((x) & 0x3) << 16)
 918 #define K3_RINGACC_PROXY_ELSIZE(x)      (((x) & 0x7) << 24)
 919 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
 920                                      enum k3_ringacc_proxy_access_mode mode)
 921 {
 922         u32 val;
 923
 924         val = ring->ring_id;
 925         val |= K3_RINGACC_PROXY_MODE(mode);
 926         val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
 927         writel(val, &ring->proxy->control);
 928         return 0;
 929 }
 930
 931 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
 932                                         enum k3_ringacc_access_mode access_mode)
 933 {
 934         void __iomem *ptr;
 935
 936         ptr = (void __iomem *)&ring->proxy->data;
 937
 938         switch (access_mode) {
 939         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 940         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 941                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
 942                 break;
 943         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 944         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 945                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
 946                 break;
 947         default:
 948                 return -EINVAL;
 949         }
 950
 951         ptr += k3_ringacc_ring_get_fifo_pos(ring);
 952
 953         switch (access_mode) {
 954         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 955         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 956                 dev_dbg(ring->parent->dev,
 957                         "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
 958                         access_mode);
 959                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 960                 ring->state.occ--;
 961                 break;
 962         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 963         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 964                 dev_dbg(ring->parent->dev,
 965                         "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
 966                         access_mode);
 967                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 968                 ring->state.free--;
 969                 break;
 970         default:
 971                 return -EINVAL;
 972         }
 973
 974         dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free,
 975                 ring->state.occ);
 976         return 0;
 977 }
 978
 979 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
 980 {
 981         return k3_ringacc_ring_access_proxy(ring, elem,
 982                                             K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 983 }
 984
 985 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
 986 {
 987         return k3_ringacc_ring_access_proxy(ring, elem,
 988                                             K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 989 }
 990
 991 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
 992 {
 993         return k3_ringacc_ring_access_proxy(ring, elem,
 994                                             K3_RINGACC_ACCESS_MODE_POP_HEAD);
 995 }
 996
 997 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
 998 {
 999         return k3_ringacc_ring_access_proxy(ring, elem,
1000                                             K3_RINGACC_ACCESS_MODE_POP_HEAD);
1001 }
1002
1003 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
1004                                      enum k3_ringacc_access_mode access_mode)
1005 {
1006         void __iomem *ptr;
1007
1008         switch (access_mode) {
1009         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1010         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1011                 ptr = (void __iomem *)&ring->fifos->head_data;
1012                 break;
1013         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1014         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1015                 ptr = (void __iomem *)&ring->fifos->tail_data;
1016                 break;
1017         default:
1018                 return -EINVAL;
1019         }
1020
1021         ptr += k3_ringacc_ring_get_fifo_pos(ring);
1022
1023         switch (access_mode) {
1024         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1025         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1026                 dev_dbg(ring->parent->dev,
1027                         "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
1028                         access_mode);
1029                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
1030                 ring->state.occ--;
1031                 break;
1032         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1033         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1034                 dev_dbg(ring->parent->dev,
1035                         "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
1036                         access_mode);
1037                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
1038                 ring->state.free--;
1039                 break;
1040         default:
1041                 return -EINVAL;
1042         }
1043
1044         dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
1045                 ring->state.free, ring->state.windex, ring->state.occ,
1046                 ring->state.rindex);
1047         return 0;
1048 }
1049
1050 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
1051 {
1052         return k3_ringacc_ring_access_io(ring, elem,
1053                                          K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
1054 }
1055
1056 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
1057 {
1058         return k3_ringacc_ring_access_io(ring, elem,
1059                                          K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
1060 }
1061
1062 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
1063 {
1064         return k3_ringacc_ring_access_io(ring, elem,
1065                                          K3_RINGACC_ACCESS_MODE_POP_HEAD);
1066 }
1067
1068 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
1069 {
1070         return k3_ringacc_ring_access_io(ring, elem,
1071                                          K3_RINGACC_ACCESS_MODE_POP_HEAD);
1072 }
1073
1074 /*
1075  * The element is 48 bits of address + ASEL bits in the ring.
1076  * ASEL is used by the DMAs and should be removed for the kernel as it is not
1077  * part of the physical memory address.
1078  */
1079 static void k3_dmaring_remove_asel_from_elem(u64 *elem)
1080 {
1081         *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1082 }
1083
1084 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem)
1085 {
1086         void *elem_ptr;
1087         u32 elem_idx;
1088
1089         /*
1090          * DMA rings: forward ring is always tied DMA channel and HW does not
1091          * maintain any state data required for POP operation and its unknown
1092          * how much elements were consumed by HW. So, to actually
1093          * do POP, the read pointer has to be recalculated every time.
1094          */
1095         ring->state.occ = k3_ringacc_ring_read_occ(ring);
1096         if (ring->state.windex >= ring->state.occ)
1097                 elem_idx = ring->state.windex - ring->state.occ;
1098         else
1099                 elem_idx = ring->size - (ring->state.occ - ring->state.windex);
1100
1101         elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx);
1102         memcpy(elem, elem_ptr, (4 << ring->elm_size));
1103         k3_dmaring_remove_asel_from_elem(elem);
1104
1105         ring->state.occ--;
1106         writel(-1, &ring->rt->db);
1107
1108         dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
1109                 __func__, ring->state.occ, ring->state.windex, elem_idx,
1110                 elem_ptr);
1111         return 0;
1112 }
1113
1114 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem)
1115 {
1116         void *elem_ptr;
1117
1118         elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1119
1120         if (ring->state.occ) {
1121                 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1122                 k3_dmaring_remove_asel_from_elem(elem);
1123
1124                 ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1125                 ring->state.occ--;
1126                 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db);
1127         } else if (ring->state.tdown_complete) {
1128                 dma_addr_t *value = elem;
1129
1130                 *value = CPPI5_TDCM_MARKER;
1131                 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db);
1132                 ring->state.tdown_complete = false;
1133         }
1134
1135         dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
1136                 __func__, ring->state.occ, ring->state.rindex, elem_ptr);
1137         return 0;
1138 }
1139
1140 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
1141 {
1142         void *elem_ptr;
1143
1144         elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
1145
1146         memcpy(elem_ptr, elem, (4 << ring->elm_size));
1147         if (ring->parent->dma_rings) {
1148                 u64 *addr = elem_ptr;
1149
1150                 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT);
1151         }
1152
1153         ring->state.windex = (ring->state.windex + 1) % ring->size;
1154         ring->state.free--;
1155         writel(1, &ring->rt->db);
1156
1157         dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
1158                 ring->state.free, ring->state.windex);
1159
1160         return 0;
1161 }
1162
1163 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
1164 {
1165         void *elem_ptr;
1166
1167         elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1168
1169         memcpy(elem, elem_ptr, (4 << ring->elm_size));
1170
1171         ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1172         ring->state.occ--;
1173         writel(-1, &ring->rt->db);
1174
1175         dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
1176                 ring->state.occ, ring->state.rindex, elem_ptr);
1177         return 0;
1178 }
1179
1180 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
1181 {
1182         int ret = -EOPNOTSUPP;
1183
1184         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1185                 return -EINVAL;
1186
1187         dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
1188                 ring->state.free, ring->state.windex);
1189
1190         if (k3_ringacc_ring_is_full(ring))
1191                 return -ENOMEM;
1192
1193         if (ring->ops && ring->ops->push_tail)
1194                 ret = ring->ops->push_tail(ring, elem);
1195
1196         return ret;
1197 }
1198 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
1199
1200 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
1201 {
1202         int ret = -EOPNOTSUPP;
1203
1204         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1205                 return -EINVAL;
1206
1207         dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
1208                 ring->state.free, ring->state.windex);
1209
1210         if (k3_ringacc_ring_is_full(ring))
1211                 return -ENOMEM;
1212
1213         if (ring->ops && ring->ops->push_head)
1214                 ret = ring->ops->push_head(ring, elem);
1215
1216         return ret;
1217 }
1218 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
1219
1220 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
1221 {
1222         int ret = -EOPNOTSUPP;
1223
1224         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1225                 return -EINVAL;
1226
1227         if (!ring->state.occ)
1228                 k3_ringacc_ring_update_occ(ring);
1229
1230         dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
1231                 ring->state.rindex);
1232
1233         if (!ring->state.occ && !ring->state.tdown_complete)
1234                 return -ENODATA;
1235
1236         if (ring->ops && ring->ops->pop_head)
1237                 ret = ring->ops->pop_head(ring, elem);
1238
1239         return ret;
1240 }
1241 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
1242
1243 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
1244 {
1245         int ret = -EOPNOTSUPP;
1246
1247         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1248                 return -EINVAL;
1249
1250         if (!ring->state.occ)
1251                 k3_ringacc_ring_update_occ(ring);
1252
1253         dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
1254                 ring->state.occ, ring->state.rindex);
1255
1256         if (!ring->state.occ)
1257                 return -ENODATA;
1258
1259         if (ring->ops && ring->ops->pop_tail)
1260                 ret = ring->ops->pop_tail(ring, elem);
1261
1262         return ret;
1263 }
1264 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
1265
1266 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
1267                                                 const char *property)
1268 {
1269         struct device_node *ringacc_np;
1270         struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
1271         struct k3_ringacc *entry;
1272
1273         ringacc_np = of_parse_phandle(np, property, 0);
1274         if (!ringacc_np)
1275                 return ERR_PTR(-ENODEV);
1276
1277         mutex_lock(&k3_ringacc_list_lock);
1278         list_for_each_entry(entry, &k3_ringacc_list, list)
1279                 if (entry->dev->of_node == ringacc_np) {
1280                         ringacc = entry;
1281                         break;
1282                 }
1283         mutex_unlock(&k3_ringacc_list_lock);
1284         of_node_put(ringacc_np);
1285
1286         return ringacc;
1287 }
1288 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
1289
1290 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
1291 {
1292         struct device_node *node = ringacc->dev->of_node;
1293         struct device *dev = ringacc->dev;
1294         struct platform_device *pdev = to_platform_device(dev);
1295         int ret;
1296
1297         if (!node) {
1298                 dev_err(dev, "device tree info unavailable\n");
1299                 return -ENODEV;
1300         }
1301
1302         ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
1303         if (ret) {
1304                 dev_err(dev, "ti,num-rings read failure %d\n", ret);
1305                 return ret;
1306         }
1307
1308         ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
1309         if (IS_ERR(ringacc->tisci)) {
1310                 ret = PTR_ERR(ringacc->tisci);
1311                 if (ret != -EPROBE_DEFER)
1312                         dev_err(dev, "ti,sci read fail %d\n", ret);
1313                 ringacc->tisci = NULL;
1314                 return ret;
1315         }
1316
1317         ret = of_property_read_u32(node, "ti,sci-dev-id",
1318                                    &ringacc->tisci_dev_id);
1319         if (ret) {
1320                 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
1321                 return ret;
1322         }
1323
1324         pdev->id = ringacc->tisci_dev_id;
1325
1326         ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
1327                                                 ringacc->tisci_dev_id,
1328                                                 "ti,sci-rm-range-gp-rings");
1329         if (IS_ERR(ringacc->rm_gp_range)) {
1330                 dev_err(dev, "Failed to allocate MSI interrupts\n");
1331                 return PTR_ERR(ringacc->rm_gp_range);
1332         }
1333
1334         return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,
1335                                                  ringacc->rm_gp_range);
1336 }
1337
1338 static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = {
1339         .dma_ring_reset_quirk = 1,
1340 };
1341
1342 static const struct soc_device_attribute k3_ringacc_socinfo[] = {
1343         { .family = "AM65X",
1344           .revision = "SR1.0",
1345           .data = &k3_ringacc_soc_data_sr1
1346         },
1347         {/* sentinel */}
1348 };
1349
1350 static int k3_ringacc_init(struct platform_device *pdev,
1351                            struct k3_ringacc *ringacc)
1352 {
1353         const struct soc_device_attribute *soc;
1354         void __iomem *base_fifo, *base_rt;
1355         struct device *dev = &pdev->dev;
1356         struct resource *res;
1357         int ret, i;
1358
1359         dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
1360                                             DOMAIN_BUS_TI_SCI_INTA_MSI);
1361         if (!dev->msi_domain) {
1362                 dev_err(dev, "Failed to get MSI domain\n");
1363                 return -EPROBE_DEFER;
1364         }
1365
1366         ret = k3_ringacc_probe_dt(ringacc);
1367         if (ret)
1368                 return ret;
1369
1370         soc = soc_device_match(k3_ringacc_socinfo);
1371         if (soc && soc->data) {
1372                 const struct k3_ringacc_soc_data *soc_data = soc->data;
1373
1374                 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk;
1375         }
1376
1377         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt");
1378         base_rt = devm_ioremap_resource(dev, res);
1379         if (IS_ERR(base_rt))
1380                 return PTR_ERR(base_rt);
1381
1382         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos");
1383         base_fifo = devm_ioremap_resource(dev, res);
1384         if (IS_ERR(base_fifo))
1385                 return PTR_ERR(base_fifo);
1386
1387         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg");
1388         ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);
1389         if (IS_ERR(ringacc->proxy_gcfg))
1390                 return PTR_ERR(ringacc->proxy_gcfg);
1391
1392         res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1393                                            "proxy_target");
1394         ringacc->proxy_target_base = devm_ioremap_resource(dev, res);
1395         if (IS_ERR(ringacc->proxy_target_base))
1396                 return PTR_ERR(ringacc->proxy_target_base);
1397
1398         ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &
1399                                      K3_RINGACC_PROXY_CFG_THREADS_MASK;
1400
1401         ringacc->rings = devm_kzalloc(dev,
1402                                       sizeof(*ringacc->rings) *
1403                                       ringacc->num_rings,
1404                                       GFP_KERNEL);
1405         ringacc->rings_inuse = devm_kcalloc(dev,
1406                                             BITS_TO_LONGS(ringacc->num_rings),
1407                                             sizeof(unsigned long), GFP_KERNEL);
1408         ringacc->proxy_inuse = devm_kcalloc(dev,
1409                                             BITS_TO_LONGS(ringacc->num_proxies),
1410                                             sizeof(unsigned long), GFP_KERNEL);
1411
1412         if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1413                 return -ENOMEM;
1414
1415         for (i = 0; i < ringacc->num_rings; i++) {
1416                 ringacc->rings[i].rt = base_rt +
1417                                        K3_RINGACC_RT_REGS_STEP * i;
1418                 ringacc->rings[i].fifos = base_fifo +
1419                                           K3_RINGACC_FIFO_REGS_STEP * i;
1420                 ringacc->rings[i].parent = ringacc;
1421                 ringacc->rings[i].ring_id = i;
1422                 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1423         }
1424
1425         ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1426
1427         dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1428                  ringacc->num_rings,
1429                  ringacc->rm_gp_range->desc[0].start,
1430                  ringacc->rm_gp_range->desc[0].num,
1431                  ringacc->tisci_dev_id);
1432         dev_info(dev, "dma-ring-reset-quirk: %s\n",
1433                  ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1434         dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1435                  readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);
1436
1437         return 0;
1438 }
1439
1440 struct ringacc_match_data {
1441         struct k3_ringacc_ops ops;
1442 };
1443
1444 static struct ringacc_match_data k3_ringacc_data = {
1445         .ops = {
1446                 .init = k3_ringacc_init,
1447         },
1448 };
1449
1450 /* Match table for of_platform binding */
1451 static const struct of_device_id k3_ringacc_of_match[] = {
1452         { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, },
1453         {},
1454 };
1455
1456 struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev,
1457                                             struct k3_ringacc_init_data *data)
1458 {
1459         struct device *dev = &pdev->dev;
1460         struct k3_ringacc *ringacc;
1461         void __iomem *base_rt;
1462         struct resource *res;
1463         int i;
1464
1465         ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1466         if (!ringacc)
1467                 return ERR_PTR(-ENOMEM);
1468
1469         ringacc->dev = dev;
1470         ringacc->dma_rings = true;
1471         ringacc->num_rings = data->num_rings;
1472         ringacc->tisci = data->tisci;
1473         ringacc->tisci_dev_id = data->tisci_dev_id;
1474
1475         mutex_init(&ringacc->req_lock);
1476
1477         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt");
1478         base_rt = devm_ioremap_resource(dev, res);
1479         if (IS_ERR(base_rt))
1480                 return ERR_CAST(base_rt);
1481
1482         ringacc->rings = devm_kzalloc(dev,
1483                                       sizeof(*ringacc->rings) *
1484                                       ringacc->num_rings * 2,
1485                                       GFP_KERNEL);
1486         ringacc->rings_inuse = devm_kcalloc(dev,
1487                                             BITS_TO_LONGS(ringacc->num_rings),
1488                                             sizeof(unsigned long), GFP_KERNEL);
1489
1490         if (!ringacc->rings || !ringacc->rings_inuse)
1491                 return ERR_PTR(-ENOMEM);
1492
1493         for (i = 0; i < ringacc->num_rings; i++) {
1494                 struct k3_ring *ring = &ringacc->rings[i];
1495
1496                 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i;
1497                 ring->parent = ringacc;
1498                 ring->ring_id = i;
1499                 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1500
1501                 ring = &ringacc->rings[ringacc->num_rings + i];
1502                 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i +
1503                            K3_DMARING_RT_REGS_REVERSE_OFS;
1504                 ring->parent = ringacc;
1505                 ring->ring_id = i;
1506                 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1507                 ring->flags = K3_RING_FLAG_REVERSE;
1508         }
1509
1510         ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1511
1512         dev_info(dev, "Number of rings: %u\n", ringacc->num_rings);
1513
1514         return ringacc;
1515 }
1516 EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init);
1517
1518 static int k3_ringacc_probe(struct platform_device *pdev)
1519 {
1520         const struct ringacc_match_data *match_data;
1521         struct device *dev = &pdev->dev;
1522         struct k3_ringacc *ringacc;
1523         int ret;
1524
1525         match_data = of_device_get_match_data(&pdev->dev);
1526         if (!match_data)
1527                 return -ENODEV;
1528
1529         ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1530         if (!ringacc)
1531                 return -ENOMEM;
1532
1533         ringacc->dev = dev;
1534         mutex_init(&ringacc->req_lock);
1535         ringacc->ops = &match_data->ops;
1536
1537         ret = ringacc->ops->init(pdev, ringacc);
1538         if (ret)
1539                 return ret;
1540
1541         dev_set_drvdata(dev, ringacc);
1542
1543         mutex_lock(&k3_ringacc_list_lock);
1544         list_add_tail(&ringacc->list, &k3_ringacc_list);
1545         mutex_unlock(&k3_ringacc_list_lock);
1546
1547         return 0;
1548 }
1549
1550 static struct platform_driver k3_ringacc_driver = {
1551         .probe          = k3_ringacc_probe,
1552         .driver         = {
1553                 .name   = "k3-ringacc",
1554                 .of_match_table = k3_ringacc_of_match,
1555                 .suppress_bind_attrs = true,
1556         },
1557 };
1558 builtin_platform_driver(k3_ringacc_driver);