drivers/soc/ti/k3-ringacc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * TI K3 NAVSS Ring Accelerator subsystem driver
   4  *
   5  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   6  */
   7
   8 #include <linux/dma-mapping.h>
   9 #include <linux/io.h>
  10 #include <linux/init.h>
  11 #include <linux/of.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/soc/ti/k3-ringacc.h>
  14 #include <linux/soc/ti/ti_sci_protocol.h>
  15 #include <linux/soc/ti/ti_sci_inta_msi.h>
  16 #include <linux/of_irq.h>
  17 #include <linux/irqdomain.h>
  18
  19 static LIST_HEAD(k3_ringacc_list);
  20 static DEFINE_MUTEX(k3_ringacc_list_lock);
  21
  22 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK             GENMASK(19, 0)
  23
  24 /**
  25  * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
  26  *
  27  * @resv_16: Reserved
  28  * @db: Ring Doorbell Register
  29  * @resv_4: Reserved
  30  * @occ: Ring Occupancy Register
  31  * @indx: Ring Current Index Register
  32  * @hwocc: Ring Hardware Occupancy Register
  33  * @hwindx: Ring Hardware Current Index Register
  34  */
  35 struct k3_ring_rt_regs {
  36         u32     resv_16[4];
  37         u32     db;
  38         u32     resv_4[1];
  39         u32     occ;
  40         u32     indx;
  41         u32     hwocc;
  42         u32     hwindx;
  43 };
  44
  45 #define K3_RINGACC_RT_REGS_STEP 0x1000
  46
  47 /**
  48  * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
  49  *
  50  * @head_data: Ring Head Entry Data Registers
  51  * @tail_data: Ring Tail Entry Data Registers
  52  * @peek_head_data: Ring Peek Head Entry Data Regs
  53  * @peek_tail_data: Ring Peek Tail Entry Data Regs
  54  */
  55 struct k3_ring_fifo_regs {
  56         u32     head_data[128];
  57         u32     tail_data[128];
  58         u32     peek_head_data[128];
  59         u32     peek_tail_data[128];
  60 };
  61
  62 /**
  63  * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
  64  *
  65  * @revision: Revision Register
  66  * @config: Config Register
  67  */
  68 struct k3_ringacc_proxy_gcfg_regs {
  69         u32     revision;
  70         u32     config;
  71 };
  72
  73 #define K3_RINGACC_PROXY_CFG_THREADS_MASK               GENMASK(15, 0)
  74
  75 /**
  76  * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region
  77  *
  78  * @control: Proxy Control Register
  79  * @status: Proxy Status Register
  80  * @resv_512: Reserved
  81  * @data: Proxy Data Register
  82  */
  83 struct k3_ringacc_proxy_target_regs {
  84         u32     control;
  85         u32     status;
  86         u8      resv_512[504];
  87         u32     data[128];
  88 };
  89
  90 #define K3_RINGACC_PROXY_TARGET_STEP    0x1000
  91 #define K3_RINGACC_PROXY_NOT_USED       (-1)
  92
  93 enum k3_ringacc_proxy_access_mode {
  94         PROXY_ACCESS_MODE_HEAD = 0,
  95         PROXY_ACCESS_MODE_TAIL = 1,
  96         PROXY_ACCESS_MODE_PEEK_HEAD = 2,
  97         PROXY_ACCESS_MODE_PEEK_TAIL = 3,
  98 };
  99
 100 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
 101 #define K3_RINGACC_FIFO_REGS_STEP       0x1000
 102 #define K3_RINGACC_MAX_DB_RING_CNT    (127U)
 103
 104 struct k3_ring_ops {
 105         int (*push_tail)(struct k3_ring *ring, void *elm);
 106         int (*push_head)(struct k3_ring *ring, void *elm);
 107         int (*pop_tail)(struct k3_ring *ring, void *elm);
 108         int (*pop_head)(struct k3_ring *ring, void *elm);
 109 };
 110
 111 /**
 112  * struct k3_ring_state - Internal state tracking structure
 113  *
 114  * @free: Number of free entries
 115  * @occ: Occupancy
 116  * @windex: Write index
 117  * @rindex: Read index
 118  */
 119 struct k3_ring_state {
 120         u32 free;
 121         u32 occ;
 122         u32 windex;
 123         u32 rindex;
 124 };
 125
 126 /**
 127  * struct k3_ring - RA Ring descriptor
 128  *
 129  * @rt: Ring control/status registers
 130  * @fifos: Ring queues registers
 131  * @proxy: Ring Proxy Datapath registers
 132  * @ring_mem_dma: Ring buffer dma address
 133  * @ring_mem_virt: Ring buffer virt address
 134  * @ops: Ring operations
 135  * @size: Ring size in elements
 136  * @elm_size: Size of the ring element
 137  * @mode: Ring mode
 138  * @flags: flags
 139  * @ring_id: Ring Id
 140  * @parent: Pointer on struct @k3_ringacc
 141  * @use_count: Use count for shared rings
 142  * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
 143  */
 144 struct k3_ring {
 145         struct k3_ring_rt_regs __iomem *rt;
 146         struct k3_ring_fifo_regs __iomem *fifos;
 147         struct k3_ringacc_proxy_target_regs  __iomem *proxy;
 148         dma_addr_t      ring_mem_dma;
 149         void            *ring_mem_virt;
 150         struct k3_ring_ops *ops;
 151         u32             size;
 152         enum k3_ring_size elm_size;
 153         enum k3_ring_mode mode;
 154         u32             flags;
 155 #define K3_RING_FLAG_BUSY       BIT(1)
 156 #define K3_RING_FLAG_SHARED     BIT(2)
 157         struct k3_ring_state state;
 158         u32             ring_id;
 159         struct k3_ringacc       *parent;
 160         u32             use_count;
 161         int             proxy_id;
 162 };
 163
 164 struct k3_ringacc_ops {
 165         int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc);
 166 };
 167
 168 /**
 169  * struct k3_ringacc - Rings accelerator descriptor
 170  *
 171  * @dev: pointer on RA device
 172  * @proxy_gcfg: RA proxy global config registers
 173  * @proxy_target_base: RA proxy datapath region
 174  * @num_rings: number of ring in RA
 175  * @rings_inuse: bitfield for ring usage tracking
 176  * @rm_gp_range: general purpose rings range from tisci
 177  * @dma_ring_reset_quirk: DMA reset w/a enable
 178  * @num_proxies: number of RA proxies
 179  * @proxy_inuse: bitfield for proxy usage tracking
 180  * @rings: array of rings descriptors (struct @k3_ring)
 181  * @list: list of RAs in the system
 182  * @req_lock: protect rings allocation
 183  * @tisci: pointer ti-sci handle
 184  * @tisci_ring_ops: ti-sci rings ops
 185  * @tisci_dev_id: ti-sci device id
 186  * @ops: SoC specific ringacc operation
 187  */
 188 struct k3_ringacc {
 189         struct device *dev;
 190         struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
 191         void __iomem *proxy_target_base;
 192         u32 num_rings; /* number of rings in Ringacc module */
 193         unsigned long *rings_inuse;
 194         struct ti_sci_resource *rm_gp_range;
 195
 196         bool dma_ring_reset_quirk;
 197         u32 num_proxies;
 198         unsigned long *proxy_inuse;
 199
 200         struct k3_ring *rings;
 201         struct list_head list;
 202         struct mutex req_lock; /* protect rings allocation */
 203
 204         const struct ti_sci_handle *tisci;
 205         const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
 206         u32 tisci_dev_id;
 207
 208         const struct k3_ringacc_ops *ops;
 209 };
 210
 211 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
 212 {
 213         return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
 214                (4 << ring->elm_size);
 215 }
 216
 217 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
 218 {
 219         return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
 220 }
 221
 222 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
 223 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
 224
 225 static struct k3_ring_ops k3_ring_mode_ring_ops = {
 226                 .push_tail = k3_ringacc_ring_push_mem,
 227                 .pop_head = k3_ringacc_ring_pop_mem,
 228 };
 229
 230 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
 231 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
 232 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
 233 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
 234
 235 static struct k3_ring_ops k3_ring_mode_msg_ops = {
 236                 .push_tail = k3_ringacc_ring_push_io,
 237                 .push_head = k3_ringacc_ring_push_head_io,
 238                 .pop_tail = k3_ringacc_ring_pop_tail_io,
 239                 .pop_head = k3_ringacc_ring_pop_io,
 240 };
 241
 242 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
 243 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
 244 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
 245 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
 246
 247 static struct k3_ring_ops k3_ring_mode_proxy_ops = {
 248                 .push_tail = k3_ringacc_ring_push_tail_proxy,
 249                 .push_head = k3_ringacc_ring_push_head_proxy,
 250                 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
 251                 .pop_head = k3_ringacc_ring_pop_head_proxy,
 252 };
 253
 254 static void k3_ringacc_ring_dump(struct k3_ring *ring)
 255 {
 256         struct device *dev = ring->parent->dev;
 257
 258         dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
 259         dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
 260                 &ring->ring_mem_dma);
 261         dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
 262                 ring->elm_size, ring->size, ring->mode, ring->proxy_id);
 263         dev_dbg(dev, "dump flags %08X\n", ring->flags);
 264
 265         dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
 266         dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
 267         dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
 268         dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
 269         dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
 270
 271         if (ring->ring_mem_virt)
 272                 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE,
 273                                      16, 1, ring->ring_mem_virt, 16 * 8, false);
 274 }
 275
 276 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
 277                                         int id, u32 flags)
 278 {
 279         int proxy_id = K3_RINGACC_PROXY_NOT_USED;
 280
 281         mutex_lock(&ringacc->req_lock);
 282
 283         if (id == K3_RINGACC_RING_ID_ANY) {
 284                 /* Request for any general purpose ring */
 285                 struct ti_sci_resource_desc *gp_rings =
 286                                                 &ringacc->rm_gp_range->desc[0];
 287                 unsigned long size;
 288
 289                 size = gp_rings->start + gp_rings->num;
 290                 id = find_next_zero_bit(ringacc->rings_inuse, size,
 291                                         gp_rings->start);
 292                 if (id == size)
 293                         goto error;
 294         } else if (id < 0) {
 295                 goto error;
 296         }
 297
 298         if (test_bit(id, ringacc->rings_inuse) &&
 299             !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
 300                 goto error;
 301         else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
 302                 goto out;
 303
 304         if (flags & K3_RINGACC_RING_USE_PROXY) {
 305                 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
 306                                               ringacc->num_proxies, 0);
 307                 if (proxy_id == ringacc->num_proxies)
 308                         goto error;
 309         }
 310
 311         if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 312                 set_bit(proxy_id, ringacc->proxy_inuse);
 313                 ringacc->rings[id].proxy_id = proxy_id;
 314                 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
 315                         proxy_id);
 316         } else {
 317                 dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
 318         }
 319
 320         set_bit(id, ringacc->rings_inuse);
 321 out:
 322         ringacc->rings[id].use_count++;
 323         mutex_unlock(&ringacc->req_lock);
 324         return &ringacc->rings[id];
 325
 326 error:
 327         mutex_unlock(&ringacc->req_lock);
 328         return NULL;
 329 }
 330 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
 331
 332 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
 333                                   int fwd_id, int compl_id,
 334                                   struct k3_ring **fwd_ring,
 335                                   struct k3_ring **compl_ring)
 336 {
 337         int ret = 0;
 338
 339         if (!fwd_ring || !compl_ring)
 340                 return -EINVAL;
 341
 342         *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
 343         if (!(*fwd_ring))
 344                 return -ENODEV;
 345
 346         *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0);
 347         if (!(*compl_ring)) {
 348                 k3_ringacc_ring_free(*fwd_ring);
 349                 ret = -ENODEV;
 350         }
 351
 352         return ret;
 353 }
 354 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair);
 355
 356 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
 357 {
 358         struct k3_ringacc *ringacc = ring->parent;
 359         int ret;
 360
 361         ret = ringacc->tisci_ring_ops->config(
 362                         ringacc->tisci,
 363                         TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
 364                         ringacc->tisci_dev_id,
 365                         ring->ring_id,
 366                         0,
 367                         0,
 368                         ring->size,
 369                         0,
 370                         0,
 371                         0);
 372         if (ret)
 373                 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
 374                         ret, ring->ring_id);
 375 }
 376
 377 void k3_ringacc_ring_reset(struct k3_ring *ring)
 378 {
 379         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 380                 return;
 381
 382         memset(&ring->state, 0, sizeof(ring->state));
 383
 384         k3_ringacc_ring_reset_sci(ring);
 385 }
 386 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
 387
 388 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
 389                                                enum k3_ring_mode mode)
 390 {
 391         struct k3_ringacc *ringacc = ring->parent;
 392         int ret;
 393
 394         ret = ringacc->tisci_ring_ops->config(
 395                         ringacc->tisci,
 396                         TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
 397                         ringacc->tisci_dev_id,
 398                         ring->ring_id,
 399                         0,
 400                         0,
 401                         0,
 402                         mode,
 403                         0,
 404                         0);
 405         if (ret)
 406                 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
 407                         ret, ring->ring_id);
 408 }
 409
 410 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
 411 {
 412         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 413                 return;
 414
 415         if (!ring->parent->dma_ring_reset_quirk)
 416                 goto reset;
 417
 418         if (!occ)
 419                 occ = readl(&ring->rt->occ);
 420
 421         if (occ) {
 422                 u32 db_ring_cnt, db_ring_cnt_cur;
 423
 424                 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
 425                         ring->ring_id, occ);
 426                 /* TI-SCI ring reset */
 427                 k3_ringacc_ring_reset_sci(ring);
 428
 429                 /*
 430                  * Setup the ring in ring/doorbell mode (if not already in this
 431                  * mode)
 432                  */
 433                 if (ring->mode != K3_RINGACC_RING_MODE_RING)
 434                         k3_ringacc_ring_reconfig_qmode_sci(
 435                                         ring, K3_RINGACC_RING_MODE_RING);
 436                 /*
 437                  * Ring the doorbell 2**22 – ringOcc times.
 438                  * This will wrap the internal UDMAP ring state occupancy
 439                  * counter (which is 21-bits wide) to 0.
 440                  */
 441                 db_ring_cnt = (1U << 22) - occ;
 442
 443                 while (db_ring_cnt != 0) {
 444                         /*
 445                          * Ring the doorbell with the maximum count each
 446                          * iteration if possible to minimize the total
 447                          * of writes
 448                          */
 449                         if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
 450                                 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
 451                         else
 452                                 db_ring_cnt_cur = db_ring_cnt;
 453
 454                         writel(db_ring_cnt_cur, &ring->rt->db);
 455                         db_ring_cnt -= db_ring_cnt_cur;
 456                 }
 457
 458                 /* Restore the original ring mode (if not ring mode) */
 459                 if (ring->mode != K3_RINGACC_RING_MODE_RING)
 460                         k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
 461         }
 462
 463 reset:
 464         /* Reset the ring */
 465         k3_ringacc_ring_reset(ring);
 466 }
 467 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
 468
 469 static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
 470 {
 471         struct k3_ringacc *ringacc = ring->parent;
 472         int ret;
 473
 474         ret = ringacc->tisci_ring_ops->config(
 475                         ringacc->tisci,
 476                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 477                         ringacc->tisci_dev_id,
 478                         ring->ring_id,
 479                         0,
 480                         0,
 481                         0,
 482                         0,
 483                         0,
 484                         0);
 485         if (ret)
 486                 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
 487                         ret, ring->ring_id);
 488 }
 489
 490 int k3_ringacc_ring_free(struct k3_ring *ring)
 491 {
 492         struct k3_ringacc *ringacc;
 493
 494         if (!ring)
 495                 return -EINVAL;
 496
 497         ringacc = ring->parent;
 498
 499         dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
 500
 501         if (!test_bit(ring->ring_id, ringacc->rings_inuse))
 502                 return -EINVAL;
 503
 504         mutex_lock(&ringacc->req_lock);
 505
 506         if (--ring->use_count)
 507                 goto out;
 508
 509         if (!(ring->flags & K3_RING_FLAG_BUSY))
 510                 goto no_init;
 511
 512         k3_ringacc_ring_free_sci(ring);
 513
 514         dma_free_coherent(ringacc->dev,
 515                           ring->size * (4 << ring->elm_size),
 516                           ring->ring_mem_virt, ring->ring_mem_dma);
 517         ring->flags = 0;
 518         ring->ops = NULL;
 519         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
 520                 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
 521                 ring->proxy = NULL;
 522                 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
 523         }
 524
 525 no_init:
 526         clear_bit(ring->ring_id, ringacc->rings_inuse);
 527
 528 out:
 529         mutex_unlock(&ringacc->req_lock);
 530         return 0;
 531 }
 532 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
 533
 534 u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
 535 {
 536         if (!ring)
 537                 return -EINVAL;
 538
 539         return ring->ring_id;
 540 }
 541 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
 542
 543 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
 544 {
 545         if (!ring)
 546                 return -EINVAL;
 547
 548         return ring->parent->tisci_dev_id;
 549 }
 550 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
 551
 552 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
 553 {
 554         int irq_num;
 555
 556         if (!ring)
 557                 return -EINVAL;
 558
 559         irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id);
 560         if (irq_num <= 0)
 561                 irq_num = -EINVAL;
 562         return irq_num;
 563 }
 564 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num);
 565
 566 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
 567 {
 568         struct k3_ringacc *ringacc = ring->parent;
 569         u32 ring_idx;
 570         int ret;
 571
 572         if (!ringacc->tisci)
 573                 return -EINVAL;
 574
 575         ring_idx = ring->ring_id;
 576         ret = ringacc->tisci_ring_ops->config(
 577                         ringacc->tisci,
 578                         TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
 579                         ringacc->tisci_dev_id,
 580                         ring_idx,
 581                         lower_32_bits(ring->ring_mem_dma),
 582                         upper_32_bits(ring->ring_mem_dma),
 583                         ring->size,
 584                         ring->mode,
 585                         ring->elm_size,
 586                         0);
 587         if (ret)
 588                 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
 589                         ret, ring_idx);
 590
 591         return ret;
 592 }
 593
 594 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
 595 {
 596         struct k3_ringacc *ringacc = ring->parent;
 597         int ret = 0;
 598
 599         if (!ring || !cfg)
 600                 return -EINVAL;
 601         if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
 602             cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
 603             cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
 604             !test_bit(ring->ring_id, ringacc->rings_inuse))
 605                 return -EINVAL;
 606
 607         if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE &&
 608             ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
 609             cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) {
 610                 dev_err(ringacc->dev,
 611                         "Message mode must use proxy for %u element size\n",
 612                         4 << ring->elm_size);
 613                 return -EINVAL;
 614         }
 615
 616         /*
 617          * In case of shared ring only the first user (master user) can
 618          * configure the ring. The sequence should be by the client:
 619          * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user
 620          * k3_ringacc_ring_cfg(ring, cfg); # master configuration
 621          * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
 622          * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
 623          */
 624         if (ring->use_count != 1)
 625                 return 0;
 626
 627         ring->size = cfg->size;
 628         ring->elm_size = cfg->elm_size;
 629         ring->mode = cfg->mode;
 630         memset(&ring->state, 0, sizeof(ring->state));
 631
 632         if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
 633                 ring->proxy = ringacc->proxy_target_base +
 634                               ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
 635
 636         switch (ring->mode) {
 637         case K3_RINGACC_RING_MODE_RING:
 638                 ring->ops = &k3_ring_mode_ring_ops;
 639                 break;
 640         case K3_RINGACC_RING_MODE_MESSAGE:
 641                 if (ring->proxy)
 642                         ring->ops = &k3_ring_mode_proxy_ops;
 643                 else
 644                         ring->ops = &k3_ring_mode_msg_ops;
 645                 break;
 646         default:
 647                 ring->ops = NULL;
 648                 ret = -EINVAL;
 649                 goto err_free_proxy;
 650         };
 651
 652         ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev,
 653                                         ring->size * (4 << ring->elm_size),
 654                                         &ring->ring_mem_dma, GFP_KERNEL);
 655         if (!ring->ring_mem_virt) {
 656                 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
 657                 ret = -ENOMEM;
 658                 goto err_free_ops;
 659         }
 660
 661         ret = k3_ringacc_ring_cfg_sci(ring);
 662
 663         if (ret)
 664                 goto err_free_mem;
 665
 666         ring->flags |= K3_RING_FLAG_BUSY;
 667         ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
 668                         K3_RING_FLAG_SHARED : 0;
 669
 670         k3_ringacc_ring_dump(ring);
 671
 672         return 0;
 673
 674 err_free_mem:
 675         dma_free_coherent(ringacc->dev,
 676                           ring->size * (4 << ring->elm_size),
 677                           ring->ring_mem_virt,
 678                           ring->ring_mem_dma);
 679 err_free_ops:
 680         ring->ops = NULL;
 681 err_free_proxy:
 682         ring->proxy = NULL;
 683         return ret;
 684 }
 685 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
 686
 687 u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
 688 {
 689         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 690                 return -EINVAL;
 691
 692         return ring->size;
 693 }
 694 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
 695
 696 u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
 697 {
 698         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 699                 return -EINVAL;
 700
 701         if (!ring->state.free)
 702                 ring->state.free = ring->size - readl(&ring->rt->occ);
 703
 704         return ring->state.free;
 705 }
 706 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
 707
 708 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
 709 {
 710         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 711                 return -EINVAL;
 712
 713         return readl(&ring->rt->occ);
 714 }
 715 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
 716
 717 u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
 718 {
 719         return !k3_ringacc_ring_get_free(ring);
 720 }
 721 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
 722
 723 enum k3_ringacc_access_mode {
 724         K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
 725         K3_RINGACC_ACCESS_MODE_POP_HEAD,
 726         K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
 727         K3_RINGACC_ACCESS_MODE_POP_TAIL,
 728         K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
 729         K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
 730 };
 731
 732 #define K3_RINGACC_PROXY_MODE(x)        (((x) & 0x3) << 16)
 733 #define K3_RINGACC_PROXY_ELSIZE(x)      (((x) & 0x7) << 24)
 734 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
 735                                      enum k3_ringacc_proxy_access_mode mode)
 736 {
 737         u32 val;
 738
 739         val = ring->ring_id;
 740         val |= K3_RINGACC_PROXY_MODE(mode);
 741         val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
 742         writel(val, &ring->proxy->control);
 743         return 0;
 744 }
 745
 746 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
 747                                         enum k3_ringacc_access_mode access_mode)
 748 {
 749         void __iomem *ptr;
 750
 751         ptr = (void __iomem *)&ring->proxy->data;
 752
 753         switch (access_mode) {
 754         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 755         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 756                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
 757                 break;
 758         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 759         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 760                 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
 761                 break;
 762         default:
 763                 return -EINVAL;
 764         }
 765
 766         ptr += k3_ringacc_ring_get_fifo_pos(ring);
 767
 768         switch (access_mode) {
 769         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 770         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 771                 dev_dbg(ring->parent->dev,
 772                         "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
 773                         access_mode);
 774                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 775                 ring->state.occ--;
 776                 break;
 777         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 778         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 779                 dev_dbg(ring->parent->dev,
 780                         "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
 781                         access_mode);
 782                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 783                 ring->state.free--;
 784                 break;
 785         default:
 786                 return -EINVAL;
 787         }
 788
 789         dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free,
 790                 ring->state.occ);
 791         return 0;
 792 }
 793
 794 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
 795 {
 796         return k3_ringacc_ring_access_proxy(ring, elem,
 797                                             K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 798 }
 799
 800 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
 801 {
 802         return k3_ringacc_ring_access_proxy(ring, elem,
 803                                             K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 804 }
 805
 806 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
 807 {
 808         return k3_ringacc_ring_access_proxy(ring, elem,
 809                                             K3_RINGACC_ACCESS_MODE_POP_HEAD);
 810 }
 811
 812 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
 813 {
 814         return k3_ringacc_ring_access_proxy(ring, elem,
 815                                             K3_RINGACC_ACCESS_MODE_POP_HEAD);
 816 }
 817
 818 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
 819                                      enum k3_ringacc_access_mode access_mode)
 820 {
 821         void __iomem *ptr;
 822
 823         switch (access_mode) {
 824         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 825         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 826                 ptr = (void __iomem *)&ring->fifos->head_data;
 827                 break;
 828         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 829         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 830                 ptr = (void __iomem *)&ring->fifos->tail_data;
 831                 break;
 832         default:
 833                 return -EINVAL;
 834         }
 835
 836         ptr += k3_ringacc_ring_get_fifo_pos(ring);
 837
 838         switch (access_mode) {
 839         case K3_RINGACC_ACCESS_MODE_POP_HEAD:
 840         case K3_RINGACC_ACCESS_MODE_POP_TAIL:
 841                 dev_dbg(ring->parent->dev,
 842                         "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
 843                         access_mode);
 844                 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
 845                 ring->state.occ--;
 846                 break;
 847         case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
 848         case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
 849                 dev_dbg(ring->parent->dev,
 850                         "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
 851                         access_mode);
 852                 memcpy_toio(ptr, elem, (4 << ring->elm_size));
 853                 ring->state.free--;
 854                 break;
 855         default:
 856                 return -EINVAL;
 857         }
 858
 859         dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
 860                 ring->state.free, ring->state.windex, ring->state.occ,
 861                 ring->state.rindex);
 862         return 0;
 863 }
 864
 865 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
 866 {
 867         return k3_ringacc_ring_access_io(ring, elem,
 868                                          K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
 869 }
 870
 871 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
 872 {
 873         return k3_ringacc_ring_access_io(ring, elem,
 874                                          K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
 875 }
 876
 877 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
 878 {
 879         return k3_ringacc_ring_access_io(ring, elem,
 880                                          K3_RINGACC_ACCESS_MODE_POP_HEAD);
 881 }
 882
 883 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
 884 {
 885         return k3_ringacc_ring_access_io(ring, elem,
 886                                          K3_RINGACC_ACCESS_MODE_POP_HEAD);
 887 }
 888
 889 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
 890 {
 891         void *elem_ptr;
 892
 893         elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
 894
 895         memcpy(elem_ptr, elem, (4 << ring->elm_size));
 896
 897         ring->state.windex = (ring->state.windex + 1) % ring->size;
 898         ring->state.free--;
 899         writel(1, &ring->rt->db);
 900
 901         dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
 902                 ring->state.free, ring->state.windex);
 903
 904         return 0;
 905 }
 906
 907 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
 908 {
 909         void *elem_ptr;
 910
 911         elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
 912
 913         memcpy(elem, elem_ptr, (4 << ring->elm_size));
 914
 915         ring->state.rindex = (ring->state.rindex + 1) % ring->size;
 916         ring->state.occ--;
 917         writel(-1, &ring->rt->db);
 918
 919         dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
 920                 ring->state.occ, ring->state.rindex, elem_ptr);
 921         return 0;
 922 }
 923
 924 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
 925 {
 926         int ret = -EOPNOTSUPP;
 927
 928         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 929                 return -EINVAL;
 930
 931         dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
 932                 ring->state.free, ring->state.windex);
 933
 934         if (k3_ringacc_ring_is_full(ring))
 935                 return -ENOMEM;
 936
 937         if (ring->ops && ring->ops->push_tail)
 938                 ret = ring->ops->push_tail(ring, elem);
 939
 940         return ret;
 941 }
 942 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
 943
 944 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
 945 {
 946         int ret = -EOPNOTSUPP;
 947
 948         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 949                 return -EINVAL;
 950
 951         dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
 952                 ring->state.free, ring->state.windex);
 953
 954         if (k3_ringacc_ring_is_full(ring))
 955                 return -ENOMEM;
 956
 957         if (ring->ops && ring->ops->push_head)
 958                 ret = ring->ops->push_head(ring, elem);
 959
 960         return ret;
 961 }
 962 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
 963
 964 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
 965 {
 966         int ret = -EOPNOTSUPP;
 967
 968         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 969                 return -EINVAL;
 970
 971         if (!ring->state.occ)
 972                 ring->state.occ = k3_ringacc_ring_get_occ(ring);
 973
 974         dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
 975                 ring->state.rindex);
 976
 977         if (!ring->state.occ)
 978                 return -ENODATA;
 979
 980         if (ring->ops && ring->ops->pop_head)
 981                 ret = ring->ops->pop_head(ring, elem);
 982
 983         return ret;
 984 }
 985 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
 986
 987 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
 988 {
 989         int ret = -EOPNOTSUPP;
 990
 991         if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
 992                 return -EINVAL;
 993
 994         if (!ring->state.occ)
 995                 ring->state.occ = k3_ringacc_ring_get_occ(ring);
 996
 997         dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
 998                 ring->state.occ, ring->state.rindex);
 999
1000         if (!ring->state.occ)
1001                 return -ENODATA;
1002
1003         if (ring->ops && ring->ops->pop_tail)
1004                 ret = ring->ops->pop_tail(ring, elem);
1005
1006         return ret;
1007 }
1008 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
1009
1010 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
1011                                                 const char *property)
1012 {
1013         struct device_node *ringacc_np;
1014         struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
1015         struct k3_ringacc *entry;
1016
1017         ringacc_np = of_parse_phandle(np, property, 0);
1018         if (!ringacc_np)
1019                 return ERR_PTR(-ENODEV);
1020
1021         mutex_lock(&k3_ringacc_list_lock);
1022         list_for_each_entry(entry, &k3_ringacc_list, list)
1023                 if (entry->dev->of_node == ringacc_np) {
1024                         ringacc = entry;
1025                         break;
1026                 }
1027         mutex_unlock(&k3_ringacc_list_lock);
1028         of_node_put(ringacc_np);
1029
1030         return ringacc;
1031 }
1032 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
1033
1034 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
1035 {
1036         struct device_node *node = ringacc->dev->of_node;
1037         struct device *dev = ringacc->dev;
1038         struct platform_device *pdev = to_platform_device(dev);
1039         int ret;
1040
1041         if (!node) {
1042                 dev_err(dev, "device tree info unavailable\n");
1043                 return -ENODEV;
1044         }
1045
1046         ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
1047         if (ret) {
1048                 dev_err(dev, "ti,num-rings read failure %d\n", ret);
1049                 return ret;
1050         }
1051
1052         ringacc->dma_ring_reset_quirk =
1053                         of_property_read_bool(node, "ti,dma-ring-reset-quirk");
1054
1055         ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
1056         if (IS_ERR(ringacc->tisci)) {
1057                 ret = PTR_ERR(ringacc->tisci);
1058                 if (ret != -EPROBE_DEFER)
1059                         dev_err(dev, "ti,sci read fail %d\n", ret);
1060                 ringacc->tisci = NULL;
1061                 return ret;
1062         }
1063
1064         ret = of_property_read_u32(node, "ti,sci-dev-id",
1065                                    &ringacc->tisci_dev_id);
1066         if (ret) {
1067                 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
1068                 return ret;
1069         }
1070
1071         pdev->id = ringacc->tisci_dev_id;
1072
1073         ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
1074                                                 ringacc->tisci_dev_id,
1075                                                 "ti,sci-rm-range-gp-rings");
1076         if (IS_ERR(ringacc->rm_gp_range)) {
1077                 dev_err(dev, "Failed to allocate MSI interrupts\n");
1078                 return PTR_ERR(ringacc->rm_gp_range);
1079         }
1080
1081         return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,
1082                                                  ringacc->rm_gp_range);
1083 }
1084
1085 static int k3_ringacc_init(struct platform_device *pdev,
1086                            struct k3_ringacc *ringacc)
1087 {
1088         void __iomem *base_fifo, *base_rt;
1089         struct device *dev = &pdev->dev;
1090         struct resource *res;
1091         int ret, i;
1092
1093         dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
1094                                             DOMAIN_BUS_TI_SCI_INTA_MSI);
1095         if (!dev->msi_domain) {
1096                 dev_err(dev, "Failed to get MSI domain\n");
1097                 return -EPROBE_DEFER;
1098         }
1099
1100         ret = k3_ringacc_probe_dt(ringacc);
1101         if (ret)
1102                 return ret;
1103
1104         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt");
1105         base_rt = devm_ioremap_resource(dev, res);
1106         if (IS_ERR(base_rt))
1107                 return PTR_ERR(base_rt);
1108
1109         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos");
1110         base_fifo = devm_ioremap_resource(dev, res);
1111         if (IS_ERR(base_fifo))
1112                 return PTR_ERR(base_fifo);
1113
1114         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg");
1115         ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);
1116         if (IS_ERR(ringacc->proxy_gcfg))
1117                 return PTR_ERR(ringacc->proxy_gcfg);
1118
1119         res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1120                                            "proxy_target");
1121         ringacc->proxy_target_base = devm_ioremap_resource(dev, res);
1122         if (IS_ERR(ringacc->proxy_target_base))
1123                 return PTR_ERR(ringacc->proxy_target_base);
1124
1125         ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &
1126                                      K3_RINGACC_PROXY_CFG_THREADS_MASK;
1127
1128         ringacc->rings = devm_kzalloc(dev,
1129                                       sizeof(*ringacc->rings) *
1130                                       ringacc->num_rings,
1131                                       GFP_KERNEL);
1132         ringacc->rings_inuse = devm_kcalloc(dev,
1133                                             BITS_TO_LONGS(ringacc->num_rings),
1134                                             sizeof(unsigned long), GFP_KERNEL);
1135         ringacc->proxy_inuse = devm_kcalloc(dev,
1136                                             BITS_TO_LONGS(ringacc->num_proxies),
1137                                             sizeof(unsigned long), GFP_KERNEL);
1138
1139         if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1140                 return -ENOMEM;
1141
1142         for (i = 0; i < ringacc->num_rings; i++) {
1143                 ringacc->rings[i].rt = base_rt +
1144                                        K3_RINGACC_RT_REGS_STEP * i;
1145                 ringacc->rings[i].fifos = base_fifo +
1146                                           K3_RINGACC_FIFO_REGS_STEP * i;
1147                 ringacc->rings[i].parent = ringacc;
1148                 ringacc->rings[i].ring_id = i;
1149                 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1150         }
1151
1152         ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1153
1154         dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1155                  ringacc->num_rings,
1156                  ringacc->rm_gp_range->desc[0].start,
1157                  ringacc->rm_gp_range->desc[0].num,
1158                  ringacc->tisci_dev_id);
1159         dev_info(dev, "dma-ring-reset-quirk: %s\n",
1160                  ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1161         dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1162                  readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);
1163
1164         return 0;
1165 }
1166
1167 struct ringacc_match_data {
1168         struct k3_ringacc_ops ops;
1169 };
1170
1171 static struct ringacc_match_data k3_ringacc_data = {
1172         .ops = {
1173                 .init = k3_ringacc_init,
1174         },
1175 };
1176
1177 /* Match table for of_platform binding */
1178 static const struct of_device_id k3_ringacc_of_match[] = {
1179         { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, },
1180         {},
1181 };
1182
1183 static int k3_ringacc_probe(struct platform_device *pdev)
1184 {
1185         const struct ringacc_match_data *match_data;
1186         const struct of_device_id *match;
1187         struct device *dev = &pdev->dev;
1188         struct k3_ringacc *ringacc;
1189         int ret;
1190
1191         match = of_match_node(k3_ringacc_of_match, dev->of_node);
1192         if (!match)
1193                 return -ENODEV;
1194         match_data = match->data;
1195
1196         ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1197         if (!ringacc)
1198                 return -ENOMEM;
1199
1200         ringacc->dev = dev;
1201         mutex_init(&ringacc->req_lock);
1202         ringacc->ops = &match_data->ops;
1203
1204         ret = ringacc->ops->init(pdev, ringacc);
1205         if (ret)
1206                 return ret;
1207
1208         dev_set_drvdata(dev, ringacc);
1209
1210         mutex_lock(&k3_ringacc_list_lock);
1211         list_add_tail(&ringacc->list, &k3_ringacc_list);
1212         mutex_unlock(&k3_ringacc_list_lock);
1213
1214         return 0;
1215 }
1216
1217 static struct platform_driver k3_ringacc_driver = {
1218         .probe          = k3_ringacc_probe,
1219         .driver         = {
1220                 .name   = "k3-ringacc",
1221                 .of_match_table = k3_ringacc_of_match,
1222                 .suppress_bind_attrs = true,
1223         },
1224 };
1225 builtin_platform_driver(k3_ringacc_driver);