drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright 2019 NXP
   3
   4 #include <linux/init.h>
   5 #include <linux/module.h>
   6 #include <linux/dmapool.h>
   7 #include <linux/of_irq.h>
   8 #include <linux/iommu.h>
   9 #include <linux/sys_soc.h>
  10 #include <linux/fsl/mc.h>
  11 #include <soc/fsl/dpaa2-io.h>
  12
  13 #include "../virt-dma.h"
  14 #include "dpdmai.h"
  15 #include "dpaa2-qdma.h"
  16
  17 static bool smmu_disable = true;
  18
  19 static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan)
  20 {
  21         return container_of(chan, struct dpaa2_qdma_chan, vchan.chan);
  22 }
  23
  24 static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
  25 {
  26         return container_of(vd, struct dpaa2_qdma_comp, vdesc);
  27 }
  28
  29 static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan)
  30 {
  31         struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
  32         struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
  33         struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev;
  34
  35         dpaa2_chan->fd_pool = dma_pool_create("fd_pool", dev,
  36                                               sizeof(struct dpaa2_fd),
  37                                               sizeof(struct dpaa2_fd), 0);
  38         if (!dpaa2_chan->fd_pool)
  39                 goto err;
  40
  41         dpaa2_chan->fl_pool = dma_pool_create("fl_pool", dev,
  42                                               sizeof(struct dpaa2_fl_entry),
  43                                               sizeof(struct dpaa2_fl_entry), 0);
  44         if (!dpaa2_chan->fl_pool)
  45                 goto err_fd;
  46
  47         dpaa2_chan->sdd_pool =
  48                 dma_pool_create("sdd_pool", dev,
  49                                 sizeof(struct dpaa2_qdma_sd_d),
  50                                 sizeof(struct dpaa2_qdma_sd_d), 0);
  51         if (!dpaa2_chan->sdd_pool)
  52                 goto err_fl;
  53
  54         return dpaa2_qdma->desc_allocated++;
  55 err_fl:
  56         dma_pool_destroy(dpaa2_chan->fl_pool);
  57 err_fd:
  58         dma_pool_destroy(dpaa2_chan->fd_pool);
  59 err:
  60         return -ENOMEM;
  61 }
  62
  63 static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan)
  64 {
  65         struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
  66         struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
  67         unsigned long flags;
  68
  69         LIST_HEAD(head);
  70
  71         spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags);
  72         vchan_get_all_descriptors(&dpaa2_chan->vchan, &head);
  73         spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags);
  74
  75         vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head);
  76
  77         dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_used);
  78         dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_free);
  79
  80         dma_pool_destroy(dpaa2_chan->fd_pool);
  81         dma_pool_destroy(dpaa2_chan->fl_pool);
  82         dma_pool_destroy(dpaa2_chan->sdd_pool);
  83         dpaa2_qdma->desc_allocated--;
  84 }
  85
  86 /*
  87  * Request a command descriptor for enqueue.
  88  */
  89 static struct dpaa2_qdma_comp *
  90 dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan)
  91 {
  92         struct dpaa2_qdma_priv *qdma_priv = dpaa2_chan->qdma->priv;
  93         struct device *dev = &qdma_priv->dpdmai_dev->dev;
  94         struct dpaa2_qdma_comp *comp_temp = NULL;
  95         unsigned long flags;
  96
  97         spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
  98         if (list_empty(&dpaa2_chan->comp_free)) {
  99                 spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 100                 comp_temp = kzalloc(sizeof(*comp_temp), GFP_NOWAIT);
 101                 if (!comp_temp)
 102                         goto err;
 103                 comp_temp->fd_virt_addr =
 104                         dma_pool_alloc(dpaa2_chan->fd_pool, GFP_NOWAIT,
 105                                        &comp_temp->fd_bus_addr);
 106                 if (!comp_temp->fd_virt_addr)
 107                         goto err_comp;
 108
 109                 comp_temp->fl_virt_addr =
 110                         dma_pool_alloc(dpaa2_chan->fl_pool, GFP_NOWAIT,
 111                                        &comp_temp->fl_bus_addr);
 112                 if (!comp_temp->fl_virt_addr)
 113                         goto err_fd_virt;
 114
 115                 comp_temp->desc_virt_addr =
 116                         dma_pool_alloc(dpaa2_chan->sdd_pool, GFP_NOWAIT,
 117                                        &comp_temp->desc_bus_addr);
 118                 if (!comp_temp->desc_virt_addr)
 119                         goto err_fl_virt;
 120
 121                 comp_temp->qchan = dpaa2_chan;
 122                 return comp_temp;
 123         }
 124
 125         comp_temp = list_first_entry(&dpaa2_chan->comp_free,
 126                                      struct dpaa2_qdma_comp, list);
 127         list_del(&comp_temp->list);
 128         spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 129
 130         comp_temp->qchan = dpaa2_chan;
 131
 132         return comp_temp;
 133
 134 err_fl_virt:
 135                 dma_pool_free(dpaa2_chan->fl_pool,
 136                               comp_temp->fl_virt_addr,
 137                               comp_temp->fl_bus_addr);
 138 err_fd_virt:
 139                 dma_pool_free(dpaa2_chan->fd_pool,
 140                               comp_temp->fd_virt_addr,
 141                               comp_temp->fd_bus_addr);
 142 err_comp:
 143         kfree(comp_temp);
 144 err:
 145         dev_err(dev, "Failed to request descriptor\n");
 146         return NULL;
 147 }
 148
 149 static void
 150 dpaa2_qdma_populate_fd(u32 format, struct dpaa2_qdma_comp *dpaa2_comp)
 151 {
 152         struct dpaa2_fd *fd;
 153
 154         fd = dpaa2_comp->fd_virt_addr;
 155         memset(fd, 0, sizeof(struct dpaa2_fd));
 156
 157         /* fd populated */
 158         dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr);
 159
 160         /*
 161          * Bypass memory translation, Frame list format, short length disable
 162          * we need to disable BMT if fsl-mc use iova addr
 163          */
 164         if (smmu_disable)
 165                 dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE);
 166         dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE);
 167
 168         dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX);
 169 }
 170
 171 /* first frame list for descriptor buffer */
 172 static void
 173 dpaa2_qdma_populate_first_framel(struct dpaa2_fl_entry *f_list,
 174                                  struct dpaa2_qdma_comp *dpaa2_comp,
 175                                  bool wrt_changed)
 176 {
 177         struct dpaa2_qdma_sd_d *sdd;
 178
 179         sdd = dpaa2_comp->desc_virt_addr;
 180         memset(sdd, 0, 2 * (sizeof(*sdd)));
 181
 182         /* source descriptor CMD */
 183         sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT);
 184         sdd++;
 185
 186         /* dest descriptor CMD */
 187         if (wrt_changed)
 188                 sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT);
 189         else
 190                 sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT);
 191
 192         memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 193
 194         /* first frame list to source descriptor */
 195         dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr);
 196         dpaa2_fl_set_len(f_list, 0x20);
 197         dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG);
 198
 199         /* bypass memory translation */
 200         if (smmu_disable)
 201                 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
 202 }
 203
 204 /* source and destination frame list */
 205 static void
 206 dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list,
 207                            dma_addr_t dst, dma_addr_t src,
 208                            size_t len, uint8_t fmt)
 209 {
 210         /* source frame list to source buffer */
 211         memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 212
 213         dpaa2_fl_set_addr(f_list, src);
 214         dpaa2_fl_set_len(f_list, len);
 215
 216         /* single buffer frame or scatter gather frame */
 217         dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));
 218
 219         /* bypass memory translation */
 220         if (smmu_disable)
 221                 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
 222
 223         f_list++;
 224
 225         /* destination frame list to destination buffer */
 226         memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 227
 228         dpaa2_fl_set_addr(f_list, dst);
 229         dpaa2_fl_set_len(f_list, len);
 230         dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));
 231         /* single buffer frame or scatter gather frame */
 232         dpaa2_fl_set_final(f_list, QDMA_FL_F);
 233         /* bypass memory translation */
 234         if (smmu_disable)
 235                 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
 236 }
 237
 238 static struct dma_async_tx_descriptor
 239 *dpaa2_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,
 240                         dma_addr_t src, size_t len, ulong flags)
 241 {
 242         struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
 243         struct dpaa2_qdma_engine *dpaa2_qdma;
 244         struct dpaa2_qdma_comp *dpaa2_comp;
 245         struct dpaa2_fl_entry *f_list;
 246         bool wrt_changed;
 247
 248         dpaa2_qdma = dpaa2_chan->qdma;
 249         dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan);
 250         if (!dpaa2_comp)
 251                 return NULL;
 252
 253         wrt_changed = (bool)dpaa2_qdma->qdma_wrtype_fixup;
 254
 255         /* populate Frame descriptor */
 256         dpaa2_qdma_populate_fd(QDMA_FD_LONG_FORMAT, dpaa2_comp);
 257
 258         f_list = dpaa2_comp->fl_virt_addr;
 259
 260         /* first frame list for descriptor buffer (logn format) */
 261         dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed);
 262
 263         f_list++;
 264
 265         dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF);
 266
 267         return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags);
 268 }
 269
 270 static void dpaa2_qdma_issue_pending(struct dma_chan *chan)
 271 {
 272         struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
 273         struct dpaa2_qdma_comp *dpaa2_comp;
 274         struct virt_dma_desc *vdesc;
 275         struct dpaa2_fd *fd;
 276         unsigned long flags;
 277         int err;
 278
 279         spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
 280         spin_lock(&dpaa2_chan->vchan.lock);
 281         if (vchan_issue_pending(&dpaa2_chan->vchan)) {
 282                 vdesc = vchan_next_desc(&dpaa2_chan->vchan);
 283                 if (!vdesc)
 284                         goto err_enqueue;
 285                 dpaa2_comp = to_fsl_qdma_comp(vdesc);
 286
 287                 fd = dpaa2_comp->fd_virt_addr;
 288
 289                 list_del(&vdesc->node);
 290                 list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used);
 291
 292                 err = dpaa2_io_service_enqueue_fq(NULL, dpaa2_chan->fqid, fd);
 293                 if (err) {
 294                         list_del(&dpaa2_comp->list);
 295                         list_add_tail(&dpaa2_comp->list,
 296                                       &dpaa2_chan->comp_free);
 297                 }
 298         }
 299 err_enqueue:
 300         spin_unlock(&dpaa2_chan->vchan.lock);
 301         spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 302 }
 303
 304 static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev)
 305 {
 306         struct dpaa2_qdma_priv_per_prio *ppriv;
 307         struct device *dev = &ls_dev->dev;
 308         struct dpaa2_qdma_priv *priv;
 309         u8 prio_def = DPDMAI_PRIO_NUM;
 310         int err = -EINVAL;
 311         int i;
 312
 313         priv = dev_get_drvdata(dev);
 314
 315         priv->dev = dev;
 316         priv->dpqdma_id = ls_dev->obj_desc.id;
 317
 318         /* Get the handle for the DPDMAI this interface is associate with */
 319         err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle);
 320         if (err) {
 321                 dev_err(dev, "dpdmai_open() failed\n");
 322                 return err;
 323         }
 324
 325         dev_dbg(dev, "Opened dpdmai object successfully\n");
 326
 327         err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
 328                                     &priv->dpdmai_attr);
 329         if (err) {
 330                 dev_err(dev, "dpdmai_get_attributes() failed\n");
 331                 goto exit;
 332         }
 333
 334         if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) {
 335                 dev_err(dev, "DPDMAI major version mismatch\n"
 336                              "Found %u.%u, supported version is %u.%u\n",
 337                                 priv->dpdmai_attr.version.major,
 338                                 priv->dpdmai_attr.version.minor,
 339                                 DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
 340                 goto exit;
 341         }
 342
 343         if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) {
 344                 dev_err(dev, "DPDMAI minor version mismatch\n"
 345                              "Found %u.%u, supported version is %u.%u\n",
 346                                 priv->dpdmai_attr.version.major,
 347                                 priv->dpdmai_attr.version.minor,
 348                                 DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
 349                 goto exit;
 350         }
 351
 352         priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def);
 353         ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL);
 354         if (!ppriv) {
 355                 err = -ENOMEM;
 356                 goto exit;
 357         }
 358         priv->ppriv = ppriv;
 359
 360         for (i = 0; i < priv->num_pairs; i++) {
 361                 err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
 362                                           i, &priv->rx_queue_attr[i]);
 363                 if (err) {
 364                         dev_err(dev, "dpdmai_get_rx_queue() failed\n");
 365                         goto exit;
 366                 }
 367                 ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;
 368
 369                 err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle,
 370                                           i, &priv->tx_fqid[i]);
 371                 if (err) {
 372                         dev_err(dev, "dpdmai_get_tx_queue() failed\n");
 373                         goto exit;
 374                 }
 375                 ppriv->req_fqid = priv->tx_fqid[i];
 376                 ppriv->prio = i;
 377                 ppriv->priv = priv;
 378                 ppriv++;
 379         }
 380
 381         return 0;
 382 exit:
 383         dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
 384         return err;
 385 }
 386
 387 static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx)
 388 {
 389         struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx,
 390                         struct dpaa2_qdma_priv_per_prio, nctx);
 391         struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp;
 392         struct dpaa2_qdma_priv *priv = ppriv->priv;
 393         u32 n_chans = priv->dpaa2_qdma->n_chans;
 394         struct dpaa2_qdma_chan *qchan;
 395         const struct dpaa2_fd *fd_eq;
 396         const struct dpaa2_fd *fd;
 397         struct dpaa2_dq *dq;
 398         int is_last = 0;
 399         int found;
 400         u8 status;
 401         int err;
 402         int i;
 403
 404         do {
 405                 err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,
 406                                                ppriv->store);
 407         } while (err);
 408
 409         while (!is_last) {
 410                 do {
 411                         dq = dpaa2_io_store_next(ppriv->store, &is_last);
 412                 } while (!is_last && !dq);
 413                 if (!dq) {
 414                         dev_err(priv->dev, "FQID returned no valid frames!\n");
 415                         continue;
 416                 }
 417
 418                 /* obtain FD and process the error */
 419                 fd = dpaa2_dq_fd(dq);
 420
 421                 status = dpaa2_fd_get_ctrl(fd) & 0xff;
 422                 if (status)
 423                         dev_err(priv->dev, "FD error occurred\n");
 424                 found = 0;
 425                 for (i = 0; i < n_chans; i++) {
 426                         qchan = &priv->dpaa2_qdma->chans[i];
 427                         spin_lock(&qchan->queue_lock);
 428                         if (list_empty(&qchan->comp_used)) {
 429                                 spin_unlock(&qchan->queue_lock);
 430                                 continue;
 431                         }
 432                         list_for_each_entry_safe(dpaa2_comp, _comp_tmp,
 433                                                  &qchan->comp_used, list) {
 434                                 fd_eq = dpaa2_comp->fd_virt_addr;
 435
 436                                 if (le64_to_cpu(fd_eq->simple.addr) ==
 437                                     le64_to_cpu(fd->simple.addr)) {
 438                                         spin_lock(&qchan->vchan.lock);
 439                                         vchan_cookie_complete(&
 440                                                         dpaa2_comp->vdesc);
 441                                         spin_unlock(&qchan->vchan.lock);
 442                                         found = 1;
 443                                         break;
 444                                 }
 445                         }
 446                         spin_unlock(&qchan->queue_lock);
 447                         if (found)
 448                                 break;
 449                 }
 450         }
 451
 452         dpaa2_io_service_rearm(NULL, ctx);
 453 }
 454
 455 static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv)
 456 {
 457         struct dpaa2_qdma_priv_per_prio *ppriv;
 458         struct device *dev = priv->dev;
 459         int err = -EINVAL;
 460         int i, num;
 461
 462         num = priv->num_pairs;
 463         ppriv = priv->ppriv;
 464         for (i = 0; i < num; i++) {
 465                 ppriv->nctx.is_cdan = 0;
 466                 ppriv->nctx.desired_cpu = DPAA2_IO_ANY_CPU;
 467                 ppriv->nctx.id = ppriv->rsp_fqid;
 468                 ppriv->nctx.cb = dpaa2_qdma_fqdan_cb;
 469                 err = dpaa2_io_service_register(NULL, &ppriv->nctx, dev);
 470                 if (err) {
 471                         dev_err(dev, "Notification register failed\n");
 472                         goto err_service;
 473                 }
 474
 475                 ppriv->store =
 476                         dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, dev);
 477                 if (!ppriv->store) {
 478                         dev_err(dev, "dpaa2_io_store_create() failed\n");
 479                         goto err_store;
 480                 }
 481
 482                 ppriv++;
 483         }
 484         return 0;
 485
 486 err_store:
 487         dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
 488 err_service:
 489         ppriv--;
 490         while (ppriv >= priv->ppriv) {
 491                 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
 492                 dpaa2_io_store_destroy(ppriv->store);
 493                 ppriv--;
 494         }
 495         return err;
 496 }
 497
 498 static void dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv)
 499 {
 500         struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
 501         int i;
 502
 503         for (i = 0; i < priv->num_pairs; i++) {
 504                 dpaa2_io_store_destroy(ppriv->store);
 505                 ppriv++;
 506         }
 507 }
 508
 509 static void dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv)
 510 {
 511         struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
 512         struct device *dev = priv->dev;
 513         int i;
 514
 515         for (i = 0; i < priv->num_pairs; i++) {
 516                 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
 517                 ppriv++;
 518         }
 519 }
 520
 521 static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv)
 522 {
 523         struct dpdmai_rx_queue_cfg rx_queue_cfg;
 524         struct dpaa2_qdma_priv_per_prio *ppriv;
 525         struct device *dev = priv->dev;
 526         struct fsl_mc_device *ls_dev;
 527         int i, num;
 528         int err;
 529
 530         ls_dev = to_fsl_mc_device(dev);
 531         num = priv->num_pairs;
 532         ppriv = priv->ppriv;
 533         for (i = 0; i < num; i++) {
 534                 rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX |
 535                                         DPDMAI_QUEUE_OPT_DEST;
 536                 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
 537                 rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO;
 538                 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
 539                 rx_queue_cfg.dest_cfg.priority = ppriv->prio;
 540                 err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
 541                                           rx_queue_cfg.dest_cfg.priority,
 542                                           &rx_queue_cfg);
 543                 if (err) {
 544                         dev_err(dev, "dpdmai_set_rx_queue() failed\n");
 545                         return err;
 546                 }
 547
 548                 ppriv++;
 549         }
 550
 551         return 0;
 552 }
 553
 554 static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv)
 555 {
 556         struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
 557         struct device *dev = priv->dev;
 558         struct fsl_mc_device *ls_dev;
 559         int err = 0;
 560         int i;
 561
 562         ls_dev = to_fsl_mc_device(dev);
 563
 564         for (i = 0; i < priv->num_pairs; i++) {
 565                 ppriv->nctx.qman64 = 0;
 566                 ppriv->nctx.dpio_id = 0;
 567                 ppriv++;
 568         }
 569
 570         err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle);
 571         if (err)
 572                 dev_err(dev, "dpdmai_reset() failed\n");
 573
 574         return err;
 575 }
 576
 577 static void dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan,
 578                                    struct list_head *head)
 579 {
 580         struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp;
 581         unsigned long flags;
 582
 583         list_for_each_entry_safe(comp_tmp, _comp_tmp,
 584                                  head, list) {
 585                 spin_lock_irqsave(&qchan->queue_lock, flags);
 586                 list_del(&comp_tmp->list);
 587                 spin_unlock_irqrestore(&qchan->queue_lock, flags);
 588                 dma_pool_free(qchan->fd_pool,
 589                               comp_tmp->fd_virt_addr,
 590                               comp_tmp->fd_bus_addr);
 591                 dma_pool_free(qchan->fl_pool,
 592                               comp_tmp->fl_virt_addr,
 593                               comp_tmp->fl_bus_addr);
 594                 dma_pool_free(qchan->sdd_pool,
 595                               comp_tmp->desc_virt_addr,
 596                               comp_tmp->desc_bus_addr);
 597                 kfree(comp_tmp);
 598         }
 599 }
 600
 601 static void dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
 602 {
 603         struct dpaa2_qdma_chan *qchan;
 604         int num, i;
 605
 606         num = dpaa2_qdma->n_chans;
 607         for (i = 0; i < num; i++) {
 608                 qchan = &dpaa2_qdma->chans[i];
 609                 dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used);
 610                 dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free);
 611                 dma_pool_destroy(qchan->fd_pool);
 612                 dma_pool_destroy(qchan->fl_pool);
 613                 dma_pool_destroy(qchan->sdd_pool);
 614         }
 615 }
 616
 617 static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc)
 618 {
 619         struct dpaa2_qdma_comp *dpaa2_comp;
 620         struct dpaa2_qdma_chan *qchan;
 621         unsigned long flags;
 622
 623         dpaa2_comp = to_fsl_qdma_comp(vdesc);
 624         qchan = dpaa2_comp->qchan;
 625         spin_lock_irqsave(&qchan->queue_lock, flags);
 626         list_del(&dpaa2_comp->list);
 627         list_add_tail(&dpaa2_comp->list, &qchan->comp_free);
 628         spin_unlock_irqrestore(&qchan->queue_lock, flags);
 629 }
 630
 631 static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
 632 {
 633         struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv;
 634         struct dpaa2_qdma_chan *dpaa2_chan;
 635         int num = priv->num_pairs;
 636         int i;
 637
 638         INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels);
 639         for (i = 0; i < dpaa2_qdma->n_chans; i++) {
 640                 dpaa2_chan = &dpaa2_qdma->chans[i];
 641                 dpaa2_chan->qdma = dpaa2_qdma;
 642                 dpaa2_chan->fqid = priv->tx_fqid[i % num];
 643                 dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc;
 644                 vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev);
 645                 spin_lock_init(&dpaa2_chan->queue_lock);
 646                 INIT_LIST_HEAD(&dpaa2_chan->comp_used);
 647                 INIT_LIST_HEAD(&dpaa2_chan->comp_free);
 648         }
 649         return 0;
 650 }
 651
 652 static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev)
 653 {
 654         struct device *dev = &dpdmai_dev->dev;
 655         struct dpaa2_qdma_engine *dpaa2_qdma;
 656         struct dpaa2_qdma_priv *priv;
 657         int err;
 658
 659         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 660         if (!priv)
 661                 return -ENOMEM;
 662         dev_set_drvdata(dev, priv);
 663         priv->dpdmai_dev = dpdmai_dev;
 664
 665         priv->iommu_domain = iommu_get_domain_for_dev(dev);
 666         if (priv->iommu_domain)
 667                 smmu_disable = false;
 668
 669         /* obtain a MC portal */
 670         err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io);
 671         if (err) {
 672                 if (err == -ENXIO)
 673                         err = -EPROBE_DEFER;
 674                 else
 675                         dev_err(dev, "MC portal allocation failed\n");
 676                 goto err_mcportal;
 677         }
 678
 679         /* DPDMAI initialization */
 680         err = dpaa2_qdma_setup(dpdmai_dev);
 681         if (err) {
 682                 dev_err(dev, "dpaa2_dpdmai_setup() failed\n");
 683                 goto err_dpdmai_setup;
 684         }
 685
 686         /* DPIO */
 687         err = dpaa2_qdma_dpio_setup(priv);
 688         if (err) {
 689                 dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n");
 690                 goto err_dpio_setup;
 691         }
 692
 693         /* DPDMAI binding to DPIO */
 694         err = dpaa2_dpdmai_bind(priv);
 695         if (err) {
 696                 dev_err(dev, "dpaa2_dpdmai_bind() failed\n");
 697                 goto err_bind;
 698         }
 699
 700         /* DPDMAI enable */
 701         err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle);
 702         if (err) {
 703                 dev_err(dev, "dpdmai_enable() faile\n");
 704                 goto err_enable;
 705         }
 706
 707         dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL);
 708         if (!dpaa2_qdma) {
 709                 err = -ENOMEM;
 710                 goto err_eng;
 711         }
 712
 713         priv->dpaa2_qdma = dpaa2_qdma;
 714         dpaa2_qdma->priv = priv;
 715
 716         dpaa2_qdma->desc_allocated = 0;
 717         dpaa2_qdma->n_chans = NUM_CH;
 718
 719         dpaa2_dpdmai_init_channels(dpaa2_qdma);
 720
 721         if (soc_device_match(soc_fixup_tuning))
 722                 dpaa2_qdma->qdma_wrtype_fixup = true;
 723         else
 724                 dpaa2_qdma->qdma_wrtype_fixup = false;
 725
 726         dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask);
 727         dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask);
 728         dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask);
 729
 730         dpaa2_qdma->dma_dev.dev = dev;
 731         dpaa2_qdma->dma_dev.device_alloc_chan_resources =
 732                 dpaa2_qdma_alloc_chan_resources;
 733         dpaa2_qdma->dma_dev.device_free_chan_resources =
 734                 dpaa2_qdma_free_chan_resources;
 735         dpaa2_qdma->dma_dev.device_tx_status = dma_cookie_status;
 736         dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy;
 737         dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending;
 738
 739         err = dma_async_device_register(&dpaa2_qdma->dma_dev);
 740         if (err) {
 741                 dev_err(dev, "Can't register NXP QDMA engine.\n");
 742                 goto err_dpaa2_qdma;
 743         }
 744
 745         return 0;
 746
 747 err_dpaa2_qdma:
 748         kfree(dpaa2_qdma);
 749 err_eng:
 750         dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle);
 751 err_enable:
 752         dpaa2_dpdmai_dpio_unbind(priv);
 753 err_bind:
 754         dpaa2_dpmai_store_free(priv);
 755         dpaa2_dpdmai_dpio_free(priv);
 756 err_dpio_setup:
 757         kfree(priv->ppriv);
 758         dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle);
 759 err_dpdmai_setup:
 760         fsl_mc_portal_free(priv->mc_io);
 761 err_mcportal:
 762         kfree(priv);
 763         dev_set_drvdata(dev, NULL);
 764         return err;
 765 }
 766
 767 static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev)
 768 {
 769         struct dpaa2_qdma_engine *dpaa2_qdma;
 770         struct dpaa2_qdma_priv *priv;
 771         struct device *dev;
 772
 773         dev = &ls_dev->dev;
 774         priv = dev_get_drvdata(dev);
 775         dpaa2_qdma = priv->dpaa2_qdma;
 776
 777         dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
 778         dpaa2_dpdmai_dpio_unbind(priv);
 779         dpaa2_dpmai_store_free(priv);
 780         dpaa2_dpdmai_dpio_free(priv);
 781         dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
 782         fsl_mc_portal_free(priv->mc_io);
 783         dev_set_drvdata(dev, NULL);
 784         dpaa2_dpdmai_free_channels(dpaa2_qdma);
 785
 786         dma_async_device_unregister(&dpaa2_qdma->dma_dev);
 787         kfree(priv);
 788         kfree(dpaa2_qdma);
 789
 790         return 0;
 791 }
 792
 793 static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev)
 794 {
 795         struct dpaa2_qdma_priv *priv;
 796         struct device *dev;
 797
 798         dev = &ls_dev->dev;
 799         priv = dev_get_drvdata(dev);
 800
 801         dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
 802         dpaa2_dpdmai_dpio_unbind(priv);
 803         dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
 804         dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle);
 805 }
 806
 807 static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = {
 808         {
 809                 .vendor = FSL_MC_VENDOR_FREESCALE,
 810                 .obj_type = "dpdmai",
 811         },
 812         { .vendor = 0x0 }
 813 };
 814
 815 static struct fsl_mc_driver dpaa2_qdma_driver = {
 816         .driver         = {
 817                 .name   = "dpaa2-qdma",
 818                 .owner  = THIS_MODULE,
 819         },
 820         .probe          = dpaa2_qdma_probe,
 821         .remove         = dpaa2_qdma_remove,
 822         .shutdown       = dpaa2_qdma_shutdown,
 823         .match_id_table = dpaa2_qdma_id_table
 824 };
 825
 826 static int __init dpaa2_qdma_driver_init(void)
 827 {
 828         return fsl_mc_driver_register(&(dpaa2_qdma_driver));
 829 }
 830 late_initcall(dpaa2_qdma_driver_init);
 831
 832 static void __exit fsl_qdma_exit(void)
 833 {
 834         fsl_mc_driver_unregister(&(dpaa2_qdma_driver));
 835 }
 836 module_exit(fsl_qdma_exit);
 837
 838 MODULE_ALIAS("platform:fsl-dpaa2-qdma");
 839 MODULE_LICENSE("GPL v2");
 840 MODULE_DESCRIPTION("NXP Layerscape DPAA2 qDMA engine driver");