drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright 2019 NXP
   3
   4 #include <linux/init.h>
   5 #include <linux/module.h>
   6 #include <linux/dmapool.h>
   7 #include <linux/of_irq.h>
   8 #include <linux/iommu.h>
   9 #include <linux/sys_soc.h>
  10 #include <linux/fsl/mc.h>
  11 #include <soc/fsl/dpaa2-io.h>
  12
  13 #include "../virt-dma.h"
  14 #include "dpdmai.h"
  15 #include "dpaa2-qdma.h"
  16
  17 static bool smmu_disable = true;
  18
  19 static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan)
  20 {
  21         return container_of(chan, struct dpaa2_qdma_chan, vchan.chan);
  22 }
  23
  24 static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
  25 {
  26         return container_of(vd, struct dpaa2_qdma_comp, vdesc);
  27 }
  28
  29 static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan)
  30 {
  31         struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
  32         struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
  33         struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev;
  34
  35         dpaa2_chan->fd_pool = dma_pool_create("fd_pool", dev,
  36                                               sizeof(struct dpaa2_fd),
  37                                               sizeof(struct dpaa2_fd), 0);
  38         if (!dpaa2_chan->fd_pool)
  39                 goto err;
  40
  41         dpaa2_chan->fl_pool = dma_pool_create("fl_pool", dev,
  42                                               sizeof(struct dpaa2_fl_entry),
  43                                               sizeof(struct dpaa2_fl_entry), 0);
  44         if (!dpaa2_chan->fl_pool)
  45                 goto err_fd;
  46
  47         dpaa2_chan->sdd_pool =
  48                 dma_pool_create("sdd_pool", dev,
  49                                 sizeof(struct dpaa2_qdma_sd_d),
  50                                 sizeof(struct dpaa2_qdma_sd_d), 0);
  51         if (!dpaa2_chan->sdd_pool)
  52                 goto err_fl;
  53
  54         return dpaa2_qdma->desc_allocated++;
  55 err_fl:
  56         dma_pool_destroy(dpaa2_chan->fl_pool);
  57 err_fd:
  58         dma_pool_destroy(dpaa2_chan->fd_pool);
  59 err:
  60         return -ENOMEM;
  61 }
  62
  63 static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan)
  64 {
  65         struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
  66         struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
  67         unsigned long flags;
  68
  69         LIST_HEAD(head);
  70
  71         spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags);
  72         vchan_get_all_descriptors(&dpaa2_chan->vchan, &head);
  73         spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags);
  74
  75         vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head);
  76
  77         dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_used);
  78         dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_free);
  79
  80         dma_pool_destroy(dpaa2_chan->fd_pool);
  81         dma_pool_destroy(dpaa2_chan->fl_pool);
  82         dma_pool_destroy(dpaa2_chan->sdd_pool);
  83         dpaa2_qdma->desc_allocated--;
  84 }
  85
  86 /*
  87  * Request a command descriptor for enqueue.
  88  */
  89 static struct dpaa2_qdma_comp *
  90 dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan)
  91 {
  92         struct dpaa2_qdma_priv *qdma_priv = dpaa2_chan->qdma->priv;
  93         struct device *dev = &qdma_priv->dpdmai_dev->dev;
  94         struct dpaa2_qdma_comp *comp_temp = NULL;
  95         unsigned long flags;
  96
  97         spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
  98         if (list_empty(&dpaa2_chan->comp_free)) {
  99                 spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 100                 comp_temp = kzalloc(sizeof(*comp_temp), GFP_NOWAIT);
 101                 if (!comp_temp)
 102                         goto err;
 103                 comp_temp->fd_virt_addr =
 104                         dma_pool_alloc(dpaa2_chan->fd_pool, GFP_NOWAIT,
 105                                        &comp_temp->fd_bus_addr);
 106                 if (!comp_temp->fd_virt_addr)
 107                         goto err_comp;
 108
 109                 comp_temp->fl_virt_addr =
 110                         dma_pool_alloc(dpaa2_chan->fl_pool, GFP_NOWAIT,
 111                                        &comp_temp->fl_bus_addr);
 112                 if (!comp_temp->fl_virt_addr)
 113                         goto err_fd_virt;
 114
 115                 comp_temp->desc_virt_addr =
 116                         dma_pool_alloc(dpaa2_chan->sdd_pool, GFP_NOWAIT,
 117                                        &comp_temp->desc_bus_addr);
 118                 if (!comp_temp->desc_virt_addr)
 119                         goto err_fl_virt;
 120
 121                 comp_temp->qchan = dpaa2_chan;
 122                 return comp_temp;
 123         }
 124
 125         comp_temp = list_first_entry(&dpaa2_chan->comp_free,
 126                                      struct dpaa2_qdma_comp, list);
 127         list_del(&comp_temp->list);
 128         spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 129
 130         comp_temp->qchan = dpaa2_chan;
 131
 132         return comp_temp;
 133
 134 err_fl_virt:
 135                 dma_pool_free(dpaa2_chan->fl_pool,
 136                               comp_temp->fl_virt_addr,
 137                               comp_temp->fl_bus_addr);
 138 err_fd_virt:
 139                 dma_pool_free(dpaa2_chan->fd_pool,
 140                               comp_temp->fd_virt_addr,
 141                               comp_temp->fd_bus_addr);
 142 err_comp:
 143         kfree(comp_temp);
 144 err:
 145         dev_err(dev, "Failed to request descriptor\n");
 146         return NULL;
 147 }
 148
 149 static void
 150 dpaa2_qdma_populate_fd(u32 format, struct dpaa2_qdma_comp *dpaa2_comp)
 151 {
 152         struct dpaa2_fd *fd;
 153
 154         fd = dpaa2_comp->fd_virt_addr;
 155         memset(fd, 0, sizeof(struct dpaa2_fd));
 156
 157         /* fd populated */
 158         dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr);
 159
 160         /*
 161          * Bypass memory translation, Frame list format, short length disable
 162          * we need to disable BMT if fsl-mc use iova addr
 163          */
 164         if (smmu_disable)
 165                 dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE);
 166         dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE);
 167
 168         dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX);
 169 }
 170
 171 /* first frame list for descriptor buffer */
 172 static void
 173 dpaa2_qdma_populate_first_framel(struct dpaa2_fl_entry *f_list,
 174                                  struct dpaa2_qdma_comp *dpaa2_comp,
 175                                  bool wrt_changed)
 176 {
 177         struct dpaa2_qdma_sd_d *sdd;
 178
 179         sdd = dpaa2_comp->desc_virt_addr;
 180         memset(sdd, 0, 2 * (sizeof(*sdd)));
 181
 182         /* source descriptor CMD */
 183         sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT);
 184         sdd++;
 185
 186         /* dest descriptor CMD */
 187         if (wrt_changed)
 188                 sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT);
 189         else
 190                 sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT);
 191
 192         memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 193
 194         /* first frame list to source descriptor */
 195         dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr);
 196         dpaa2_fl_set_len(f_list, 0x20);
 197         dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG);
 198
 199         /* bypass memory translation */
 200         if (smmu_disable)
 201                 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
 202 }
 203
 204 /* source and destination frame list */
 205 static void
 206 dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list,
 207                            dma_addr_t dst, dma_addr_t src,
 208                            size_t len, uint8_t fmt)
 209 {
 210         /* source frame list to source buffer */
 211         memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 212
 213         dpaa2_fl_set_addr(f_list, src);
 214         dpaa2_fl_set_len(f_list, len);
 215
 216         /* single buffer frame or scatter gather frame */
 217         dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));
 218
 219         /* bypass memory translation */
 220         if (smmu_disable)
 221                 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
 222
 223         f_list++;
 224
 225         /* destination frame list to destination buffer */
 226         memset(f_list, 0, sizeof(struct dpaa2_fl_entry));
 227
 228         dpaa2_fl_set_addr(f_list, dst);
 229         dpaa2_fl_set_len(f_list, len);
 230         dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));
 231         /* single buffer frame or scatter gather frame */
 232         dpaa2_fl_set_final(f_list, QDMA_FL_F);
 233         /* bypass memory translation */
 234         if (smmu_disable)
 235                 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);
 236 }
 237
 238 static struct dma_async_tx_descriptor
 239 *dpaa2_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,
 240                         dma_addr_t src, size_t len, ulong flags)
 241 {
 242         struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
 243         struct dpaa2_qdma_engine *dpaa2_qdma;
 244         struct dpaa2_qdma_comp *dpaa2_comp;
 245         struct dpaa2_fl_entry *f_list;
 246         bool wrt_changed;
 247
 248         dpaa2_qdma = dpaa2_chan->qdma;
 249         dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan);
 250         if (!dpaa2_comp)
 251                 return NULL;
 252
 253         wrt_changed = (bool)dpaa2_qdma->qdma_wrtype_fixup;
 254
 255         /* populate Frame descriptor */
 256         dpaa2_qdma_populate_fd(QDMA_FD_LONG_FORMAT, dpaa2_comp);
 257
 258         f_list = dpaa2_comp->fl_virt_addr;
 259
 260         /* first frame list for descriptor buffer (logn format) */
 261         dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed);
 262
 263         f_list++;
 264
 265         dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF);
 266
 267         return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags);
 268 }
 269
 270 static void dpaa2_qdma_issue_pending(struct dma_chan *chan)
 271 {
 272         struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
 273         struct dpaa2_qdma_comp *dpaa2_comp;
 274         struct virt_dma_desc *vdesc;
 275         struct dpaa2_fd *fd;
 276         unsigned long flags;
 277         int err;
 278
 279         spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
 280         spin_lock(&dpaa2_chan->vchan.lock);
 281         if (vchan_issue_pending(&dpaa2_chan->vchan)) {
 282                 vdesc = vchan_next_desc(&dpaa2_chan->vchan);
 283                 if (!vdesc)
 284                         goto err_enqueue;
 285                 dpaa2_comp = to_fsl_qdma_comp(vdesc);
 286
 287                 fd = dpaa2_comp->fd_virt_addr;
 288
 289                 list_del(&vdesc->node);
 290                 list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used);
 291
 292                 err = dpaa2_io_service_enqueue_fq(NULL, dpaa2_chan->fqid, fd);
 293                 if (err) {
 294                         list_del(&dpaa2_comp->list);
 295                         list_add_tail(&dpaa2_comp->list,
 296                                       &dpaa2_chan->comp_free);
 297                 }
 298         }
 299 err_enqueue:
 300         spin_unlock(&dpaa2_chan->vchan.lock);
 301         spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
 302 }
 303
 304 static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev)
 305 {
 306         struct dpaa2_qdma_priv_per_prio *ppriv;
 307         struct device *dev = &ls_dev->dev;
 308         struct dpaa2_qdma_priv *priv;
 309         u8 prio_def = DPDMAI_PRIO_NUM;
 310         int err = -EINVAL;
 311         int i;
 312
 313         priv = dev_get_drvdata(dev);
 314
 315         priv->dev = dev;
 316         priv->dpqdma_id = ls_dev->obj_desc.id;
 317
 318         /* Get the handle for the DPDMAI this interface is associate with */
 319         err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle);
 320         if (err) {
 321                 dev_err(dev, "dpdmai_open() failed\n");
 322                 return err;
 323         }
 324
 325         dev_dbg(dev, "Opened dpdmai object successfully\n");
 326
 327         err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
 328                                     &priv->dpdmai_attr);
 329         if (err) {
 330                 dev_err(dev, "dpdmai_get_attributes() failed\n");
 331                 goto exit;
 332         }
 333
 334         if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) {
 335                 err = -EINVAL;
 336                 dev_err(dev, "DPDMAI major version mismatch\n"
 337                              "Found %u.%u, supported version is %u.%u\n",
 338                                 priv->dpdmai_attr.version.major,
 339                                 priv->dpdmai_attr.version.minor,
 340                                 DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
 341                 goto exit;
 342         }
 343
 344         if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) {
 345                 err = -EINVAL;
 346                 dev_err(dev, "DPDMAI minor version mismatch\n"
 347                              "Found %u.%u, supported version is %u.%u\n",
 348                                 priv->dpdmai_attr.version.major,
 349                                 priv->dpdmai_attr.version.minor,
 350                                 DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
 351                 goto exit;
 352         }
 353
 354         priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def);
 355         ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL);
 356         if (!ppriv) {
 357                 err = -ENOMEM;
 358                 goto exit;
 359         }
 360         priv->ppriv = ppriv;
 361
 362         for (i = 0; i < priv->num_pairs; i++) {
 363                 err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
 364                                           i, &priv->rx_queue_attr[i]);
 365                 if (err) {
 366                         dev_err(dev, "dpdmai_get_rx_queue() failed\n");
 367                         goto exit;
 368                 }
 369                 ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;
 370
 371                 err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle,
 372                                           i, &priv->tx_fqid[i]);
 373                 if (err) {
 374                         dev_err(dev, "dpdmai_get_tx_queue() failed\n");
 375                         goto exit;
 376                 }
 377                 ppriv->req_fqid = priv->tx_fqid[i];
 378                 ppriv->prio = i;
 379                 ppriv->priv = priv;
 380                 ppriv++;
 381         }
 382
 383         return 0;
 384 exit:
 385         dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
 386         return err;
 387 }
 388
 389 static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx)
 390 {
 391         struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx,
 392                         struct dpaa2_qdma_priv_per_prio, nctx);
 393         struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp;
 394         struct dpaa2_qdma_priv *priv = ppriv->priv;
 395         u32 n_chans = priv->dpaa2_qdma->n_chans;
 396         struct dpaa2_qdma_chan *qchan;
 397         const struct dpaa2_fd *fd_eq;
 398         const struct dpaa2_fd *fd;
 399         struct dpaa2_dq *dq;
 400         int is_last = 0;
 401         int found;
 402         u8 status;
 403         int err;
 404         int i;
 405
 406         do {
 407                 err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,
 408                                                ppriv->store);
 409         } while (err);
 410
 411         while (!is_last) {
 412                 do {
 413                         dq = dpaa2_io_store_next(ppriv->store, &is_last);
 414                 } while (!is_last && !dq);
 415                 if (!dq) {
 416                         dev_err(priv->dev, "FQID returned no valid frames!\n");
 417                         continue;
 418                 }
 419
 420                 /* obtain FD and process the error */
 421                 fd = dpaa2_dq_fd(dq);
 422
 423                 status = dpaa2_fd_get_ctrl(fd) & 0xff;
 424                 if (status)
 425                         dev_err(priv->dev, "FD error occurred\n");
 426                 found = 0;
 427                 for (i = 0; i < n_chans; i++) {
 428                         qchan = &priv->dpaa2_qdma->chans[i];
 429                         spin_lock(&qchan->queue_lock);
 430                         if (list_empty(&qchan->comp_used)) {
 431                                 spin_unlock(&qchan->queue_lock);
 432                                 continue;
 433                         }
 434                         list_for_each_entry_safe(dpaa2_comp, _comp_tmp,
 435                                                  &qchan->comp_used, list) {
 436                                 fd_eq = dpaa2_comp->fd_virt_addr;
 437
 438                                 if (le64_to_cpu(fd_eq->simple.addr) ==
 439                                     le64_to_cpu(fd->simple.addr)) {
 440                                         spin_lock(&qchan->vchan.lock);
 441                                         vchan_cookie_complete(&
 442                                                         dpaa2_comp->vdesc);
 443                                         spin_unlock(&qchan->vchan.lock);
 444                                         found = 1;
 445                                         break;
 446                                 }
 447                         }
 448                         spin_unlock(&qchan->queue_lock);
 449                         if (found)
 450                                 break;
 451                 }
 452         }
 453
 454         dpaa2_io_service_rearm(NULL, ctx);
 455 }
 456
 457 static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv)
 458 {
 459         struct dpaa2_qdma_priv_per_prio *ppriv;
 460         struct device *dev = priv->dev;
 461         int err = -EINVAL;
 462         int i, num;
 463
 464         num = priv->num_pairs;
 465         ppriv = priv->ppriv;
 466         for (i = 0; i < num; i++) {
 467                 ppriv->nctx.is_cdan = 0;
 468                 ppriv->nctx.desired_cpu = DPAA2_IO_ANY_CPU;
 469                 ppriv->nctx.id = ppriv->rsp_fqid;
 470                 ppriv->nctx.cb = dpaa2_qdma_fqdan_cb;
 471                 err = dpaa2_io_service_register(NULL, &ppriv->nctx, dev);
 472                 if (err) {
 473                         dev_err(dev, "Notification register failed\n");
 474                         goto err_service;
 475                 }
 476
 477                 ppriv->store =
 478                         dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, dev);
 479                 if (!ppriv->store) {
 480                         err = -ENOMEM;
 481                         dev_err(dev, "dpaa2_io_store_create() failed\n");
 482                         goto err_store;
 483                 }
 484
 485                 ppriv++;
 486         }
 487         return 0;
 488
 489 err_store:
 490         dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
 491 err_service:
 492         ppriv--;
 493         while (ppriv >= priv->ppriv) {
 494                 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
 495                 dpaa2_io_store_destroy(ppriv->store);
 496                 ppriv--;
 497         }
 498         return err;
 499 }
 500
 501 static void dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv)
 502 {
 503         struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
 504         int i;
 505
 506         for (i = 0; i < priv->num_pairs; i++) {
 507                 dpaa2_io_store_destroy(ppriv->store);
 508                 ppriv++;
 509         }
 510 }
 511
 512 static void dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv)
 513 {
 514         struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
 515         struct device *dev = priv->dev;
 516         int i;
 517
 518         for (i = 0; i < priv->num_pairs; i++) {
 519                 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);
 520                 ppriv++;
 521         }
 522 }
 523
 524 static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv)
 525 {
 526         struct dpdmai_rx_queue_cfg rx_queue_cfg;
 527         struct dpaa2_qdma_priv_per_prio *ppriv;
 528         struct device *dev = priv->dev;
 529         struct fsl_mc_device *ls_dev;
 530         int i, num;
 531         int err;
 532
 533         ls_dev = to_fsl_mc_device(dev);
 534         num = priv->num_pairs;
 535         ppriv = priv->ppriv;
 536         for (i = 0; i < num; i++) {
 537                 rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX |
 538                                         DPDMAI_QUEUE_OPT_DEST;
 539                 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
 540                 rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO;
 541                 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
 542                 rx_queue_cfg.dest_cfg.priority = ppriv->prio;
 543                 err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
 544                                           rx_queue_cfg.dest_cfg.priority,
 545                                           &rx_queue_cfg);
 546                 if (err) {
 547                         dev_err(dev, "dpdmai_set_rx_queue() failed\n");
 548                         return err;
 549                 }
 550
 551                 ppriv++;
 552         }
 553
 554         return 0;
 555 }
 556
 557 static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv)
 558 {
 559         struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
 560         struct device *dev = priv->dev;
 561         struct fsl_mc_device *ls_dev;
 562         int err = 0;
 563         int i;
 564
 565         ls_dev = to_fsl_mc_device(dev);
 566
 567         for (i = 0; i < priv->num_pairs; i++) {
 568                 ppriv->nctx.qman64 = 0;
 569                 ppriv->nctx.dpio_id = 0;
 570                 ppriv++;
 571         }
 572
 573         err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle);
 574         if (err)
 575                 dev_err(dev, "dpdmai_reset() failed\n");
 576
 577         return err;
 578 }
 579
 580 static void dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan,
 581                                    struct list_head *head)
 582 {
 583         struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp;
 584         unsigned long flags;
 585
 586         list_for_each_entry_safe(comp_tmp, _comp_tmp,
 587                                  head, list) {
 588                 spin_lock_irqsave(&qchan->queue_lock, flags);
 589                 list_del(&comp_tmp->list);
 590                 spin_unlock_irqrestore(&qchan->queue_lock, flags);
 591                 dma_pool_free(qchan->fd_pool,
 592                               comp_tmp->fd_virt_addr,
 593                               comp_tmp->fd_bus_addr);
 594                 dma_pool_free(qchan->fl_pool,
 595                               comp_tmp->fl_virt_addr,
 596                               comp_tmp->fl_bus_addr);
 597                 dma_pool_free(qchan->sdd_pool,
 598                               comp_tmp->desc_virt_addr,
 599                               comp_tmp->desc_bus_addr);
 600                 kfree(comp_tmp);
 601         }
 602 }
 603
 604 static void dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
 605 {
 606         struct dpaa2_qdma_chan *qchan;
 607         int num, i;
 608
 609         num = dpaa2_qdma->n_chans;
 610         for (i = 0; i < num; i++) {
 611                 qchan = &dpaa2_qdma->chans[i];
 612                 dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used);
 613                 dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free);
 614                 dma_pool_destroy(qchan->fd_pool);
 615                 dma_pool_destroy(qchan->fl_pool);
 616                 dma_pool_destroy(qchan->sdd_pool);
 617         }
 618 }
 619
 620 static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc)
 621 {
 622         struct dpaa2_qdma_comp *dpaa2_comp;
 623         struct dpaa2_qdma_chan *qchan;
 624         unsigned long flags;
 625
 626         dpaa2_comp = to_fsl_qdma_comp(vdesc);
 627         qchan = dpaa2_comp->qchan;
 628         spin_lock_irqsave(&qchan->queue_lock, flags);
 629         list_del(&dpaa2_comp->list);
 630         list_add_tail(&dpaa2_comp->list, &qchan->comp_free);
 631         spin_unlock_irqrestore(&qchan->queue_lock, flags);
 632 }
 633
 634 static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
 635 {
 636         struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv;
 637         struct dpaa2_qdma_chan *dpaa2_chan;
 638         int num = priv->num_pairs;
 639         int i;
 640
 641         INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels);
 642         for (i = 0; i < dpaa2_qdma->n_chans; i++) {
 643                 dpaa2_chan = &dpaa2_qdma->chans[i];
 644                 dpaa2_chan->qdma = dpaa2_qdma;
 645                 dpaa2_chan->fqid = priv->tx_fqid[i % num];
 646                 dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc;
 647                 vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev);
 648                 spin_lock_init(&dpaa2_chan->queue_lock);
 649                 INIT_LIST_HEAD(&dpaa2_chan->comp_used);
 650                 INIT_LIST_HEAD(&dpaa2_chan->comp_free);
 651         }
 652         return 0;
 653 }
 654
 655 static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev)
 656 {
 657         struct device *dev = &dpdmai_dev->dev;
 658         struct dpaa2_qdma_engine *dpaa2_qdma;
 659         struct dpaa2_qdma_priv *priv;
 660         int err;
 661
 662         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 663         if (!priv)
 664                 return -ENOMEM;
 665         dev_set_drvdata(dev, priv);
 666         priv->dpdmai_dev = dpdmai_dev;
 667
 668         priv->iommu_domain = iommu_get_domain_for_dev(dev);
 669         if (priv->iommu_domain)
 670                 smmu_disable = false;
 671
 672         /* obtain a MC portal */
 673         err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io);
 674         if (err) {
 675                 if (err == -ENXIO)
 676                         err = -EPROBE_DEFER;
 677                 else
 678                         dev_err(dev, "MC portal allocation failed\n");
 679                 goto err_mcportal;
 680         }
 681
 682         /* DPDMAI initialization */
 683         err = dpaa2_qdma_setup(dpdmai_dev);
 684         if (err) {
 685                 dev_err(dev, "dpaa2_dpdmai_setup() failed\n");
 686                 goto err_dpdmai_setup;
 687         }
 688
 689         /* DPIO */
 690         err = dpaa2_qdma_dpio_setup(priv);
 691         if (err) {
 692                 dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n");
 693                 goto err_dpio_setup;
 694         }
 695
 696         /* DPDMAI binding to DPIO */
 697         err = dpaa2_dpdmai_bind(priv);
 698         if (err) {
 699                 dev_err(dev, "dpaa2_dpdmai_bind() failed\n");
 700                 goto err_bind;
 701         }
 702
 703         /* DPDMAI enable */
 704         err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle);
 705         if (err) {
 706                 dev_err(dev, "dpdmai_enable() faile\n");
 707                 goto err_enable;
 708         }
 709
 710         dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL);
 711         if (!dpaa2_qdma) {
 712                 err = -ENOMEM;
 713                 goto err_eng;
 714         }
 715
 716         priv->dpaa2_qdma = dpaa2_qdma;
 717         dpaa2_qdma->priv = priv;
 718
 719         dpaa2_qdma->desc_allocated = 0;
 720         dpaa2_qdma->n_chans = NUM_CH;
 721
 722         dpaa2_dpdmai_init_channels(dpaa2_qdma);
 723
 724         if (soc_device_match(soc_fixup_tuning))
 725                 dpaa2_qdma->qdma_wrtype_fixup = true;
 726         else
 727                 dpaa2_qdma->qdma_wrtype_fixup = false;
 728
 729         dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask);
 730         dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask);
 731         dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask);
 732
 733         dpaa2_qdma->dma_dev.dev = dev;
 734         dpaa2_qdma->dma_dev.device_alloc_chan_resources =
 735                 dpaa2_qdma_alloc_chan_resources;
 736         dpaa2_qdma->dma_dev.device_free_chan_resources =
 737                 dpaa2_qdma_free_chan_resources;
 738         dpaa2_qdma->dma_dev.device_tx_status = dma_cookie_status;
 739         dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy;
 740         dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending;
 741
 742         err = dma_async_device_register(&dpaa2_qdma->dma_dev);
 743         if (err) {
 744                 dev_err(dev, "Can't register NXP QDMA engine.\n");
 745                 goto err_dpaa2_qdma;
 746         }
 747
 748         return 0;
 749
 750 err_dpaa2_qdma:
 751         kfree(dpaa2_qdma);
 752 err_eng:
 753         dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle);
 754 err_enable:
 755         dpaa2_dpdmai_dpio_unbind(priv);
 756 err_bind:
 757         dpaa2_dpmai_store_free(priv);
 758         dpaa2_dpdmai_dpio_free(priv);
 759 err_dpio_setup:
 760         kfree(priv->ppriv);
 761         dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle);
 762 err_dpdmai_setup:
 763         fsl_mc_portal_free(priv->mc_io);
 764 err_mcportal:
 765         kfree(priv);
 766         dev_set_drvdata(dev, NULL);
 767         return err;
 768 }
 769
 770 static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev)
 771 {
 772         struct dpaa2_qdma_engine *dpaa2_qdma;
 773         struct dpaa2_qdma_priv *priv;
 774         struct device *dev;
 775
 776         dev = &ls_dev->dev;
 777         priv = dev_get_drvdata(dev);
 778         dpaa2_qdma = priv->dpaa2_qdma;
 779
 780         dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
 781         dpaa2_dpdmai_dpio_unbind(priv);
 782         dpaa2_dpmai_store_free(priv);
 783         dpaa2_dpdmai_dpio_free(priv);
 784         dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
 785         fsl_mc_portal_free(priv->mc_io);
 786         dev_set_drvdata(dev, NULL);
 787         dpaa2_dpdmai_free_channels(dpaa2_qdma);
 788
 789         dma_async_device_unregister(&dpaa2_qdma->dma_dev);
 790         kfree(priv);
 791         kfree(dpaa2_qdma);
 792
 793         return 0;
 794 }
 795
 796 static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev)
 797 {
 798         struct dpaa2_qdma_priv *priv;
 799         struct device *dev;
 800
 801         dev = &ls_dev->dev;
 802         priv = dev_get_drvdata(dev);
 803
 804         dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
 805         dpaa2_dpdmai_dpio_unbind(priv);
 806         dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
 807         dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle);
 808 }
 809
 810 static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = {
 811         {
 812                 .vendor = FSL_MC_VENDOR_FREESCALE,
 813                 .obj_type = "dpdmai",
 814         },
 815         { .vendor = 0x0 }
 816 };
 817
 818 static struct fsl_mc_driver dpaa2_qdma_driver = {
 819         .driver         = {
 820                 .name   = "dpaa2-qdma",
 821                 .owner  = THIS_MODULE,
 822         },
 823         .probe          = dpaa2_qdma_probe,
 824         .remove         = dpaa2_qdma_remove,
 825         .shutdown       = dpaa2_qdma_shutdown,
 826         .match_id_table = dpaa2_qdma_id_table
 827 };
 828
 829 static int __init dpaa2_qdma_driver_init(void)
 830 {
 831         return fsl_mc_driver_register(&(dpaa2_qdma_driver));
 832 }
 833 late_initcall(dpaa2_qdma_driver_init);
 834
 835 static void __exit fsl_qdma_exit(void)
 836 {
 837         fsl_mc_driver_unregister(&(dpaa2_qdma_driver));
 838 }
 839 module_exit(fsl_qdma_exit);
 840
 841 MODULE_ALIAS("platform:fsl-dpaa2-qdma");
 842 MODULE_LICENSE("GPL v2");
 843 MODULE_DESCRIPTION("NXP Layerscape DPAA2 qDMA engine driver");