drivers/crypto/ccp/ccp-dmaengine.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Cryptographic Coprocessor (CCP) driver
   4  *
   5  * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
   6  *
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  */
   9
  10 #include <linux/module.h>
  11 #include <linux/kernel.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/mutex.h>
  16 #include <linux/ccp.h>
  17
  18 #include "ccp-dev.h"
  19 #include "../../dma/dmaengine.h"
  20
  21 #define CCP_DMA_WIDTH(_mask)            \
  22 ({                                      \
  23         u64 mask = _mask + 1;           \
  24         (mask == 0) ? 64 : fls64(mask); \
  25 })
  26
  27 /* The CCP as a DMA provider can be configured for public or private
  28  * channels. Default is specified in the vdata for the device (PCI ID).
  29  * This module parameter will override for all channels on all devices:
  30  *   dma_chan_attr = 0x2 to force all channels public
  31  *                 = 0x1 to force all channels private
  32  *                 = 0x0 to defer to the vdata setting
  33  *                 = any other value: warning, revert to 0x0
  34  */
  35 static unsigned int dma_chan_attr = CCP_DMA_DFLT;
  36 module_param(dma_chan_attr, uint, 0444);
  37 MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public");
  38
  39 static unsigned int dmaengine = 1;
  40 module_param(dmaengine, uint, 0444);
  41 MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)");
  42
  43 static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp)
  44 {
  45         switch (dma_chan_attr) {
  46         case CCP_DMA_DFLT:
  47                 return ccp->vdata->dma_chan_attr;
  48
  49         case CCP_DMA_PRIV:
  50                 return DMA_PRIVATE;
  51
  52         case CCP_DMA_PUB:
  53                 return 0;
  54
  55         default:
  56                 dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n",
  57                               dma_chan_attr);
  58                 return ccp->vdata->dma_chan_attr;
  59         }
  60 }
  61
  62 static void ccp_free_cmd_resources(struct ccp_device *ccp,
  63                                    struct list_head *list)
  64 {
  65         struct ccp_dma_cmd *cmd, *ctmp;
  66
  67         list_for_each_entry_safe(cmd, ctmp, list, entry) {
  68                 list_del(&cmd->entry);
  69                 kmem_cache_free(ccp->dma_cmd_cache, cmd);
  70         }
  71 }
  72
  73 static void ccp_free_desc_resources(struct ccp_device *ccp,
  74                                     struct list_head *list)
  75 {
  76         struct ccp_dma_desc *desc, *dtmp;
  77
  78         list_for_each_entry_safe(desc, dtmp, list, entry) {
  79                 ccp_free_cmd_resources(ccp, &desc->active);
  80                 ccp_free_cmd_resources(ccp, &desc->pending);
  81
  82                 list_del(&desc->entry);
  83                 kmem_cache_free(ccp->dma_desc_cache, desc);
  84         }
  85 }
  86
  87 static void ccp_free_chan_resources(struct dma_chan *dma_chan)
  88 {
  89         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  90                                                  dma_chan);
  91         unsigned long flags;
  92
  93         dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan);
  94
  95         spin_lock_irqsave(&chan->lock, flags);
  96
  97         ccp_free_desc_resources(chan->ccp, &chan->complete);
  98         ccp_free_desc_resources(chan->ccp, &chan->active);
  99         ccp_free_desc_resources(chan->ccp, &chan->pending);
 100         ccp_free_desc_resources(chan->ccp, &chan->created);
 101
 102         spin_unlock_irqrestore(&chan->lock, flags);
 103 }
 104
 105 static void ccp_cleanup_desc_resources(struct ccp_device *ccp,
 106                                        struct list_head *list)
 107 {
 108         struct ccp_dma_desc *desc, *dtmp;
 109
 110         list_for_each_entry_safe_reverse(desc, dtmp, list, entry) {
 111                 if (!async_tx_test_ack(&desc->tx_desc))
 112                         continue;
 113
 114                 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
 115
 116                 ccp_free_cmd_resources(ccp, &desc->active);
 117                 ccp_free_cmd_resources(ccp, &desc->pending);
 118
 119                 list_del(&desc->entry);
 120                 kmem_cache_free(ccp->dma_desc_cache, desc);
 121         }
 122 }
 123
 124 static void ccp_do_cleanup(unsigned long data)
 125 {
 126         struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data;
 127         unsigned long flags;
 128
 129         dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__,
 130                 dma_chan_name(&chan->dma_chan));
 131
 132         spin_lock_irqsave(&chan->lock, flags);
 133
 134         ccp_cleanup_desc_resources(chan->ccp, &chan->complete);
 135
 136         spin_unlock_irqrestore(&chan->lock, flags);
 137 }
 138
 139 static int ccp_issue_next_cmd(struct ccp_dma_desc *desc)
 140 {
 141         struct ccp_dma_cmd *cmd;
 142         int ret;
 143
 144         cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry);
 145         list_move(&cmd->entry, &desc->active);
 146
 147         dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__,
 148                 desc->tx_desc.cookie, cmd);
 149
 150         ret = ccp_enqueue_cmd(&cmd->ccp_cmd);
 151         if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY))
 152                 return 0;
 153
 154         dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__,
 155                 ret, desc->tx_desc.cookie, cmd);
 156
 157         return ret;
 158 }
 159
 160 static void ccp_free_active_cmd(struct ccp_dma_desc *desc)
 161 {
 162         struct ccp_dma_cmd *cmd;
 163
 164         cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd,
 165                                        entry);
 166         if (!cmd)
 167                 return;
 168
 169         dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n",
 170                 __func__, desc->tx_desc.cookie, cmd);
 171
 172         list_del(&cmd->entry);
 173         kmem_cache_free(desc->ccp->dma_cmd_cache, cmd);
 174 }
 175
 176 static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan,
 177                                                 struct ccp_dma_desc *desc)
 178 {
 179         /* Move current DMA descriptor to the complete list */
 180         if (desc)
 181                 list_move(&desc->entry, &chan->complete);
 182
 183         /* Get the next DMA descriptor on the active list */
 184         desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
 185                                         entry);
 186
 187         return desc;
 188 }
 189
 190 static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan,
 191                                                    struct ccp_dma_desc *desc)
 192 {
 193         struct dma_async_tx_descriptor *tx_desc;
 194         unsigned long flags;
 195
 196         /* Loop over descriptors until one is found with commands */
 197         do {
 198                 if (desc) {
 199                         /* Remove the DMA command from the list and free it */
 200                         ccp_free_active_cmd(desc);
 201
 202                         if (!list_empty(&desc->pending)) {
 203                                 /* No errors, keep going */
 204                                 if (desc->status != DMA_ERROR)
 205                                         return desc;
 206
 207                                 /* Error, free remaining commands and move on */
 208                                 ccp_free_cmd_resources(desc->ccp,
 209                                                        &desc->pending);
 210                         }
 211
 212                         tx_desc = &desc->tx_desc;
 213                 } else {
 214                         tx_desc = NULL;
 215                 }
 216
 217                 spin_lock_irqsave(&chan->lock, flags);
 218
 219                 if (desc) {
 220                         if (desc->status != DMA_ERROR)
 221                                 desc->status = DMA_COMPLETE;
 222
 223                         dev_dbg(desc->ccp->dev,
 224                                 "%s - tx %d complete, status=%u\n", __func__,
 225                                 desc->tx_desc.cookie, desc->status);
 226
 227                         dma_cookie_complete(tx_desc);
 228                         dma_descriptor_unmap(tx_desc);
 229                 }
 230
 231                 desc = __ccp_next_dma_desc(chan, desc);
 232
 233                 spin_unlock_irqrestore(&chan->lock, flags);
 234
 235                 if (tx_desc) {
 236                         dmaengine_desc_get_callback_invoke(tx_desc, NULL);
 237
 238                         dma_run_dependencies(tx_desc);
 239                 }
 240         } while (desc);
 241
 242         return NULL;
 243 }
 244
 245 static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan)
 246 {
 247         struct ccp_dma_desc *desc;
 248
 249         if (list_empty(&chan->pending))
 250                 return NULL;
 251
 252         desc = list_empty(&chan->active)
 253                 ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry)
 254                 : NULL;
 255
 256         list_splice_tail_init(&chan->pending, &chan->active);
 257
 258         return desc;
 259 }
 260
 261 static void ccp_cmd_callback(void *data, int err)
 262 {
 263         struct ccp_dma_desc *desc = data;
 264         struct ccp_dma_chan *chan;
 265         int ret;
 266
 267         if (err == -EINPROGRESS)
 268                 return;
 269
 270         chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan,
 271                             dma_chan);
 272
 273         dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n",
 274                 __func__, desc->tx_desc.cookie, err);
 275
 276         if (err)
 277                 desc->status = DMA_ERROR;
 278
 279         while (true) {
 280                 /* Check for DMA descriptor completion */
 281                 desc = ccp_handle_active_desc(chan, desc);
 282
 283                 /* Don't submit cmd if no descriptor or DMA is paused */
 284                 if (!desc || (chan->status == DMA_PAUSED))
 285                         break;
 286
 287                 ret = ccp_issue_next_cmd(desc);
 288                 if (!ret)
 289                         break;
 290
 291                 desc->status = DMA_ERROR;
 292         }
 293
 294         tasklet_schedule(&chan->cleanup_tasklet);
 295 }
 296
 297 static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc)
 298 {
 299         struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc,
 300                                                  tx_desc);
 301         struct ccp_dma_chan *chan;
 302         dma_cookie_t cookie;
 303         unsigned long flags;
 304
 305         chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan);
 306
 307         spin_lock_irqsave(&chan->lock, flags);
 308
 309         cookie = dma_cookie_assign(tx_desc);
 310         list_del(&desc->entry);
 311         list_add_tail(&desc->entry, &chan->pending);
 312
 313         spin_unlock_irqrestore(&chan->lock, flags);
 314
 315         dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n",
 316                 __func__, cookie);
 317
 318         return cookie;
 319 }
 320
 321 static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan)
 322 {
 323         struct ccp_dma_cmd *cmd;
 324
 325         cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT);
 326         if (cmd)
 327                 memset(cmd, 0, sizeof(*cmd));
 328
 329         return cmd;
 330 }
 331
 332 static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan,
 333                                                unsigned long flags)
 334 {
 335         struct ccp_dma_desc *desc;
 336
 337         desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);
 338         if (!desc)
 339                 return NULL;
 340
 341         dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);
 342         desc->tx_desc.flags = flags;
 343         desc->tx_desc.tx_submit = ccp_tx_submit;
 344         desc->ccp = chan->ccp;
 345         INIT_LIST_HEAD(&desc->entry);
 346         INIT_LIST_HEAD(&desc->pending);
 347         INIT_LIST_HEAD(&desc->active);
 348         desc->status = DMA_IN_PROGRESS;
 349
 350         return desc;
 351 }
 352
 353 static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan,
 354                                             struct scatterlist *dst_sg,
 355                                             unsigned int dst_nents,
 356                                             struct scatterlist *src_sg,
 357                                             unsigned int src_nents,
 358                                             unsigned long flags)
 359 {
 360         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 361                                                  dma_chan);
 362         struct ccp_device *ccp = chan->ccp;
 363         struct ccp_dma_desc *desc;
 364         struct ccp_dma_cmd *cmd;
 365         struct ccp_cmd *ccp_cmd;
 366         struct ccp_passthru_nomap_engine *ccp_pt;
 367         unsigned int src_offset, src_len;
 368         unsigned int dst_offset, dst_len;
 369         unsigned int len;
 370         unsigned long sflags;
 371         size_t total_len;
 372
 373         if (!dst_sg || !src_sg)
 374                 return NULL;
 375
 376         if (!dst_nents || !src_nents)
 377                 return NULL;
 378
 379         desc = ccp_alloc_dma_desc(chan, flags);
 380         if (!desc)
 381                 return NULL;
 382
 383         total_len = 0;
 384
 385         src_len = sg_dma_len(src_sg);
 386         src_offset = 0;
 387
 388         dst_len = sg_dma_len(dst_sg);
 389         dst_offset = 0;
 390
 391         while (true) {
 392                 if (!src_len) {
 393                         src_nents--;
 394                         if (!src_nents)
 395                                 break;
 396
 397                         src_sg = sg_next(src_sg);
 398                         if (!src_sg)
 399                                 break;
 400
 401                         src_len = sg_dma_len(src_sg);
 402                         src_offset = 0;
 403                         continue;
 404                 }
 405
 406                 if (!dst_len) {
 407                         dst_nents--;
 408                         if (!dst_nents)
 409                                 break;
 410
 411                         dst_sg = sg_next(dst_sg);
 412                         if (!dst_sg)
 413                                 break;
 414
 415                         dst_len = sg_dma_len(dst_sg);
 416                         dst_offset = 0;
 417                         continue;
 418                 }
 419
 420                 len = min(dst_len, src_len);
 421
 422                 cmd = ccp_alloc_dma_cmd(chan);
 423                 if (!cmd)
 424                         goto err;
 425
 426                 ccp_cmd = &cmd->ccp_cmd;
 427                 ccp_cmd->ccp = chan->ccp;
 428                 ccp_pt = &ccp_cmd->u.passthru_nomap;
 429                 ccp_cmd->flags = CCP_CMD_MAY_BACKLOG;
 430                 ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP;
 431                 ccp_cmd->engine = CCP_ENGINE_PASSTHRU;
 432                 ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
 433                 ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
 434                 ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset;
 435                 ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset;
 436                 ccp_pt->src_len = len;
 437                 ccp_pt->final = 1;
 438                 ccp_cmd->callback = ccp_cmd_callback;
 439                 ccp_cmd->data = desc;
 440
 441                 list_add_tail(&cmd->entry, &desc->pending);
 442
 443                 dev_dbg(ccp->dev,
 444                         "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__,
 445                         cmd, &ccp_pt->src_dma,
 446                         &ccp_pt->dst_dma, ccp_pt->src_len);
 447
 448                 total_len += len;
 449
 450                 src_len -= len;
 451                 src_offset += len;
 452
 453                 dst_len -= len;
 454                 dst_offset += len;
 455         }
 456
 457         desc->len = total_len;
 458
 459         if (list_empty(&desc->pending))
 460                 goto err;
 461
 462         dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
 463
 464         spin_lock_irqsave(&chan->lock, sflags);
 465
 466         list_add_tail(&desc->entry, &chan->created);
 467
 468         spin_unlock_irqrestore(&chan->lock, sflags);
 469
 470         return desc;
 471
 472 err:
 473         ccp_free_cmd_resources(ccp, &desc->pending);
 474         kmem_cache_free(ccp->dma_desc_cache, desc);
 475
 476         return NULL;
 477 }
 478
 479 static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy(
 480         struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len,
 481         unsigned long flags)
 482 {
 483         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 484                                                  dma_chan);
 485         struct ccp_dma_desc *desc;
 486         struct scatterlist dst_sg, src_sg;
 487
 488         dev_dbg(chan->ccp->dev,
 489                 "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n",
 490                 __func__, &src, &dst, len, flags);
 491
 492         sg_init_table(&dst_sg, 1);
 493         sg_dma_address(&dst_sg) = dst;
 494         sg_dma_len(&dst_sg) = len;
 495
 496         sg_init_table(&src_sg, 1);
 497         sg_dma_address(&src_sg) = src;
 498         sg_dma_len(&src_sg) = len;
 499
 500         desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags);
 501         if (!desc)
 502                 return NULL;
 503
 504         return &desc->tx_desc;
 505 }
 506
 507 static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt(
 508         struct dma_chan *dma_chan, unsigned long flags)
 509 {
 510         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 511                                                  dma_chan);
 512         struct ccp_dma_desc *desc;
 513
 514         desc = ccp_alloc_dma_desc(chan, flags);
 515         if (!desc)
 516                 return NULL;
 517
 518         return &desc->tx_desc;
 519 }
 520
 521 static void ccp_issue_pending(struct dma_chan *dma_chan)
 522 {
 523         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 524                                                  dma_chan);
 525         struct ccp_dma_desc *desc;
 526         unsigned long flags;
 527
 528         dev_dbg(chan->ccp->dev, "%s\n", __func__);
 529
 530         spin_lock_irqsave(&chan->lock, flags);
 531
 532         desc = __ccp_pending_to_active(chan);
 533
 534         spin_unlock_irqrestore(&chan->lock, flags);
 535
 536         /* If there was nothing active, start processing */
 537         if (desc)
 538                 ccp_cmd_callback(desc, 0);
 539 }
 540
 541 static enum dma_status ccp_tx_status(struct dma_chan *dma_chan,
 542                                      dma_cookie_t cookie,
 543                                      struct dma_tx_state *state)
 544 {
 545         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 546                                                  dma_chan);
 547         struct ccp_dma_desc *desc;
 548         enum dma_status ret;
 549         unsigned long flags;
 550
 551         if (chan->status == DMA_PAUSED) {
 552                 ret = DMA_PAUSED;
 553                 goto out;
 554         }
 555
 556         ret = dma_cookie_status(dma_chan, cookie, state);
 557         if (ret == DMA_COMPLETE) {
 558                 spin_lock_irqsave(&chan->lock, flags);
 559
 560                 /* Get status from complete chain, if still there */
 561                 list_for_each_entry(desc, &chan->complete, entry) {
 562                         if (desc->tx_desc.cookie != cookie)
 563                                 continue;
 564
 565                         ret = desc->status;
 566                         break;
 567                 }
 568
 569                 spin_unlock_irqrestore(&chan->lock, flags);
 570         }
 571
 572 out:
 573         dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret);
 574
 575         return ret;
 576 }
 577
 578 static int ccp_pause(struct dma_chan *dma_chan)
 579 {
 580         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 581                                                  dma_chan);
 582
 583         chan->status = DMA_PAUSED;
 584
 585         /*TODO: Wait for active DMA to complete before returning? */
 586
 587         return 0;
 588 }
 589
 590 static int ccp_resume(struct dma_chan *dma_chan)
 591 {
 592         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 593                                                  dma_chan);
 594         struct ccp_dma_desc *desc;
 595         unsigned long flags;
 596
 597         spin_lock_irqsave(&chan->lock, flags);
 598
 599         desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
 600                                         entry);
 601
 602         spin_unlock_irqrestore(&chan->lock, flags);
 603
 604         /* Indicate the channel is running again */
 605         chan->status = DMA_IN_PROGRESS;
 606
 607         /* If there was something active, re-start */
 608         if (desc)
 609                 ccp_cmd_callback(desc, 0);
 610
 611         return 0;
 612 }
 613
 614 static int ccp_terminate_all(struct dma_chan *dma_chan)
 615 {
 616         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 617                                                  dma_chan);
 618         unsigned long flags;
 619
 620         dev_dbg(chan->ccp->dev, "%s\n", __func__);
 621
 622         /*TODO: Wait for active DMA to complete before continuing */
 623
 624         spin_lock_irqsave(&chan->lock, flags);
 625
 626         /*TODO: Purge the complete list? */
 627         ccp_free_desc_resources(chan->ccp, &chan->active);
 628         ccp_free_desc_resources(chan->ccp, &chan->pending);
 629         ccp_free_desc_resources(chan->ccp, &chan->created);
 630
 631         spin_unlock_irqrestore(&chan->lock, flags);
 632
 633         return 0;
 634 }
 635
 636 int ccp_dmaengine_register(struct ccp_device *ccp)
 637 {
 638         struct ccp_dma_chan *chan;
 639         struct dma_device *dma_dev = &ccp->dma_dev;
 640         struct dma_chan *dma_chan;
 641         char *dma_cmd_cache_name;
 642         char *dma_desc_cache_name;
 643         unsigned int i;
 644         int ret;
 645
 646         if (!dmaengine)
 647                 return 0;
 648
 649         ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count,
 650                                          sizeof(*(ccp->ccp_dma_chan)),
 651                                          GFP_KERNEL);
 652         if (!ccp->ccp_dma_chan)
 653                 return -ENOMEM;
 654
 655         dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
 656                                             "%s-dmaengine-cmd-cache",
 657                                             ccp->name);
 658         if (!dma_cmd_cache_name)
 659                 return -ENOMEM;
 660
 661         ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name,
 662                                                sizeof(struct ccp_dma_cmd),
 663                                                sizeof(void *),
 664                                                SLAB_HWCACHE_ALIGN, NULL);
 665         if (!ccp->dma_cmd_cache)
 666                 return -ENOMEM;
 667
 668         dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
 669                                              "%s-dmaengine-desc-cache",
 670                                              ccp->name);
 671         if (!dma_desc_cache_name) {
 672                 ret = -ENOMEM;
 673                 goto err_cache;
 674         }
 675
 676         ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name,
 677                                                 sizeof(struct ccp_dma_desc),
 678                                                 sizeof(void *),
 679                                                 SLAB_HWCACHE_ALIGN, NULL);
 680         if (!ccp->dma_desc_cache) {
 681                 ret = -ENOMEM;
 682                 goto err_cache;
 683         }
 684
 685         dma_dev->dev = ccp->dev;
 686         dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
 687         dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
 688         dma_dev->directions = DMA_MEM_TO_MEM;
 689         dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 690         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
 691         dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
 692
 693         /* The DMA channels for this device can be set to public or private,
 694          * and overridden by the module parameter dma_chan_attr.
 695          * Default: according to the value in vdata (dma_chan_attr=0)
 696          * dma_chan_attr=0x1: all channels private (override vdata)
 697          * dma_chan_attr=0x2: all channels public (override vdata)
 698          */
 699         if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE)
 700                 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
 701
 702         INIT_LIST_HEAD(&dma_dev->channels);
 703         for (i = 0; i < ccp->cmd_q_count; i++) {
 704                 chan = ccp->ccp_dma_chan + i;
 705                 dma_chan = &chan->dma_chan;
 706
 707                 chan->ccp = ccp;
 708
 709                 spin_lock_init(&chan->lock);
 710                 INIT_LIST_HEAD(&chan->created);
 711                 INIT_LIST_HEAD(&chan->pending);
 712                 INIT_LIST_HEAD(&chan->active);
 713                 INIT_LIST_HEAD(&chan->complete);
 714
 715                 tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup,
 716                              (unsigned long)chan);
 717
 718                 dma_chan->device = dma_dev;
 719                 dma_cookie_init(dma_chan);
 720
 721                 list_add_tail(&dma_chan->device_node, &dma_dev->channels);
 722         }
 723
 724         dma_dev->device_free_chan_resources = ccp_free_chan_resources;
 725         dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy;
 726         dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt;
 727         dma_dev->device_issue_pending = ccp_issue_pending;
 728         dma_dev->device_tx_status = ccp_tx_status;
 729         dma_dev->device_pause = ccp_pause;
 730         dma_dev->device_resume = ccp_resume;
 731         dma_dev->device_terminate_all = ccp_terminate_all;
 732
 733         ret = dma_async_device_register(dma_dev);
 734         if (ret)
 735                 goto err_reg;
 736
 737         return 0;
 738
 739 err_reg:
 740         kmem_cache_destroy(ccp->dma_desc_cache);
 741
 742 err_cache:
 743         kmem_cache_destroy(ccp->dma_cmd_cache);
 744
 745         return ret;
 746 }
 747
 748 void ccp_dmaengine_unregister(struct ccp_device *ccp)
 749 {
 750         struct dma_device *dma_dev = &ccp->dma_dev;
 751
 752         if (!dmaengine)
 753                 return;
 754
 755         dma_async_device_unregister(dma_dev);
 756
 757         kmem_cache_destroy(ccp->dma_desc_cache);
 758         kmem_cache_destroy(ccp->dma_cmd_cache);
 759 }