drivers/crypto/ccp/ccp-dev-v3.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Cryptographic Coprocessor (CCP) driver
   4  *
   5  * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
   6  *
   7  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   8  * Author: Gary R Hook <gary.hook@amd.com>
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/kernel.h>
  13 #include <linux/pci.h>
  14 #include <linux/kthread.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/ccp.h>
  17
  18 #include "ccp-dev.h"
  19
  20 static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count)
  21 {
  22         int start;
  23         struct ccp_device *ccp = cmd_q->ccp;
  24
  25         for (;;) {
  26                 mutex_lock(&ccp->sb_mutex);
  27
  28                 start = (u32)bitmap_find_next_zero_area(ccp->sb,
  29                                                         ccp->sb_count,
  30                                                         ccp->sb_start,
  31                                                         count, 0);
  32                 if (start <= ccp->sb_count) {
  33                         bitmap_set(ccp->sb, start, count);
  34
  35                         mutex_unlock(&ccp->sb_mutex);
  36                         break;
  37                 }
  38
  39                 ccp->sb_avail = 0;
  40
  41                 mutex_unlock(&ccp->sb_mutex);
  42
  43                 /* Wait for KSB entries to become available */
  44                 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
  45                         return 0;
  46         }
  47
  48         return KSB_START + start;
  49 }
  50
  51 static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start,
  52                          unsigned int count)
  53 {
  54         struct ccp_device *ccp = cmd_q->ccp;
  55
  56         if (!start)
  57                 return;
  58
  59         mutex_lock(&ccp->sb_mutex);
  60
  61         bitmap_clear(ccp->sb, start - KSB_START, count);
  62
  63         ccp->sb_avail = 1;
  64
  65         mutex_unlock(&ccp->sb_mutex);
  66
  67         wake_up_interruptible_all(&ccp->sb_queue);
  68 }
  69
  70 static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q)
  71 {
  72         return CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
  73 }
  74
  75 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
  76 {
  77         struct ccp_cmd_queue *cmd_q = op->cmd_q;
  78         struct ccp_device *ccp = cmd_q->ccp;
  79         void __iomem *cr_addr;
  80         u32 cr0, cmd;
  81         unsigned int i;
  82         int ret = 0;
  83
  84         /* We could read a status register to see how many free slots
  85          * are actually available, but reading that register resets it
  86          * and you could lose some error information.
  87          */
  88         cmd_q->free_slots--;
  89
  90         cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
  91               | (op->jobid << REQ0_JOBID_SHIFT)
  92               | REQ0_WAIT_FOR_WRITE;
  93
  94         if (op->soc)
  95                 cr0 |= REQ0_STOP_ON_COMPLETE
  96                        | REQ0_INT_ON_COMPLETE;
  97
  98         if (op->ioc || !cmd_q->free_slots)
  99                 cr0 |= REQ0_INT_ON_COMPLETE;
 100
 101         /* Start at CMD_REQ1 */
 102         cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
 103
 104         mutex_lock(&ccp->req_mutex);
 105
 106         /* Write CMD_REQ1 through CMD_REQx first */
 107         for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
 108                 iowrite32(*(cr + i), cr_addr);
 109
 110         /* Tell the CCP to start */
 111         wmb();
 112         iowrite32(cr0, ccp->io_regs + CMD_REQ0);
 113
 114         mutex_unlock(&ccp->req_mutex);
 115
 116         if (cr0 & REQ0_INT_ON_COMPLETE) {
 117                 /* Wait for the job to complete */
 118                 ret = wait_event_interruptible(cmd_q->int_queue,
 119                                                cmd_q->int_rcvd);
 120                 if (ret || cmd_q->cmd_error) {
 121                         /* On error delete all related jobs from the queue */
 122                         cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
 123                               | op->jobid;
 124                         if (cmd_q->cmd_error)
 125                                 ccp_log_error(cmd_q->ccp,
 126                                               cmd_q->cmd_error);
 127
 128                         iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 129
 130                         if (!ret)
 131                                 ret = -EIO;
 132                 } else if (op->soc) {
 133                         /* Delete just head job from the queue on SoC */
 134                         cmd = DEL_Q_ACTIVE
 135                               | (cmd_q->id << DEL_Q_ID_SHIFT)
 136                               | op->jobid;
 137
 138                         iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 139                 }
 140
 141                 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
 142
 143                 cmd_q->int_rcvd = 0;
 144         }
 145
 146         return ret;
 147 }
 148
 149 static int ccp_perform_aes(struct ccp_op *op)
 150 {
 151         u32 cr[6];
 152
 153         /* Fill out the register contents for REQ1 through REQ6 */
 154         cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
 155                 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
 156                 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
 157                 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
 158                 | (op->sb_key << REQ1_KEY_KSB_SHIFT);
 159         cr[1] = op->src.u.dma.length - 1;
 160         cr[2] = ccp_addr_lo(&op->src.u.dma);
 161         cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
 162                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 163                 | ccp_addr_hi(&op->src.u.dma);
 164         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 165         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 166                 | ccp_addr_hi(&op->dst.u.dma);
 167
 168         if (op->u.aes.mode == CCP_AES_MODE_CFB)
 169                 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
 170
 171         if (op->eom)
 172                 cr[0] |= REQ1_EOM;
 173
 174         if (op->init)
 175                 cr[0] |= REQ1_INIT;
 176
 177         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 178 }
 179
 180 static int ccp_perform_xts_aes(struct ccp_op *op)
 181 {
 182         u32 cr[6];
 183
 184         /* Fill out the register contents for REQ1 through REQ6 */
 185         cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
 186                 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
 187                 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
 188                 | (op->sb_key << REQ1_KEY_KSB_SHIFT);
 189         cr[1] = op->src.u.dma.length - 1;
 190         cr[2] = ccp_addr_lo(&op->src.u.dma);
 191         cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
 192                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 193                 | ccp_addr_hi(&op->src.u.dma);
 194         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 195         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 196                 | ccp_addr_hi(&op->dst.u.dma);
 197
 198         if (op->eom)
 199                 cr[0] |= REQ1_EOM;
 200
 201         if (op->init)
 202                 cr[0] |= REQ1_INIT;
 203
 204         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 205 }
 206
 207 static int ccp_perform_sha(struct ccp_op *op)
 208 {
 209         u32 cr[6];
 210
 211         /* Fill out the register contents for REQ1 through REQ6 */
 212         cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
 213                 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
 214                 | REQ1_INIT;
 215         cr[1] = op->src.u.dma.length - 1;
 216         cr[2] = ccp_addr_lo(&op->src.u.dma);
 217         cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
 218                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 219                 | ccp_addr_hi(&op->src.u.dma);
 220
 221         if (op->eom) {
 222                 cr[0] |= REQ1_EOM;
 223                 cr[4] = lower_32_bits(op->u.sha.msg_bits);
 224                 cr[5] = upper_32_bits(op->u.sha.msg_bits);
 225         } else {
 226                 cr[4] = 0;
 227                 cr[5] = 0;
 228         }
 229
 230         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 231 }
 232
 233 static int ccp_perform_rsa(struct ccp_op *op)
 234 {
 235         u32 cr[6];
 236
 237         /* Fill out the register contents for REQ1 through REQ6 */
 238         cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
 239                 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
 240                 | (op->sb_key << REQ1_KEY_KSB_SHIFT)
 241                 | REQ1_EOM;
 242         cr[1] = op->u.rsa.input_len - 1;
 243         cr[2] = ccp_addr_lo(&op->src.u.dma);
 244         cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
 245                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 246                 | ccp_addr_hi(&op->src.u.dma);
 247         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 248         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 249                 | ccp_addr_hi(&op->dst.u.dma);
 250
 251         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 252 }
 253
 254 static int ccp_perform_passthru(struct ccp_op *op)
 255 {
 256         u32 cr[6];
 257
 258         /* Fill out the register contents for REQ1 through REQ6 */
 259         cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
 260                 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
 261                 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
 262
 263         if (op->src.type == CCP_MEMTYPE_SYSTEM)
 264                 cr[1] = op->src.u.dma.length - 1;
 265         else
 266                 cr[1] = op->dst.u.dma.length - 1;
 267
 268         if (op->src.type == CCP_MEMTYPE_SYSTEM) {
 269                 cr[2] = ccp_addr_lo(&op->src.u.dma);
 270                 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 271                         | ccp_addr_hi(&op->src.u.dma);
 272
 273                 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
 274                         cr[3] |= (op->sb_key << REQ4_KSB_SHIFT);
 275         } else {
 276                 cr[2] = op->src.u.sb * CCP_SB_BYTES;
 277                 cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT);
 278         }
 279
 280         if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
 281                 cr[4] = ccp_addr_lo(&op->dst.u.dma);
 282                 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 283                         | ccp_addr_hi(&op->dst.u.dma);
 284         } else {
 285                 cr[4] = op->dst.u.sb * CCP_SB_BYTES;
 286                 cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT);
 287         }
 288
 289         if (op->eom)
 290                 cr[0] |= REQ1_EOM;
 291
 292         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 293 }
 294
 295 static int ccp_perform_ecc(struct ccp_op *op)
 296 {
 297         u32 cr[6];
 298
 299         /* Fill out the register contents for REQ1 through REQ6 */
 300         cr[0] = REQ1_ECC_AFFINE_CONVERT
 301                 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
 302                 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
 303                 | REQ1_EOM;
 304         cr[1] = op->src.u.dma.length - 1;
 305         cr[2] = ccp_addr_lo(&op->src.u.dma);
 306         cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 307                 | ccp_addr_hi(&op->src.u.dma);
 308         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 309         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 310                 | ccp_addr_hi(&op->dst.u.dma);
 311
 312         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 313 }
 314
 315 static void ccp_disable_queue_interrupts(struct ccp_device *ccp)
 316 {
 317         iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
 318 }
 319
 320 static void ccp_enable_queue_interrupts(struct ccp_device *ccp)
 321 {
 322         iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG);
 323 }
 324
 325 static void ccp_irq_bh(unsigned long data)
 326 {
 327         struct ccp_device *ccp = (struct ccp_device *)data;
 328         struct ccp_cmd_queue *cmd_q;
 329         u32 q_int, status;
 330         unsigned int i;
 331
 332         status = ioread32(ccp->io_regs + IRQ_STATUS_REG);
 333
 334         for (i = 0; i < ccp->cmd_q_count; i++) {
 335                 cmd_q = &ccp->cmd_q[i];
 336
 337                 q_int = status & (cmd_q->int_ok | cmd_q->int_err);
 338                 if (q_int) {
 339                         cmd_q->int_status = status;
 340                         cmd_q->q_status = ioread32(cmd_q->reg_status);
 341                         cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
 342
 343                         /* On error, only save the first error value */
 344                         if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)
 345                                 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
 346
 347                         cmd_q->int_rcvd = 1;
 348
 349                         /* Acknowledge the interrupt and wake the kthread */
 350                         iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
 351                         wake_up_interruptible(&cmd_q->int_queue);
 352                 }
 353         }
 354         ccp_enable_queue_interrupts(ccp);
 355 }
 356
 357 static irqreturn_t ccp_irq_handler(int irq, void *data)
 358 {
 359         struct ccp_device *ccp = (struct ccp_device *)data;
 360
 361         ccp_disable_queue_interrupts(ccp);
 362         if (ccp->use_tasklet)
 363                 tasklet_schedule(&ccp->irq_tasklet);
 364         else
 365                 ccp_irq_bh((unsigned long)ccp);
 366
 367         return IRQ_HANDLED;
 368 }
 369
 370 static int ccp_init(struct ccp_device *ccp)
 371 {
 372         struct device *dev = ccp->dev;
 373         struct ccp_cmd_queue *cmd_q;
 374         struct dma_pool *dma_pool;
 375         char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
 376         unsigned int qmr, i;
 377         int ret;
 378
 379         /* Find available queues */
 380         ccp->qim = 0;
 381         qmr = ioread32(ccp->io_regs + Q_MASK_REG);
 382         for (i = 0; i < MAX_HW_QUEUES; i++) {
 383                 if (!(qmr & (1 << i)))
 384                         continue;
 385
 386                 /* Allocate a dma pool for this queue */
 387                 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
 388                          ccp->name, i);
 389                 dma_pool = dma_pool_create(dma_pool_name, dev,
 390                                            CCP_DMAPOOL_MAX_SIZE,
 391                                            CCP_DMAPOOL_ALIGN, 0);
 392                 if (!dma_pool) {
 393                         dev_err(dev, "unable to allocate dma pool\n");
 394                         ret = -ENOMEM;
 395                         goto e_pool;
 396                 }
 397
 398                 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
 399                 ccp->cmd_q_count++;
 400
 401                 cmd_q->ccp = ccp;
 402                 cmd_q->id = i;
 403                 cmd_q->dma_pool = dma_pool;
 404
 405                 /* Reserve 2 KSB regions for the queue */
 406                 cmd_q->sb_key = KSB_START + ccp->sb_start++;
 407                 cmd_q->sb_ctx = KSB_START + ccp->sb_start++;
 408                 ccp->sb_count -= 2;
 409
 410                 /* Preset some register values and masks that are queue
 411                  * number dependent
 412                  */
 413                 cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +
 414                                     (CMD_Q_STATUS_INCR * i);
 415                 cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
 416                                         (CMD_Q_STATUS_INCR * i);
 417                 cmd_q->int_ok = 1 << (i * 2);
 418                 cmd_q->int_err = 1 << ((i * 2) + 1);
 419
 420                 cmd_q->free_slots = ccp_get_free_slots(cmd_q);
 421
 422                 init_waitqueue_head(&cmd_q->int_queue);
 423
 424                 /* Build queue interrupt mask (two interrupts per queue) */
 425                 ccp->qim |= cmd_q->int_ok | cmd_q->int_err;
 426
 427 #ifdef CONFIG_ARM64
 428                 /* For arm64 set the recommended queue cache settings */
 429                 iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +
 430                           (CMD_Q_CACHE_INC * i));
 431 #endif
 432
 433                 dev_dbg(dev, "queue #%u available\n", i);
 434         }
 435         if (ccp->cmd_q_count == 0) {
 436                 dev_notice(dev, "no command queues available\n");
 437                 ret = -EIO;
 438                 goto e_pool;
 439         }
 440         dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
 441
 442         /* Disable and clear interrupts until ready */
 443         ccp_disable_queue_interrupts(ccp);
 444         for (i = 0; i < ccp->cmd_q_count; i++) {
 445                 cmd_q = &ccp->cmd_q[i];
 446
 447                 ioread32(cmd_q->reg_int_status);
 448                 ioread32(cmd_q->reg_status);
 449         }
 450         iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
 451
 452         /* Request an irq */
 453         ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp);
 454         if (ret) {
 455                 dev_err(dev, "unable to allocate an IRQ\n");
 456                 goto e_pool;
 457         }
 458
 459         /* Initialize the ISR tasklet? */
 460         if (ccp->use_tasklet)
 461                 tasklet_init(&ccp->irq_tasklet, ccp_irq_bh,
 462                              (unsigned long)ccp);
 463
 464         dev_dbg(dev, "Starting threads...\n");
 465         /* Create a kthread for each queue */
 466         for (i = 0; i < ccp->cmd_q_count; i++) {
 467                 struct task_struct *kthread;
 468
 469                 cmd_q = &ccp->cmd_q[i];
 470
 471                 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
 472                                          "%s-q%u", ccp->name, cmd_q->id);
 473                 if (IS_ERR(kthread)) {
 474                         dev_err(dev, "error creating queue thread (%ld)\n",
 475                                 PTR_ERR(kthread));
 476                         ret = PTR_ERR(kthread);
 477                         goto e_kthread;
 478                 }
 479
 480                 cmd_q->kthread = kthread;
 481                 wake_up_process(kthread);
 482         }
 483
 484         dev_dbg(dev, "Enabling interrupts...\n");
 485         /* Enable interrupts */
 486         ccp_enable_queue_interrupts(ccp);
 487
 488         dev_dbg(dev, "Registering device...\n");
 489         ccp_add_device(ccp);
 490
 491         ret = ccp_register_rng(ccp);
 492         if (ret)
 493                 goto e_kthread;
 494
 495         /* Register the DMA engine support */
 496         ret = ccp_dmaengine_register(ccp);
 497         if (ret)
 498                 goto e_hwrng;
 499
 500         return 0;
 501
 502 e_hwrng:
 503         ccp_unregister_rng(ccp);
 504
 505 e_kthread:
 506         for (i = 0; i < ccp->cmd_q_count; i++)
 507                 if (ccp->cmd_q[i].kthread)
 508                         kthread_stop(ccp->cmd_q[i].kthread);
 509
 510         sp_free_ccp_irq(ccp->sp, ccp);
 511
 512 e_pool:
 513         for (i = 0; i < ccp->cmd_q_count; i++)
 514                 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
 515
 516         return ret;
 517 }
 518
 519 static void ccp_destroy(struct ccp_device *ccp)
 520 {
 521         struct ccp_cmd_queue *cmd_q;
 522         struct ccp_cmd *cmd;
 523         unsigned int i;
 524
 525         /* Unregister the DMA engine */
 526         ccp_dmaengine_unregister(ccp);
 527
 528         /* Unregister the RNG */
 529         ccp_unregister_rng(ccp);
 530
 531         /* Remove this device from the list of available units */
 532         ccp_del_device(ccp);
 533
 534         /* Disable and clear interrupts */
 535         ccp_disable_queue_interrupts(ccp);
 536         for (i = 0; i < ccp->cmd_q_count; i++) {
 537                 cmd_q = &ccp->cmd_q[i];
 538
 539                 ioread32(cmd_q->reg_int_status);
 540                 ioread32(cmd_q->reg_status);
 541         }
 542         iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
 543
 544         /* Stop the queue kthreads */
 545         for (i = 0; i < ccp->cmd_q_count; i++)
 546                 if (ccp->cmd_q[i].kthread)
 547                         kthread_stop(ccp->cmd_q[i].kthread);
 548
 549         sp_free_ccp_irq(ccp->sp, ccp);
 550
 551         for (i = 0; i < ccp->cmd_q_count; i++)
 552                 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
 553
 554         /* Flush the cmd and backlog queue */
 555         while (!list_empty(&ccp->cmd)) {
 556                 /* Invoke the callback directly with an error code */
 557                 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
 558                 list_del(&cmd->entry);
 559                 cmd->callback(cmd->data, -ENODEV);
 560         }
 561         while (!list_empty(&ccp->backlog)) {
 562                 /* Invoke the callback directly with an error code */
 563                 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
 564                 list_del(&cmd->entry);
 565                 cmd->callback(cmd->data, -ENODEV);
 566         }
 567 }
 568
 569 static const struct ccp_actions ccp3_actions = {
 570         .aes = ccp_perform_aes,
 571         .xts_aes = ccp_perform_xts_aes,
 572         .des3 = NULL,
 573         .sha = ccp_perform_sha,
 574         .rsa = ccp_perform_rsa,
 575         .passthru = ccp_perform_passthru,
 576         .ecc = ccp_perform_ecc,
 577         .sballoc = ccp_alloc_ksb,
 578         .sbfree = ccp_free_ksb,
 579         .init = ccp_init,
 580         .destroy = ccp_destroy,
 581         .get_free_slots = ccp_get_free_slots,
 582         .irqhandler = ccp_irq_handler,
 583 };
 584
 585 const struct ccp_vdata ccpv3_platform = {
 586         .version = CCP_VERSION(3, 0),
 587         .setup = NULL,
 588         .perform = &ccp3_actions,
 589         .offset = 0,
 590 };
 591
 592 const struct ccp_vdata ccpv3 = {
 593         .version = CCP_VERSION(3, 0),
 594         .setup = NULL,
 595         .perform = &ccp3_actions,
 596         .offset = 0x20000,
 597         .rsamax = CCP_RSA_MAX_WIDTH,
 598 };