drivers/crypto/ccp/ccp-ops.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Cryptographic Coprocessor (CCP) driver
   4  *
   5  * Copyright (C) 2013,2018 Advanced Micro Devices, Inc.
   6  *
   7  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   8  * Author: Gary R Hook <gary.hook@amd.com>
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/kernel.h>
  13 #include <linux/pci.h>
  14 #include <linux/interrupt.h>
  15 #include <crypto/scatterwalk.h>
  16 #include <crypto/des.h>
  17 #include <linux/ccp.h>
  18
  19 #include "ccp-dev.h"
  20
  21 /* SHA initial context values */
  22 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  23         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  24         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  25         cpu_to_be32(SHA1_H4),
  26 };
  27
  28 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  29         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  30         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  31         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  32         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  33 };
  34
  35 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  36         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  37         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  38         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  39         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  40 };
  41
  42 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  43         cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  44         cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  45         cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  46         cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  47 };
  48
  49 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  50         cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  51         cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  52         cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  53         cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  54 };
  55
  56 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  57                                         ccp_gen_jobid(ccp) : 0)
  58
  59 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  60 {
  61         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  62 }
  63
  64 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  65 {
  66         if (wa->dma_count)
  67                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
  68
  69         wa->dma_count = 0;
  70 }
  71
  72 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  73                                 struct scatterlist *sg, u64 len,
  74                                 enum dma_data_direction dma_dir)
  75 {
  76         memset(wa, 0, sizeof(*wa));
  77
  78         wa->sg = sg;
  79         if (!sg)
  80                 return 0;
  81
  82         wa->nents = sg_nents_for_len(sg, len);
  83         if (wa->nents < 0)
  84                 return wa->nents;
  85
  86         wa->bytes_left = len;
  87         wa->sg_used = 0;
  88
  89         if (len == 0)
  90                 return 0;
  91
  92         if (dma_dir == DMA_NONE)
  93                 return 0;
  94
  95         wa->dma_sg = sg;
  96         wa->dma_dev = dev;
  97         wa->dma_dir = dma_dir;
  98         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
  99         if (!wa->dma_count)
 100                 return -ENOMEM;
 101
 102         return 0;
 103 }
 104
 105 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 106 {
 107         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 108
 109         if (!wa->sg)
 110                 return;
 111
 112         wa->sg_used += nbytes;
 113         wa->bytes_left -= nbytes;
 114         if (wa->sg_used == wa->sg->length) {
 115                 wa->sg = sg_next(wa->sg);
 116                 wa->sg_used = 0;
 117         }
 118 }
 119
 120 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 121 {
 122         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 123                 if (wa->address)
 124                         dma_pool_free(wa->dma_pool, wa->address,
 125                                       wa->dma.address);
 126         } else {
 127                 if (wa->dma.address)
 128                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 129                                          wa->dma.dir);
 130                 kfree(wa->address);
 131         }
 132
 133         wa->address = NULL;
 134         wa->dma.address = 0;
 135 }
 136
 137 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 138                                 struct ccp_cmd_queue *cmd_q,
 139                                 unsigned int len,
 140                                 enum dma_data_direction dir)
 141 {
 142         memset(wa, 0, sizeof(*wa));
 143
 144         if (!len)
 145                 return 0;
 146
 147         wa->dev = cmd_q->ccp->dev;
 148         wa->length = len;
 149
 150         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 151                 wa->dma_pool = cmd_q->dma_pool;
 152
 153                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 154                                              &wa->dma.address);
 155                 if (!wa->address)
 156                         return -ENOMEM;
 157
 158                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 159
 160                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 161         } else {
 162                 wa->address = kzalloc(len, GFP_KERNEL);
 163                 if (!wa->address)
 164                         return -ENOMEM;
 165
 166                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 167                                                  dir);
 168                 if (dma_mapping_error(wa->dev, wa->dma.address))
 169                         return -ENOMEM;
 170
 171                 wa->dma.length = len;
 172         }
 173         wa->dma.dir = dir;
 174
 175         return 0;
 176 }
 177
 178 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 179                            struct scatterlist *sg, unsigned int sg_offset,
 180                            unsigned int len)
 181 {
 182         WARN_ON(!wa->address);
 183
 184         if (len > (wa->length - wa_offset))
 185                 return -EINVAL;
 186
 187         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 188                                  0);
 189         return 0;
 190 }
 191
 192 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 193                             struct scatterlist *sg, unsigned int sg_offset,
 194                             unsigned int len)
 195 {
 196         WARN_ON(!wa->address);
 197
 198         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 199                                  1);
 200 }
 201
 202 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 203                                    unsigned int wa_offset,
 204                                    struct scatterlist *sg,
 205                                    unsigned int sg_offset,
 206                                    unsigned int len)
 207 {
 208         u8 *p, *q;
 209         int     rc;
 210
 211         rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 212         if (rc)
 213                 return rc;
 214
 215         p = wa->address + wa_offset;
 216         q = p + len - 1;
 217         while (p < q) {
 218                 *p = *p ^ *q;
 219                 *q = *p ^ *q;
 220                 *p = *p ^ *q;
 221                 p++;
 222                 q--;
 223         }
 224         return 0;
 225 }
 226
 227 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 228                                     unsigned int wa_offset,
 229                                     struct scatterlist *sg,
 230                                     unsigned int sg_offset,
 231                                     unsigned int len)
 232 {
 233         u8 *p, *q;
 234
 235         p = wa->address + wa_offset;
 236         q = p + len - 1;
 237         while (p < q) {
 238                 *p = *p ^ *q;
 239                 *q = *p ^ *q;
 240                 *p = *p ^ *q;
 241                 p++;
 242                 q--;
 243         }
 244
 245         ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 246 }
 247
 248 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 249 {
 250         ccp_dm_free(&data->dm_wa);
 251         ccp_sg_free(&data->sg_wa);
 252 }
 253
 254 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 255                          struct scatterlist *sg, u64 sg_len,
 256                          unsigned int dm_len,
 257                          enum dma_data_direction dir)
 258 {
 259         int ret;
 260
 261         memset(data, 0, sizeof(*data));
 262
 263         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 264                                    dir);
 265         if (ret)
 266                 goto e_err;
 267
 268         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 269         if (ret)
 270                 goto e_err;
 271
 272         return 0;
 273
 274 e_err:
 275         ccp_free_data(data, cmd_q);
 276
 277         return ret;
 278 }
 279
 280 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 281 {
 282         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 283         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 284         unsigned int buf_count, nbytes;
 285
 286         /* Clear the buffer if setting it */
 287         if (!from)
 288                 memset(dm_wa->address, 0, dm_wa->length);
 289
 290         if (!sg_wa->sg)
 291                 return 0;
 292
 293         /* Perform the copy operation
 294          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 295          *   an unsigned int
 296          */
 297         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 298         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 299                                  nbytes, from);
 300
 301         /* Update the structures and generate the count */
 302         buf_count = 0;
 303         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 304                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 305                              dm_wa->length - buf_count);
 306                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 307
 308                 buf_count += nbytes;
 309                 ccp_update_sg_workarea(sg_wa, nbytes);
 310         }
 311
 312         return buf_count;
 313 }
 314
 315 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 316 {
 317         return ccp_queue_buf(data, 0);
 318 }
 319
 320 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 321 {
 322         return ccp_queue_buf(data, 1);
 323 }
 324
 325 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 326                              struct ccp_op *op, unsigned int block_size,
 327                              bool blocksize_op)
 328 {
 329         unsigned int sg_src_len, sg_dst_len, op_len;
 330
 331         /* The CCP can only DMA from/to one address each per operation. This
 332          * requires that we find the smallest DMA area between the source
 333          * and destination. The resulting len values will always be <= UINT_MAX
 334          * because the dma length is an unsigned int.
 335          */
 336         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 337         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 338
 339         if (dst) {
 340                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 341                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 342                 op_len = min(sg_src_len, sg_dst_len);
 343         } else {
 344                 op_len = sg_src_len;
 345         }
 346
 347         /* The data operation length will be at least block_size in length
 348          * or the smaller of available sg room remaining for the source or
 349          * the destination
 350          */
 351         op_len = max(op_len, block_size);
 352
 353         /* Unless we have to buffer data, there's no reason to wait */
 354         op->soc = 0;
 355
 356         if (sg_src_len < block_size) {
 357                 /* Not enough data in the sg element, so it
 358                  * needs to be buffered into a blocksize chunk
 359                  */
 360                 int cp_len = ccp_fill_queue_buf(src);
 361
 362                 op->soc = 1;
 363                 op->src.u.dma.address = src->dm_wa.dma.address;
 364                 op->src.u.dma.offset = 0;
 365                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 366         } else {
 367                 /* Enough data in the sg element, but we need to
 368                  * adjust for any previously copied data
 369                  */
 370                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 371                 op->src.u.dma.offset = src->sg_wa.sg_used;
 372                 op->src.u.dma.length = op_len & ~(block_size - 1);
 373
 374                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 375         }
 376
 377         if (dst) {
 378                 if (sg_dst_len < block_size) {
 379                         /* Not enough room in the sg element or we're on the
 380                          * last piece of data (when using padding), so the
 381                          * output needs to be buffered into a blocksize chunk
 382                          */
 383                         op->soc = 1;
 384                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 385                         op->dst.u.dma.offset = 0;
 386                         op->dst.u.dma.length = op->src.u.dma.length;
 387                 } else {
 388                         /* Enough room in the sg element, but we need to
 389                          * adjust for any previously used area
 390                          */
 391                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 392                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 393                         op->dst.u.dma.length = op->src.u.dma.length;
 394                 }
 395         }
 396 }
 397
 398 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 399                              struct ccp_op *op)
 400 {
 401         op->init = 0;
 402
 403         if (dst) {
 404                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 405                         ccp_empty_queue_buf(dst);
 406                 else
 407                         ccp_update_sg_workarea(&dst->sg_wa,
 408                                                op->dst.u.dma.length);
 409         }
 410 }
 411
 412 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 413                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 414                                u32 byte_swap, bool from)
 415 {
 416         struct ccp_op op;
 417
 418         memset(&op, 0, sizeof(op));
 419
 420         op.cmd_q = cmd_q;
 421         op.jobid = jobid;
 422         op.eom = 1;
 423
 424         if (from) {
 425                 op.soc = 1;
 426                 op.src.type = CCP_MEMTYPE_SB;
 427                 op.src.u.sb = sb;
 428                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 429                 op.dst.u.dma.address = wa->dma.address;
 430                 op.dst.u.dma.length = wa->length;
 431         } else {
 432                 op.src.type = CCP_MEMTYPE_SYSTEM;
 433                 op.src.u.dma.address = wa->dma.address;
 434                 op.src.u.dma.length = wa->length;
 435                 op.dst.type = CCP_MEMTYPE_SB;
 436                 op.dst.u.sb = sb;
 437         }
 438
 439         op.u.passthru.byte_swap = byte_swap;
 440
 441         return cmd_q->ccp->vdata->perform->passthru(&op);
 442 }
 443
 444 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 445                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 446                           u32 byte_swap)
 447 {
 448         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 449 }
 450
 451 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 452                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 453                             u32 byte_swap)
 454 {
 455         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 456 }
 457
 458 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 459                                 struct ccp_cmd *cmd)
 460 {
 461         struct ccp_aes_engine *aes = &cmd->u.aes;
 462         struct ccp_dm_workarea key, ctx;
 463         struct ccp_data src;
 464         struct ccp_op op;
 465         unsigned int dm_offset;
 466         int ret;
 467
 468         if (!((aes->key_len == AES_KEYSIZE_128) ||
 469               (aes->key_len == AES_KEYSIZE_192) ||
 470               (aes->key_len == AES_KEYSIZE_256)))
 471                 return -EINVAL;
 472
 473         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 474                 return -EINVAL;
 475
 476         if (aes->iv_len != AES_BLOCK_SIZE)
 477                 return -EINVAL;
 478
 479         if (!aes->key || !aes->iv || !aes->src)
 480                 return -EINVAL;
 481
 482         if (aes->cmac_final) {
 483                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 484                         return -EINVAL;
 485
 486                 if (!aes->cmac_key)
 487                         return -EINVAL;
 488         }
 489
 490         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 491         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 492
 493         ret = -EIO;
 494         memset(&op, 0, sizeof(op));
 495         op.cmd_q = cmd_q;
 496         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 497         op.sb_key = cmd_q->sb_key;
 498         op.sb_ctx = cmd_q->sb_ctx;
 499         op.init = 1;
 500         op.u.aes.type = aes->type;
 501         op.u.aes.mode = aes->mode;
 502         op.u.aes.action = aes->action;
 503
 504         /* All supported key sizes fit in a single (32-byte) SB entry
 505          * and must be in little endian format. Use the 256-bit byte
 506          * swap passthru option to convert from big endian to little
 507          * endian.
 508          */
 509         ret = ccp_init_dm_workarea(&key, cmd_q,
 510                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 511                                    DMA_TO_DEVICE);
 512         if (ret)
 513                 return ret;
 514
 515         dm_offset = CCP_SB_BYTES - aes->key_len;
 516         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 517         if (ret)
 518                 goto e_key;
 519         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 520                              CCP_PASSTHRU_BYTESWAP_256BIT);
 521         if (ret) {
 522                 cmd->engine_error = cmd_q->cmd_error;
 523                 goto e_key;
 524         }
 525
 526         /* The AES context fits in a single (32-byte) SB entry and
 527          * must be in little endian format. Use the 256-bit byte swap
 528          * passthru option to convert from big endian to little endian.
 529          */
 530         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 531                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 532                                    DMA_BIDIRECTIONAL);
 533         if (ret)
 534                 goto e_key;
 535
 536         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 537         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 538         if (ret)
 539                 goto e_ctx;
 540         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 541                              CCP_PASSTHRU_BYTESWAP_256BIT);
 542         if (ret) {
 543                 cmd->engine_error = cmd_q->cmd_error;
 544                 goto e_ctx;
 545         }
 546
 547         /* Send data to the CCP AES engine */
 548         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 549                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 550         if (ret)
 551                 goto e_ctx;
 552
 553         while (src.sg_wa.bytes_left) {
 554                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 555                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 556                         op.eom = 1;
 557
 558                         /* Push the K1/K2 key to the CCP now */
 559                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 560                                                op.sb_ctx,
 561                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 562                         if (ret) {
 563                                 cmd->engine_error = cmd_q->cmd_error;
 564                                 goto e_src;
 565                         }
 566
 567                         ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 568                                               aes->cmac_key_len);
 569                         if (ret)
 570                                 goto e_src;
 571                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 572                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 573                         if (ret) {
 574                                 cmd->engine_error = cmd_q->cmd_error;
 575                                 goto e_src;
 576                         }
 577                 }
 578
 579                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 580                 if (ret) {
 581                         cmd->engine_error = cmd_q->cmd_error;
 582                         goto e_src;
 583                 }
 584
 585                 ccp_process_data(&src, NULL, &op);
 586         }
 587
 588         /* Retrieve the AES context - convert from LE to BE using
 589          * 32-byte (256-bit) byteswapping
 590          */
 591         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 592                                CCP_PASSTHRU_BYTESWAP_256BIT);
 593         if (ret) {
 594                 cmd->engine_error = cmd_q->cmd_error;
 595                 goto e_src;
 596         }
 597
 598         /* ...but we only need AES_BLOCK_SIZE bytes */
 599         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 600         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 601
 602 e_src:
 603         ccp_free_data(&src, cmd_q);
 604
 605 e_ctx:
 606         ccp_dm_free(&ctx);
 607
 608 e_key:
 609         ccp_dm_free(&key);
 610
 611         return ret;
 612 }
 613
 614 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
 615                                struct ccp_cmd *cmd)
 616 {
 617         struct ccp_aes_engine *aes = &cmd->u.aes;
 618         struct ccp_dm_workarea key, ctx, final_wa, tag;
 619         struct ccp_data src, dst;
 620         struct ccp_data aad;
 621         struct ccp_op op;
 622
 623         unsigned long long *final;
 624         unsigned int dm_offset;
 625         unsigned int ilen;
 626         bool in_place = true; /* Default value */
 627         int ret;
 628
 629         struct scatterlist *p_inp, sg_inp[2];
 630         struct scatterlist *p_tag, sg_tag[2];
 631         struct scatterlist *p_outp, sg_outp[2];
 632         struct scatterlist *p_aad;
 633
 634         if (!aes->iv)
 635                 return -EINVAL;
 636
 637         if (!((aes->key_len == AES_KEYSIZE_128) ||
 638                 (aes->key_len == AES_KEYSIZE_192) ||
 639                 (aes->key_len == AES_KEYSIZE_256)))
 640                 return -EINVAL;
 641
 642         if (!aes->key) /* Gotta have a key SGL */
 643                 return -EINVAL;
 644
 645         /* First, decompose the source buffer into AAD & PT,
 646          * and the destination buffer into AAD, CT & tag, or
 647          * the input into CT & tag.
 648          * It is expected that the input and output SGs will
 649          * be valid, even if the AAD and input lengths are 0.
 650          */
 651         p_aad = aes->src;
 652         p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 653         p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 654         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 655                 ilen = aes->src_len;
 656                 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 657         } else {
 658                 /* Input length for decryption includes tag */
 659                 ilen = aes->src_len - AES_BLOCK_SIZE;
 660                 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 661         }
 662
 663         memset(&op, 0, sizeof(op));
 664         op.cmd_q = cmd_q;
 665         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 666         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 667         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 668         op.init = 1;
 669         op.u.aes.type = aes->type;
 670
 671         /* Copy the key to the LSB */
 672         ret = ccp_init_dm_workarea(&key, cmd_q,
 673                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 674                                    DMA_TO_DEVICE);
 675         if (ret)
 676                 return ret;
 677
 678         dm_offset = CCP_SB_BYTES - aes->key_len;
 679         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 680         if (ret)
 681                 goto e_key;
 682         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 683                              CCP_PASSTHRU_BYTESWAP_256BIT);
 684         if (ret) {
 685                 cmd->engine_error = cmd_q->cmd_error;
 686                 goto e_key;
 687         }
 688
 689         /* Copy the context (IV) to the LSB.
 690          * There is an assumption here that the IV is 96 bits in length, plus
 691          * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 692          */
 693         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 694                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 695                                    DMA_BIDIRECTIONAL);
 696         if (ret)
 697                 goto e_key;
 698
 699         dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 700         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 701         if (ret)
 702                 goto e_ctx;
 703
 704         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 705                              CCP_PASSTHRU_BYTESWAP_256BIT);
 706         if (ret) {
 707                 cmd->engine_error = cmd_q->cmd_error;
 708                 goto e_ctx;
 709         }
 710
 711         op.init = 1;
 712         if (aes->aad_len > 0) {
 713                 /* Step 1: Run a GHASH over the Additional Authenticated Data */
 714                 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 715                                     AES_BLOCK_SIZE,
 716                                     DMA_TO_DEVICE);
 717                 if (ret)
 718                         goto e_ctx;
 719
 720                 op.u.aes.mode = CCP_AES_MODE_GHASH;
 721                 op.u.aes.action = CCP_AES_GHASHAAD;
 722
 723                 while (aad.sg_wa.bytes_left) {
 724                         ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 725
 726                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 727                         if (ret) {
 728                                 cmd->engine_error = cmd_q->cmd_error;
 729                                 goto e_aad;
 730                         }
 731
 732                         ccp_process_data(&aad, NULL, &op);
 733                         op.init = 0;
 734                 }
 735         }
 736
 737         op.u.aes.mode = CCP_AES_MODE_GCTR;
 738         op.u.aes.action = aes->action;
 739
 740         if (ilen > 0) {
 741                 /* Step 2: Run a GCTR over the plaintext */
 742                 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 743
 744                 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 745                                     AES_BLOCK_SIZE,
 746                                     in_place ? DMA_BIDIRECTIONAL
 747                                              : DMA_TO_DEVICE);
 748                 if (ret)
 749                         goto e_ctx;
 750
 751                 if (in_place) {
 752                         dst = src;
 753                 } else {
 754                         ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 755                                             AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 756                         if (ret)
 757                                 goto e_src;
 758                 }
 759
 760                 op.soc = 0;
 761                 op.eom = 0;
 762                 op.init = 1;
 763                 while (src.sg_wa.bytes_left) {
 764                         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 765                         if (!src.sg_wa.bytes_left) {
 766                                 unsigned int nbytes = aes->src_len
 767                                                       % AES_BLOCK_SIZE;
 768
 769                                 if (nbytes) {
 770                                         op.eom = 1;
 771                                         op.u.aes.size = (nbytes * 8) - 1;
 772                                 }
 773                         }
 774
 775                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 776                         if (ret) {
 777                                 cmd->engine_error = cmd_q->cmd_error;
 778                                 goto e_dst;
 779                         }
 780
 781                         ccp_process_data(&src, &dst, &op);
 782                         op.init = 0;
 783                 }
 784         }
 785
 786         /* Step 3: Update the IV portion of the context with the original IV */
 787         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 788                                CCP_PASSTHRU_BYTESWAP_256BIT);
 789         if (ret) {
 790                 cmd->engine_error = cmd_q->cmd_error;
 791                 goto e_dst;
 792         }
 793
 794         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 795         if (ret)
 796                 goto e_dst;
 797
 798         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 799                              CCP_PASSTHRU_BYTESWAP_256BIT);
 800         if (ret) {
 801                 cmd->engine_error = cmd_q->cmd_error;
 802                 goto e_dst;
 803         }
 804
 805         /* Step 4: Concatenate the lengths of the AAD and source, and
 806          * hash that 16 byte buffer.
 807          */
 808         ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 809                                    DMA_BIDIRECTIONAL);
 810         if (ret)
 811                 goto e_dst;
 812         final = (unsigned long long *) final_wa.address;
 813         final[0] = cpu_to_be64(aes->aad_len * 8);
 814         final[1] = cpu_to_be64(ilen * 8);
 815
 816         op.u.aes.mode = CCP_AES_MODE_GHASH;
 817         op.u.aes.action = CCP_AES_GHASHFINAL;
 818         op.src.type = CCP_MEMTYPE_SYSTEM;
 819         op.src.u.dma.address = final_wa.dma.address;
 820         op.src.u.dma.length = AES_BLOCK_SIZE;
 821         op.dst.type = CCP_MEMTYPE_SYSTEM;
 822         op.dst.u.dma.address = final_wa.dma.address;
 823         op.dst.u.dma.length = AES_BLOCK_SIZE;
 824         op.eom = 1;
 825         op.u.aes.size = 0;
 826         ret = cmd_q->ccp->vdata->perform->aes(&op);
 827         if (ret)
 828                 goto e_dst;
 829
 830         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 831                 /* Put the ciphered tag after the ciphertext. */
 832                 ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE);
 833         } else {
 834                 /* Does this ciphered tag match the input? */
 835                 ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE,
 836                                            DMA_BIDIRECTIONAL);
 837                 if (ret)
 838                         goto e_tag;
 839                 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE);
 840                 if (ret)
 841                         goto e_tag;
 842
 843                 ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE);
 844                 ccp_dm_free(&tag);
 845         }
 846
 847 e_tag:
 848         ccp_dm_free(&final_wa);
 849
 850 e_dst:
 851         if (aes->src_len && !in_place)
 852                 ccp_free_data(&dst, cmd_q);
 853
 854 e_src:
 855         if (aes->src_len)
 856                 ccp_free_data(&src, cmd_q);
 857
 858 e_aad:
 859         if (aes->aad_len)
 860                 ccp_free_data(&aad, cmd_q);
 861
 862 e_ctx:
 863         ccp_dm_free(&ctx);
 864
 865 e_key:
 866         ccp_dm_free(&key);
 867
 868         return ret;
 869 }
 870
 871 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 872 {
 873         struct ccp_aes_engine *aes = &cmd->u.aes;
 874         struct ccp_dm_workarea key, ctx;
 875         struct ccp_data src, dst;
 876         struct ccp_op op;
 877         unsigned int dm_offset;
 878         bool in_place = false;
 879         int ret;
 880
 881         if (aes->mode == CCP_AES_MODE_CMAC)
 882                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
 883
 884         if (aes->mode == CCP_AES_MODE_GCM)
 885                 return ccp_run_aes_gcm_cmd(cmd_q, cmd);
 886
 887         if (!((aes->key_len == AES_KEYSIZE_128) ||
 888               (aes->key_len == AES_KEYSIZE_192) ||
 889               (aes->key_len == AES_KEYSIZE_256)))
 890                 return -EINVAL;
 891
 892         if (((aes->mode == CCP_AES_MODE_ECB) ||
 893              (aes->mode == CCP_AES_MODE_CBC) ||
 894              (aes->mode == CCP_AES_MODE_CFB)) &&
 895             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 896                 return -EINVAL;
 897
 898         if (!aes->key || !aes->src || !aes->dst)
 899                 return -EINVAL;
 900
 901         if (aes->mode != CCP_AES_MODE_ECB) {
 902                 if (aes->iv_len != AES_BLOCK_SIZE)
 903                         return -EINVAL;
 904
 905                 if (!aes->iv)
 906                         return -EINVAL;
 907         }
 908
 909         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 910         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 911
 912         ret = -EIO;
 913         memset(&op, 0, sizeof(op));
 914         op.cmd_q = cmd_q;
 915         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 916         op.sb_key = cmd_q->sb_key;
 917         op.sb_ctx = cmd_q->sb_ctx;
 918         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 919         op.u.aes.type = aes->type;
 920         op.u.aes.mode = aes->mode;
 921         op.u.aes.action = aes->action;
 922
 923         /* All supported key sizes fit in a single (32-byte) SB entry
 924          * and must be in little endian format. Use the 256-bit byte
 925          * swap passthru option to convert from big endian to little
 926          * endian.
 927          */
 928         ret = ccp_init_dm_workarea(&key, cmd_q,
 929                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 930                                    DMA_TO_DEVICE);
 931         if (ret)
 932                 return ret;
 933
 934         dm_offset = CCP_SB_BYTES - aes->key_len;
 935         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 936         if (ret)
 937                 goto e_key;
 938         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 939                              CCP_PASSTHRU_BYTESWAP_256BIT);
 940         if (ret) {
 941                 cmd->engine_error = cmd_q->cmd_error;
 942                 goto e_key;
 943         }
 944
 945         /* The AES context fits in a single (32-byte) SB entry and
 946          * must be in little endian format. Use the 256-bit byte swap
 947          * passthru option to convert from big endian to little endian.
 948          */
 949         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 950                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 951                                    DMA_BIDIRECTIONAL);
 952         if (ret)
 953                 goto e_key;
 954
 955         if (aes->mode != CCP_AES_MODE_ECB) {
 956                 /* Load the AES context - convert to LE */
 957                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 958                 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 959                 if (ret)
 960                         goto e_ctx;
 961                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 962                                      CCP_PASSTHRU_BYTESWAP_256BIT);
 963                 if (ret) {
 964                         cmd->engine_error = cmd_q->cmd_error;
 965                         goto e_ctx;
 966                 }
 967         }
 968         switch (aes->mode) {
 969         case CCP_AES_MODE_CFB: /* CFB128 only */
 970         case CCP_AES_MODE_CTR:
 971                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
 972                 break;
 973         default:
 974                 op.u.aes.size = 0;
 975         }
 976
 977         /* Prepare the input and output data workareas. For in-place
 978          * operations we need to set the dma direction to BIDIRECTIONAL
 979          * and copy the src workarea to the dst workarea.
 980          */
 981         if (sg_virt(aes->src) == sg_virt(aes->dst))
 982                 in_place = true;
 983
 984         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 985                             AES_BLOCK_SIZE,
 986                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 987         if (ret)
 988                 goto e_ctx;
 989
 990         if (in_place) {
 991                 dst = src;
 992         } else {
 993                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
 994                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 995                 if (ret)
 996                         goto e_src;
 997         }
 998
 999         /* Send data to the CCP AES engine */
1000         while (src.sg_wa.bytes_left) {
1001                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1002                 if (!src.sg_wa.bytes_left) {
1003                         op.eom = 1;
1004
1005                         /* Since we don't retrieve the AES context in ECB
1006                          * mode we have to wait for the operation to complete
1007                          * on the last piece of data
1008                          */
1009                         if (aes->mode == CCP_AES_MODE_ECB)
1010                                 op.soc = 1;
1011                 }
1012
1013                 ret = cmd_q->ccp->vdata->perform->aes(&op);
1014                 if (ret) {
1015                         cmd->engine_error = cmd_q->cmd_error;
1016                         goto e_dst;
1017                 }
1018
1019                 ccp_process_data(&src, &dst, &op);
1020         }
1021
1022         if (aes->mode != CCP_AES_MODE_ECB) {
1023                 /* Retrieve the AES context - convert from LE to BE using
1024                  * 32-byte (256-bit) byteswapping
1025                  */
1026                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1027                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1028                 if (ret) {
1029                         cmd->engine_error = cmd_q->cmd_error;
1030                         goto e_dst;
1031                 }
1032
1033                 /* ...but we only need AES_BLOCK_SIZE bytes */
1034                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1035                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1036         }
1037
1038 e_dst:
1039         if (!in_place)
1040                 ccp_free_data(&dst, cmd_q);
1041
1042 e_src:
1043         ccp_free_data(&src, cmd_q);
1044
1045 e_ctx:
1046         ccp_dm_free(&ctx);
1047
1048 e_key:
1049         ccp_dm_free(&key);
1050
1051         return ret;
1052 }
1053
1054 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1055                                struct ccp_cmd *cmd)
1056 {
1057         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1058         struct ccp_dm_workarea key, ctx;
1059         struct ccp_data src, dst;
1060         struct ccp_op op;
1061         unsigned int unit_size, dm_offset;
1062         bool in_place = false;
1063         unsigned int sb_count;
1064         enum ccp_aes_type aestype;
1065         int ret;
1066
1067         switch (xts->unit_size) {
1068         case CCP_XTS_AES_UNIT_SIZE_16:
1069                 unit_size = 16;
1070                 break;
1071         case CCP_XTS_AES_UNIT_SIZE_512:
1072                 unit_size = 512;
1073                 break;
1074         case CCP_XTS_AES_UNIT_SIZE_1024:
1075                 unit_size = 1024;
1076                 break;
1077         case CCP_XTS_AES_UNIT_SIZE_2048:
1078                 unit_size = 2048;
1079                 break;
1080         case CCP_XTS_AES_UNIT_SIZE_4096:
1081                 unit_size = 4096;
1082                 break;
1083
1084         default:
1085                 return -EINVAL;
1086         }
1087
1088         if (xts->key_len == AES_KEYSIZE_128)
1089                 aestype = CCP_AES_TYPE_128;
1090         else if (xts->key_len == AES_KEYSIZE_256)
1091                 aestype = CCP_AES_TYPE_256;
1092         else
1093                 return -EINVAL;
1094
1095         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1096                 return -EINVAL;
1097
1098         if (xts->iv_len != AES_BLOCK_SIZE)
1099                 return -EINVAL;
1100
1101         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1102                 return -EINVAL;
1103
1104         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1105         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1106
1107         ret = -EIO;
1108         memset(&op, 0, sizeof(op));
1109         op.cmd_q = cmd_q;
1110         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1111         op.sb_key = cmd_q->sb_key;
1112         op.sb_ctx = cmd_q->sb_ctx;
1113         op.init = 1;
1114         op.u.xts.type = aestype;
1115         op.u.xts.action = xts->action;
1116         op.u.xts.unit_size = xts->unit_size;
1117
1118         /* A version 3 device only supports 128-bit keys, which fits into a
1119          * single SB entry. A version 5 device uses a 512-bit vector, so two
1120          * SB entries.
1121          */
1122         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1123                 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1124         else
1125                 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1126         ret = ccp_init_dm_workarea(&key, cmd_q,
1127                                    sb_count * CCP_SB_BYTES,
1128                                    DMA_TO_DEVICE);
1129         if (ret)
1130                 return ret;
1131
1132         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1133                 /* All supported key sizes must be in little endian format.
1134                  * Use the 256-bit byte swap passthru option to convert from
1135                  * big endian to little endian.
1136                  */
1137                 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1138                 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1139                 if (ret)
1140                         goto e_key;
1141                 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1142                 if (ret)
1143                         goto e_key;
1144         } else {
1145                 /* Version 5 CCPs use a 512-bit space for the key: each portion
1146                  * occupies 256 bits, or one entire slot, and is zero-padded.
1147                  */
1148                 unsigned int pad;
1149
1150                 dm_offset = CCP_SB_BYTES;
1151                 pad = dm_offset - xts->key_len;
1152                 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1153                 if (ret)
1154                         goto e_key;
1155                 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1156                                       xts->key_len, xts->key_len);
1157                 if (ret)
1158                         goto e_key;
1159         }
1160         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1161                              CCP_PASSTHRU_BYTESWAP_256BIT);
1162         if (ret) {
1163                 cmd->engine_error = cmd_q->cmd_error;
1164                 goto e_key;
1165         }
1166
1167         /* The AES context fits in a single (32-byte) SB entry and
1168          * for XTS is already in little endian format so no byte swapping
1169          * is needed.
1170          */
1171         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1172                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1173                                    DMA_BIDIRECTIONAL);
1174         if (ret)
1175                 goto e_key;
1176
1177         ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1178         if (ret)
1179                 goto e_ctx;
1180         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1181                              CCP_PASSTHRU_BYTESWAP_NOOP);
1182         if (ret) {
1183                 cmd->engine_error = cmd_q->cmd_error;
1184                 goto e_ctx;
1185         }
1186
1187         /* Prepare the input and output data workareas. For in-place
1188          * operations we need to set the dma direction to BIDIRECTIONAL
1189          * and copy the src workarea to the dst workarea.
1190          */
1191         if (sg_virt(xts->src) == sg_virt(xts->dst))
1192                 in_place = true;
1193
1194         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1195                             unit_size,
1196                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1197         if (ret)
1198                 goto e_ctx;
1199
1200         if (in_place) {
1201                 dst = src;
1202         } else {
1203                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1204                                     unit_size, DMA_FROM_DEVICE);
1205                 if (ret)
1206                         goto e_src;
1207         }
1208
1209         /* Send data to the CCP AES engine */
1210         while (src.sg_wa.bytes_left) {
1211                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1212                 if (!src.sg_wa.bytes_left)
1213                         op.eom = 1;
1214
1215                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1216                 if (ret) {
1217                         cmd->engine_error = cmd_q->cmd_error;
1218                         goto e_dst;
1219                 }
1220
1221                 ccp_process_data(&src, &dst, &op);
1222         }
1223
1224         /* Retrieve the AES context - convert from LE to BE using
1225          * 32-byte (256-bit) byteswapping
1226          */
1227         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1228                                CCP_PASSTHRU_BYTESWAP_256BIT);
1229         if (ret) {
1230                 cmd->engine_error = cmd_q->cmd_error;
1231                 goto e_dst;
1232         }
1233
1234         /* ...but we only need AES_BLOCK_SIZE bytes */
1235         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1236         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1237
1238 e_dst:
1239         if (!in_place)
1240                 ccp_free_data(&dst, cmd_q);
1241
1242 e_src:
1243         ccp_free_data(&src, cmd_q);
1244
1245 e_ctx:
1246         ccp_dm_free(&ctx);
1247
1248 e_key:
1249         ccp_dm_free(&key);
1250
1251         return ret;
1252 }
1253
1254 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1255 {
1256         struct ccp_des3_engine *des3 = &cmd->u.des3;
1257
1258         struct ccp_dm_workarea key, ctx;
1259         struct ccp_data src, dst;
1260         struct ccp_op op;
1261         unsigned int dm_offset;
1262         unsigned int len_singlekey;
1263         bool in_place = false;
1264         int ret;
1265
1266         /* Error checks */
1267         if (!cmd_q->ccp->vdata->perform->des3)
1268                 return -EINVAL;
1269
1270         if (des3->key_len != DES3_EDE_KEY_SIZE)
1271                 return -EINVAL;
1272
1273         if (((des3->mode == CCP_DES3_MODE_ECB) ||
1274                 (des3->mode == CCP_DES3_MODE_CBC)) &&
1275                 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1276                 return -EINVAL;
1277
1278         if (!des3->key || !des3->src || !des3->dst)
1279                 return -EINVAL;
1280
1281         if (des3->mode != CCP_DES3_MODE_ECB) {
1282                 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1283                         return -EINVAL;
1284
1285                 if (!des3->iv)
1286                         return -EINVAL;
1287         }
1288
1289         ret = -EIO;
1290         /* Zero out all the fields of the command desc */
1291         memset(&op, 0, sizeof(op));
1292
1293         /* Set up the Function field */
1294         op.cmd_q = cmd_q;
1295         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1296         op.sb_key = cmd_q->sb_key;
1297
1298         op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1299         op.u.des3.type = des3->type;
1300         op.u.des3.mode = des3->mode;
1301         op.u.des3.action = des3->action;
1302
1303         /*
1304          * All supported key sizes fit in a single (32-byte) KSB entry and
1305          * (like AES) must be in little endian format. Use the 256-bit byte
1306          * swap passthru option to convert from big endian to little endian.
1307          */
1308         ret = ccp_init_dm_workarea(&key, cmd_q,
1309                                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1310                                    DMA_TO_DEVICE);
1311         if (ret)
1312                 return ret;
1313
1314         /*
1315          * The contents of the key triplet are in the reverse order of what
1316          * is required by the engine. Copy the 3 pieces individually to put
1317          * them where they belong.
1318          */
1319         dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1320
1321         len_singlekey = des3->key_len / 3;
1322         ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1323                               des3->key, 0, len_singlekey);
1324         if (ret)
1325                 goto e_key;
1326         ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1327                               des3->key, len_singlekey, len_singlekey);
1328         if (ret)
1329                 goto e_key;
1330         ret = ccp_set_dm_area(&key, dm_offset,
1331                               des3->key, 2 * len_singlekey, len_singlekey);
1332         if (ret)
1333                 goto e_key;
1334
1335         /* Copy the key to the SB */
1336         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1337                              CCP_PASSTHRU_BYTESWAP_256BIT);
1338         if (ret) {
1339                 cmd->engine_error = cmd_q->cmd_error;
1340                 goto e_key;
1341         }
1342
1343         /*
1344          * The DES3 context fits in a single (32-byte) KSB entry and
1345          * must be in little endian format. Use the 256-bit byte swap
1346          * passthru option to convert from big endian to little endian.
1347          */
1348         if (des3->mode != CCP_DES3_MODE_ECB) {
1349                 u32 load_mode;
1350
1351                 op.sb_ctx = cmd_q->sb_ctx;
1352
1353                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1354                                            CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1355                                            DMA_BIDIRECTIONAL);
1356                 if (ret)
1357                         goto e_key;
1358
1359                 /* Load the context into the LSB */
1360                 dm_offset = CCP_SB_BYTES - des3->iv_len;
1361                 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1362                                       des3->iv_len);
1363                 if (ret)
1364                         goto e_ctx;
1365
1366                 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1367                         load_mode = CCP_PASSTHRU_BYTESWAP_NOOP;
1368                 else
1369                         load_mode = CCP_PASSTHRU_BYTESWAP_256BIT;
1370                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1371                                      load_mode);
1372                 if (ret) {
1373                         cmd->engine_error = cmd_q->cmd_error;
1374                         goto e_ctx;
1375                 }
1376         }
1377
1378         /*
1379          * Prepare the input and output data workareas. For in-place
1380          * operations we need to set the dma direction to BIDIRECTIONAL
1381          * and copy the src workarea to the dst workarea.
1382          */
1383         if (sg_virt(des3->src) == sg_virt(des3->dst))
1384                 in_place = true;
1385
1386         ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1387                         DES3_EDE_BLOCK_SIZE,
1388                         in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1389         if (ret)
1390                 goto e_ctx;
1391
1392         if (in_place)
1393                 dst = src;
1394         else {
1395                 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1396                                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1397                 if (ret)
1398                         goto e_src;
1399         }
1400
1401         /* Send data to the CCP DES3 engine */
1402         while (src.sg_wa.bytes_left) {
1403                 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1404                 if (!src.sg_wa.bytes_left) {
1405                         op.eom = 1;
1406
1407                         /* Since we don't retrieve the context in ECB mode
1408                          * we have to wait for the operation to complete
1409                          * on the last piece of data
1410                          */
1411                         op.soc = 0;
1412                 }
1413
1414                 ret = cmd_q->ccp->vdata->perform->des3(&op);
1415                 if (ret) {
1416                         cmd->engine_error = cmd_q->cmd_error;
1417                         goto e_dst;
1418                 }
1419
1420                 ccp_process_data(&src, &dst, &op);
1421         }
1422
1423         if (des3->mode != CCP_DES3_MODE_ECB) {
1424                 /* Retrieve the context and make BE */
1425                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1426                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1427                 if (ret) {
1428                         cmd->engine_error = cmd_q->cmd_error;
1429                         goto e_dst;
1430                 }
1431
1432                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1433                 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1434                         dm_offset = CCP_SB_BYTES - des3->iv_len;
1435                 else
1436                         dm_offset = 0;
1437                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1438                                 DES3_EDE_BLOCK_SIZE);
1439         }
1440 e_dst:
1441         if (!in_place)
1442                 ccp_free_data(&dst, cmd_q);
1443
1444 e_src:
1445         ccp_free_data(&src, cmd_q);
1446
1447 e_ctx:
1448         if (des3->mode != CCP_DES3_MODE_ECB)
1449                 ccp_dm_free(&ctx);
1450
1451 e_key:
1452         ccp_dm_free(&key);
1453
1454         return ret;
1455 }
1456
1457 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1458 {
1459         struct ccp_sha_engine *sha = &cmd->u.sha;
1460         struct ccp_dm_workarea ctx;
1461         struct ccp_data src;
1462         struct ccp_op op;
1463         unsigned int ioffset, ooffset;
1464         unsigned int digest_size;
1465         int sb_count;
1466         const void *init;
1467         u64 block_size;
1468         int ctx_size;
1469         int ret;
1470
1471         switch (sha->type) {
1472         case CCP_SHA_TYPE_1:
1473                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1474                         return -EINVAL;
1475                 block_size = SHA1_BLOCK_SIZE;
1476                 break;
1477         case CCP_SHA_TYPE_224:
1478                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1479                         return -EINVAL;
1480                 block_size = SHA224_BLOCK_SIZE;
1481                 break;
1482         case CCP_SHA_TYPE_256:
1483                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1484                         return -EINVAL;
1485                 block_size = SHA256_BLOCK_SIZE;
1486                 break;
1487         case CCP_SHA_TYPE_384:
1488                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1489                     || sha->ctx_len < SHA384_DIGEST_SIZE)
1490                         return -EINVAL;
1491                 block_size = SHA384_BLOCK_SIZE;
1492                 break;
1493         case CCP_SHA_TYPE_512:
1494                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1495                     || sha->ctx_len < SHA512_DIGEST_SIZE)
1496                         return -EINVAL;
1497                 block_size = SHA512_BLOCK_SIZE;
1498                 break;
1499         default:
1500                 return -EINVAL;
1501         }
1502
1503         if (!sha->ctx)
1504                 return -EINVAL;
1505
1506         if (!sha->final && (sha->src_len & (block_size - 1)))
1507                 return -EINVAL;
1508
1509         /* The version 3 device can't handle zero-length input */
1510         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1511
1512                 if (!sha->src_len) {
1513                         unsigned int digest_len;
1514                         const u8 *sha_zero;
1515
1516                         /* Not final, just return */
1517                         if (!sha->final)
1518                                 return 0;
1519
1520                         /* CCP can't do a zero length sha operation so the
1521                          * caller must buffer the data.
1522                          */
1523                         if (sha->msg_bits)
1524                                 return -EINVAL;
1525
1526                         /* The CCP cannot perform zero-length sha operations
1527                          * so the caller is required to buffer data for the
1528                          * final operation. However, a sha operation for a
1529                          * message with a total length of zero is valid so
1530                          * known values are required to supply the result.
1531                          */
1532                         switch (sha->type) {
1533                         case CCP_SHA_TYPE_1:
1534                                 sha_zero = sha1_zero_message_hash;
1535                                 digest_len = SHA1_DIGEST_SIZE;
1536                                 break;
1537                         case CCP_SHA_TYPE_224:
1538                                 sha_zero = sha224_zero_message_hash;
1539                                 digest_len = SHA224_DIGEST_SIZE;
1540                                 break;
1541                         case CCP_SHA_TYPE_256:
1542                                 sha_zero = sha256_zero_message_hash;
1543                                 digest_len = SHA256_DIGEST_SIZE;
1544                                 break;
1545                         default:
1546                                 return -EINVAL;
1547                         }
1548
1549                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1550                                                  digest_len, 1);
1551
1552                         return 0;
1553                 }
1554         }
1555
1556         /* Set variables used throughout */
1557         switch (sha->type) {
1558         case CCP_SHA_TYPE_1:
1559                 digest_size = SHA1_DIGEST_SIZE;
1560                 init = (void *) ccp_sha1_init;
1561                 ctx_size = SHA1_DIGEST_SIZE;
1562                 sb_count = 1;
1563                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1564                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1565                 else
1566                         ooffset = ioffset = 0;
1567                 break;
1568         case CCP_SHA_TYPE_224:
1569                 digest_size = SHA224_DIGEST_SIZE;
1570                 init = (void *) ccp_sha224_init;
1571                 ctx_size = SHA256_DIGEST_SIZE;
1572                 sb_count = 1;
1573                 ioffset = 0;
1574                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1575                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1576                 else
1577                         ooffset = 0;
1578                 break;
1579         case CCP_SHA_TYPE_256:
1580                 digest_size = SHA256_DIGEST_SIZE;
1581                 init = (void *) ccp_sha256_init;
1582                 ctx_size = SHA256_DIGEST_SIZE;
1583                 sb_count = 1;
1584                 ooffset = ioffset = 0;
1585                 break;
1586         case CCP_SHA_TYPE_384:
1587                 digest_size = SHA384_DIGEST_SIZE;
1588                 init = (void *) ccp_sha384_init;
1589                 ctx_size = SHA512_DIGEST_SIZE;
1590                 sb_count = 2;
1591                 ioffset = 0;
1592                 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1593                 break;
1594         case CCP_SHA_TYPE_512:
1595                 digest_size = SHA512_DIGEST_SIZE;
1596                 init = (void *) ccp_sha512_init;
1597                 ctx_size = SHA512_DIGEST_SIZE;
1598                 sb_count = 2;
1599                 ooffset = ioffset = 0;
1600                 break;
1601         default:
1602                 ret = -EINVAL;
1603                 goto e_data;
1604         }
1605
1606         /* For zero-length plaintext the src pointer is ignored;
1607          * otherwise both parts must be valid
1608          */
1609         if (sha->src_len && !sha->src)
1610                 return -EINVAL;
1611
1612         memset(&op, 0, sizeof(op));
1613         op.cmd_q = cmd_q;
1614         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1615         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1616         op.u.sha.type = sha->type;
1617         op.u.sha.msg_bits = sha->msg_bits;
1618
1619         /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1620          * SHA384/512 require 2 adjacent SB slots, with the right half in the
1621          * first slot, and the left half in the second. Each portion must then
1622          * be in little endian format: use the 256-bit byte swap option.
1623          */
1624         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1625                                    DMA_BIDIRECTIONAL);
1626         if (ret)
1627                 return ret;
1628         if (sha->first) {
1629                 switch (sha->type) {
1630                 case CCP_SHA_TYPE_1:
1631                 case CCP_SHA_TYPE_224:
1632                 case CCP_SHA_TYPE_256:
1633                         memcpy(ctx.address + ioffset, init, ctx_size);
1634                         break;
1635                 case CCP_SHA_TYPE_384:
1636                 case CCP_SHA_TYPE_512:
1637                         memcpy(ctx.address + ctx_size / 2, init,
1638                                ctx_size / 2);
1639                         memcpy(ctx.address, init + ctx_size / 2,
1640                                ctx_size / 2);
1641                         break;
1642                 default:
1643                         ret = -EINVAL;
1644                         goto e_ctx;
1645                 }
1646         } else {
1647                 /* Restore the context */
1648                 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1649                                       sb_count * CCP_SB_BYTES);
1650                 if (ret)
1651                         goto e_ctx;
1652         }
1653
1654         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1655                              CCP_PASSTHRU_BYTESWAP_256BIT);
1656         if (ret) {
1657                 cmd->engine_error = cmd_q->cmd_error;
1658                 goto e_ctx;
1659         }
1660
1661         if (sha->src) {
1662                 /* Send data to the CCP SHA engine; block_size is set above */
1663                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1664                                     block_size, DMA_TO_DEVICE);
1665                 if (ret)
1666                         goto e_ctx;
1667
1668                 while (src.sg_wa.bytes_left) {
1669                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1670                         if (sha->final && !src.sg_wa.bytes_left)
1671                                 op.eom = 1;
1672
1673                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1674                         if (ret) {
1675                                 cmd->engine_error = cmd_q->cmd_error;
1676                                 goto e_data;
1677                         }
1678
1679                         ccp_process_data(&src, NULL, &op);
1680                 }
1681         } else {
1682                 op.eom = 1;
1683                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1684                 if (ret) {
1685                         cmd->engine_error = cmd_q->cmd_error;
1686                         goto e_data;
1687                 }
1688         }
1689
1690         /* Retrieve the SHA context - convert from LE to BE using
1691          * 32-byte (256-bit) byteswapping to BE
1692          */
1693         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1694                                CCP_PASSTHRU_BYTESWAP_256BIT);
1695         if (ret) {
1696                 cmd->engine_error = cmd_q->cmd_error;
1697                 goto e_data;
1698         }
1699
1700         if (sha->final) {
1701                 /* Finishing up, so get the digest */
1702                 switch (sha->type) {
1703                 case CCP_SHA_TYPE_1:
1704                 case CCP_SHA_TYPE_224:
1705                 case CCP_SHA_TYPE_256:
1706                         ccp_get_dm_area(&ctx, ooffset,
1707                                         sha->ctx, 0,
1708                                         digest_size);
1709                         break;
1710                 case CCP_SHA_TYPE_384:
1711                 case CCP_SHA_TYPE_512:
1712                         ccp_get_dm_area(&ctx, 0,
1713                                         sha->ctx, LSB_ITEM_SIZE - ooffset,
1714                                         LSB_ITEM_SIZE);
1715                         ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1716                                         sha->ctx, 0,
1717                                         LSB_ITEM_SIZE - ooffset);
1718                         break;
1719                 default:
1720                         ret = -EINVAL;
1721                         goto e_ctx;
1722                 }
1723         } else {
1724                 /* Stash the context */
1725                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1726                                 sb_count * CCP_SB_BYTES);
1727         }
1728
1729         if (sha->final && sha->opad) {
1730                 /* HMAC operation, recursively perform final SHA */
1731                 struct ccp_cmd hmac_cmd;
1732                 struct scatterlist sg;
1733                 u8 *hmac_buf;
1734
1735                 if (sha->opad_len != block_size) {
1736                         ret = -EINVAL;
1737                         goto e_data;
1738                 }
1739
1740                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1741                 if (!hmac_buf) {
1742                         ret = -ENOMEM;
1743                         goto e_data;
1744                 }
1745                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1746
1747                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1748                 switch (sha->type) {
1749                 case CCP_SHA_TYPE_1:
1750                 case CCP_SHA_TYPE_224:
1751                 case CCP_SHA_TYPE_256:
1752                         memcpy(hmac_buf + block_size,
1753                                ctx.address + ooffset,
1754                                digest_size);
1755                         break;
1756                 case CCP_SHA_TYPE_384:
1757                 case CCP_SHA_TYPE_512:
1758                         memcpy(hmac_buf + block_size,
1759                                ctx.address + LSB_ITEM_SIZE + ooffset,
1760                                LSB_ITEM_SIZE);
1761                         memcpy(hmac_buf + block_size +
1762                                (LSB_ITEM_SIZE - ooffset),
1763                                ctx.address,
1764                                LSB_ITEM_SIZE);
1765                         break;
1766                 default:
1767                         ret = -EINVAL;
1768                         goto e_ctx;
1769                 }
1770
1771                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1772                 hmac_cmd.engine = CCP_ENGINE_SHA;
1773                 hmac_cmd.u.sha.type = sha->type;
1774                 hmac_cmd.u.sha.ctx = sha->ctx;
1775                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1776                 hmac_cmd.u.sha.src = &sg;
1777                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1778                 hmac_cmd.u.sha.opad = NULL;
1779                 hmac_cmd.u.sha.opad_len = 0;
1780                 hmac_cmd.u.sha.first = 1;
1781                 hmac_cmd.u.sha.final = 1;
1782                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1783
1784                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1785                 if (ret)
1786                         cmd->engine_error = hmac_cmd.engine_error;
1787
1788                 kfree(hmac_buf);
1789         }
1790
1791 e_data:
1792         if (sha->src)
1793                 ccp_free_data(&src, cmd_q);
1794
1795 e_ctx:
1796         ccp_dm_free(&ctx);
1797
1798         return ret;
1799 }
1800
1801 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1802 {
1803         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1804         struct ccp_dm_workarea exp, src, dst;
1805         struct ccp_op op;
1806         unsigned int sb_count, i_len, o_len;
1807         int ret;
1808
1809         /* Check against the maximum allowable size, in bits */
1810         if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1811                 return -EINVAL;
1812
1813         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1814                 return -EINVAL;
1815
1816         memset(&op, 0, sizeof(op));
1817         op.cmd_q = cmd_q;
1818         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1819
1820         /* The RSA modulus must precede the message being acted upon, so
1821          * it must be copied to a DMA area where the message and the
1822          * modulus can be concatenated.  Therefore the input buffer
1823          * length required is twice the output buffer length (which
1824          * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1825          * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1826          * required.
1827          */
1828         o_len = 32 * ((rsa->key_size + 255) / 256);
1829         i_len = o_len * 2;
1830
1831         sb_count = 0;
1832         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1833                 /* sb_count is the number of storage block slots required
1834                  * for the modulus.
1835                  */
1836                 sb_count = o_len / CCP_SB_BYTES;
1837                 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1838                                                                 sb_count);
1839                 if (!op.sb_key)
1840                         return -EIO;
1841         } else {
1842                 /* A version 5 device allows a modulus size that will not fit
1843                  * in the LSB, so the command will transfer it from memory.
1844                  * Set the sb key to the default, even though it's not used.
1845                  */
1846                 op.sb_key = cmd_q->sb_key;
1847         }
1848
1849         /* The RSA exponent must be in little endian format. Reverse its
1850          * byte order.
1851          */
1852         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1853         if (ret)
1854                 goto e_sb;
1855
1856         ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1857         if (ret)
1858                 goto e_exp;
1859
1860         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1861                 /* Copy the exponent to the local storage block, using
1862                  * as many 32-byte blocks as were allocated above. It's
1863                  * already little endian, so no further change is required.
1864                  */
1865                 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1866                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1867                 if (ret) {
1868                         cmd->engine_error = cmd_q->cmd_error;
1869                         goto e_exp;
1870                 }
1871         } else {
1872                 /* The exponent can be retrieved from memory via DMA. */
1873                 op.exp.u.dma.address = exp.dma.address;
1874                 op.exp.u.dma.offset = 0;
1875         }
1876
1877         /* Concatenate the modulus and the message. Both the modulus and
1878          * the operands must be in little endian format.  Since the input
1879          * is in big endian format it must be converted.
1880          */
1881         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1882         if (ret)
1883                 goto e_exp;
1884
1885         ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1886         if (ret)
1887                 goto e_src;
1888         ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1889         if (ret)
1890                 goto e_src;
1891
1892         /* Prepare the output area for the operation */
1893         ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1894         if (ret)
1895                 goto e_src;
1896
1897         op.soc = 1;
1898         op.src.u.dma.address = src.dma.address;
1899         op.src.u.dma.offset = 0;
1900         op.src.u.dma.length = i_len;
1901         op.dst.u.dma.address = dst.dma.address;
1902         op.dst.u.dma.offset = 0;
1903         op.dst.u.dma.length = o_len;
1904
1905         op.u.rsa.mod_size = rsa->key_size;
1906         op.u.rsa.input_len = i_len;
1907
1908         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1909         if (ret) {
1910                 cmd->engine_error = cmd_q->cmd_error;
1911                 goto e_dst;
1912         }
1913
1914         ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1915
1916 e_dst:
1917         ccp_dm_free(&dst);
1918
1919 e_src:
1920         ccp_dm_free(&src);
1921
1922 e_exp:
1923         ccp_dm_free(&exp);
1924
1925 e_sb:
1926         if (sb_count)
1927                 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1928
1929         return ret;
1930 }
1931
1932 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1933                                 struct ccp_cmd *cmd)
1934 {
1935         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1936         struct ccp_dm_workarea mask;
1937         struct ccp_data src, dst;
1938         struct ccp_op op;
1939         bool in_place = false;
1940         unsigned int i;
1941         int ret = 0;
1942
1943         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1944                 return -EINVAL;
1945
1946         if (!pt->src || !pt->dst)
1947                 return -EINVAL;
1948
1949         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1950                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1951                         return -EINVAL;
1952                 if (!pt->mask)
1953                         return -EINVAL;
1954         }
1955
1956         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1957
1958         memset(&op, 0, sizeof(op));
1959         op.cmd_q = cmd_q;
1960         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1961
1962         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1963                 /* Load the mask */
1964                 op.sb_key = cmd_q->sb_key;
1965
1966                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1967                                            CCP_PASSTHRU_SB_COUNT *
1968                                            CCP_SB_BYTES,
1969                                            DMA_TO_DEVICE);
1970                 if (ret)
1971                         return ret;
1972
1973                 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1974                 if (ret)
1975                         goto e_mask;
1976                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1977                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1978                 if (ret) {
1979                         cmd->engine_error = cmd_q->cmd_error;
1980                         goto e_mask;
1981                 }
1982         }
1983
1984         /* Prepare the input and output data workareas. For in-place
1985          * operations we need to set the dma direction to BIDIRECTIONAL
1986          * and copy the src workarea to the dst workarea.
1987          */
1988         if (sg_virt(pt->src) == sg_virt(pt->dst))
1989                 in_place = true;
1990
1991         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1992                             CCP_PASSTHRU_MASKSIZE,
1993                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1994         if (ret)
1995                 goto e_mask;
1996
1997         if (in_place) {
1998                 dst = src;
1999         } else {
2000                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2001                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2002                 if (ret)
2003                         goto e_src;
2004         }
2005
2006         /* Send data to the CCP Passthru engine
2007          *   Because the CCP engine works on a single source and destination
2008          *   dma address at a time, each entry in the source scatterlist
2009          *   (after the dma_map_sg call) must be less than or equal to the
2010          *   (remaining) length in the destination scatterlist entry and the
2011          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2012          */
2013         dst.sg_wa.sg_used = 0;
2014         for (i = 1; i <= src.sg_wa.dma_count; i++) {
2015                 if (!dst.sg_wa.sg ||
2016                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
2017                         ret = -EINVAL;
2018                         goto e_dst;
2019                 }
2020
2021                 if (i == src.sg_wa.dma_count) {
2022                         op.eom = 1;
2023                         op.soc = 1;
2024                 }
2025
2026                 op.src.type = CCP_MEMTYPE_SYSTEM;
2027                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2028                 op.src.u.dma.offset = 0;
2029                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2030
2031                 op.dst.type = CCP_MEMTYPE_SYSTEM;
2032                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2033                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2034                 op.dst.u.dma.length = op.src.u.dma.length;
2035
2036                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2037                 if (ret) {
2038                         cmd->engine_error = cmd_q->cmd_error;
2039                         goto e_dst;
2040                 }
2041
2042                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
2043                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
2044                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2045                         dst.sg_wa.sg_used = 0;
2046                 }
2047                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2048         }
2049
2050 e_dst:
2051         if (!in_place)
2052                 ccp_free_data(&dst, cmd_q);
2053
2054 e_src:
2055         ccp_free_data(&src, cmd_q);
2056
2057 e_mask:
2058         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2059                 ccp_dm_free(&mask);
2060
2061         return ret;
2062 }
2063
2064 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2065                                       struct ccp_cmd *cmd)
2066 {
2067         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2068         struct ccp_dm_workarea mask;
2069         struct ccp_op op;
2070         int ret;
2071
2072         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2073                 return -EINVAL;
2074
2075         if (!pt->src_dma || !pt->dst_dma)
2076                 return -EINVAL;
2077
2078         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2079                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2080                         return -EINVAL;
2081                 if (!pt->mask)
2082                         return -EINVAL;
2083         }
2084
2085         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2086
2087         memset(&op, 0, sizeof(op));
2088         op.cmd_q = cmd_q;
2089         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2090
2091         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2092                 /* Load the mask */
2093                 op.sb_key = cmd_q->sb_key;
2094
2095                 mask.length = pt->mask_len;
2096                 mask.dma.address = pt->mask;
2097                 mask.dma.length = pt->mask_len;
2098
2099                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2100                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2101                 if (ret) {
2102                         cmd->engine_error = cmd_q->cmd_error;
2103                         return ret;
2104                 }
2105         }
2106
2107         /* Send data to the CCP Passthru engine */
2108         op.eom = 1;
2109         op.soc = 1;
2110
2111         op.src.type = CCP_MEMTYPE_SYSTEM;
2112         op.src.u.dma.address = pt->src_dma;
2113         op.src.u.dma.offset = 0;
2114         op.src.u.dma.length = pt->src_len;
2115
2116         op.dst.type = CCP_MEMTYPE_SYSTEM;
2117         op.dst.u.dma.address = pt->dst_dma;
2118         op.dst.u.dma.offset = 0;
2119         op.dst.u.dma.length = pt->src_len;
2120
2121         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2122         if (ret)
2123                 cmd->engine_error = cmd_q->cmd_error;
2124
2125         return ret;
2126 }
2127
2128 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2129 {
2130         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2131         struct ccp_dm_workarea src, dst;
2132         struct ccp_op op;
2133         int ret;
2134         u8 *save;
2135
2136         if (!ecc->u.mm.operand_1 ||
2137             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2138                 return -EINVAL;
2139
2140         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2141                 if (!ecc->u.mm.operand_2 ||
2142                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2143                         return -EINVAL;
2144
2145         if (!ecc->u.mm.result ||
2146             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2147                 return -EINVAL;
2148
2149         memset(&op, 0, sizeof(op));
2150         op.cmd_q = cmd_q;
2151         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2152
2153         /* Concatenate the modulus and the operands. Both the modulus and
2154          * the operands must be in little endian format.  Since the input
2155          * is in big endian format it must be converted and placed in a
2156          * fixed length buffer.
2157          */
2158         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2159                                    DMA_TO_DEVICE);
2160         if (ret)
2161                 return ret;
2162
2163         /* Save the workarea address since it is updated in order to perform
2164          * the concatenation
2165          */
2166         save = src.address;
2167
2168         /* Copy the ECC modulus */
2169         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2170         if (ret)
2171                 goto e_src;
2172         src.address += CCP_ECC_OPERAND_SIZE;
2173
2174         /* Copy the first operand */
2175         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2176                                       ecc->u.mm.operand_1_len);
2177         if (ret)
2178                 goto e_src;
2179         src.address += CCP_ECC_OPERAND_SIZE;
2180
2181         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2182                 /* Copy the second operand */
2183                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2184                                               ecc->u.mm.operand_2_len);
2185                 if (ret)
2186                         goto e_src;
2187                 src.address += CCP_ECC_OPERAND_SIZE;
2188         }
2189
2190         /* Restore the workarea address */
2191         src.address = save;
2192
2193         /* Prepare the output area for the operation */
2194         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2195                                    DMA_FROM_DEVICE);
2196         if (ret)
2197                 goto e_src;
2198
2199         op.soc = 1;
2200         op.src.u.dma.address = src.dma.address;
2201         op.src.u.dma.offset = 0;
2202         op.src.u.dma.length = src.length;
2203         op.dst.u.dma.address = dst.dma.address;
2204         op.dst.u.dma.offset = 0;
2205         op.dst.u.dma.length = dst.length;
2206
2207         op.u.ecc.function = cmd->u.ecc.function;
2208
2209         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2210         if (ret) {
2211                 cmd->engine_error = cmd_q->cmd_error;
2212                 goto e_dst;
2213         }
2214
2215         ecc->ecc_result = le16_to_cpup(
2216                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2217         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2218                 ret = -EIO;
2219                 goto e_dst;
2220         }
2221
2222         /* Save the ECC result */
2223         ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2224                                 CCP_ECC_MODULUS_BYTES);
2225
2226 e_dst:
2227         ccp_dm_free(&dst);
2228
2229 e_src:
2230         ccp_dm_free(&src);
2231
2232         return ret;
2233 }
2234
2235 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2236 {
2237         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2238         struct ccp_dm_workarea src, dst;
2239         struct ccp_op op;
2240         int ret;
2241         u8 *save;
2242
2243         if (!ecc->u.pm.point_1.x ||
2244             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2245             !ecc->u.pm.point_1.y ||
2246             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2247                 return -EINVAL;
2248
2249         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2250                 if (!ecc->u.pm.point_2.x ||
2251                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2252                     !ecc->u.pm.point_2.y ||
2253                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2254                         return -EINVAL;
2255         } else {
2256                 if (!ecc->u.pm.domain_a ||
2257                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2258                         return -EINVAL;
2259
2260                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2261                         if (!ecc->u.pm.scalar ||
2262                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2263                                 return -EINVAL;
2264         }
2265
2266         if (!ecc->u.pm.result.x ||
2267             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2268             !ecc->u.pm.result.y ||
2269             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2270                 return -EINVAL;
2271
2272         memset(&op, 0, sizeof(op));
2273         op.cmd_q = cmd_q;
2274         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2275
2276         /* Concatenate the modulus and the operands. Both the modulus and
2277          * the operands must be in little endian format.  Since the input
2278          * is in big endian format it must be converted and placed in a
2279          * fixed length buffer.
2280          */
2281         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2282                                    DMA_TO_DEVICE);
2283         if (ret)
2284                 return ret;
2285
2286         /* Save the workarea address since it is updated in order to perform
2287          * the concatenation
2288          */
2289         save = src.address;
2290
2291         /* Copy the ECC modulus */
2292         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2293         if (ret)
2294                 goto e_src;
2295         src.address += CCP_ECC_OPERAND_SIZE;
2296
2297         /* Copy the first point X and Y coordinate */
2298         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2299                                       ecc->u.pm.point_1.x_len);
2300         if (ret)
2301                 goto e_src;
2302         src.address += CCP_ECC_OPERAND_SIZE;
2303         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2304                                       ecc->u.pm.point_1.y_len);
2305         if (ret)
2306                 goto e_src;
2307         src.address += CCP_ECC_OPERAND_SIZE;
2308
2309         /* Set the first point Z coordinate to 1 */
2310         *src.address = 0x01;
2311         src.address += CCP_ECC_OPERAND_SIZE;
2312
2313         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2314                 /* Copy the second point X and Y coordinate */
2315                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2316                                               ecc->u.pm.point_2.x_len);
2317                 if (ret)
2318                         goto e_src;
2319                 src.address += CCP_ECC_OPERAND_SIZE;
2320                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2321                                               ecc->u.pm.point_2.y_len);
2322                 if (ret)
2323                         goto e_src;
2324                 src.address += CCP_ECC_OPERAND_SIZE;
2325
2326                 /* Set the second point Z coordinate to 1 */
2327                 *src.address = 0x01;
2328                 src.address += CCP_ECC_OPERAND_SIZE;
2329         } else {
2330                 /* Copy the Domain "a" parameter */
2331                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2332                                               ecc->u.pm.domain_a_len);
2333                 if (ret)
2334                         goto e_src;
2335                 src.address += CCP_ECC_OPERAND_SIZE;
2336
2337                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2338                         /* Copy the scalar value */
2339                         ret = ccp_reverse_set_dm_area(&src, 0,
2340                                                       ecc->u.pm.scalar, 0,
2341                                                       ecc->u.pm.scalar_len);
2342                         if (ret)
2343                                 goto e_src;
2344                         src.address += CCP_ECC_OPERAND_SIZE;
2345                 }
2346         }
2347
2348         /* Restore the workarea address */
2349         src.address = save;
2350
2351         /* Prepare the output area for the operation */
2352         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2353                                    DMA_FROM_DEVICE);
2354         if (ret)
2355                 goto e_src;
2356
2357         op.soc = 1;
2358         op.src.u.dma.address = src.dma.address;
2359         op.src.u.dma.offset = 0;
2360         op.src.u.dma.length = src.length;
2361         op.dst.u.dma.address = dst.dma.address;
2362         op.dst.u.dma.offset = 0;
2363         op.dst.u.dma.length = dst.length;
2364
2365         op.u.ecc.function = cmd->u.ecc.function;
2366
2367         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2368         if (ret) {
2369                 cmd->engine_error = cmd_q->cmd_error;
2370                 goto e_dst;
2371         }
2372
2373         ecc->ecc_result = le16_to_cpup(
2374                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2375         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2376                 ret = -EIO;
2377                 goto e_dst;
2378         }
2379
2380         /* Save the workarea address since it is updated as we walk through
2381          * to copy the point math result
2382          */
2383         save = dst.address;
2384
2385         /* Save the ECC result X and Y coordinates */
2386         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2387                                 CCP_ECC_MODULUS_BYTES);
2388         dst.address += CCP_ECC_OUTPUT_SIZE;
2389         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2390                                 CCP_ECC_MODULUS_BYTES);
2391         dst.address += CCP_ECC_OUTPUT_SIZE;
2392
2393         /* Restore the workarea address */
2394         dst.address = save;
2395
2396 e_dst:
2397         ccp_dm_free(&dst);
2398
2399 e_src:
2400         ccp_dm_free(&src);
2401
2402         return ret;
2403 }
2404
2405 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2406 {
2407         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2408
2409         ecc->ecc_result = 0;
2410
2411         if (!ecc->mod ||
2412             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2413                 return -EINVAL;
2414
2415         switch (ecc->function) {
2416         case CCP_ECC_FUNCTION_MMUL_384BIT:
2417         case CCP_ECC_FUNCTION_MADD_384BIT:
2418         case CCP_ECC_FUNCTION_MINV_384BIT:
2419                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2420
2421         case CCP_ECC_FUNCTION_PADD_384BIT:
2422         case CCP_ECC_FUNCTION_PMUL_384BIT:
2423         case CCP_ECC_FUNCTION_PDBL_384BIT:
2424                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2425
2426         default:
2427                 return -EINVAL;
2428         }
2429 }
2430
2431 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2432 {
2433         int ret;
2434
2435         cmd->engine_error = 0;
2436         cmd_q->cmd_error = 0;
2437         cmd_q->int_rcvd = 0;
2438         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2439
2440         switch (cmd->engine) {
2441         case CCP_ENGINE_AES:
2442                 ret = ccp_run_aes_cmd(cmd_q, cmd);
2443                 break;
2444         case CCP_ENGINE_XTS_AES_128:
2445                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2446                 break;
2447         case CCP_ENGINE_DES3:
2448                 ret = ccp_run_des3_cmd(cmd_q, cmd);
2449                 break;
2450         case CCP_ENGINE_SHA:
2451                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2452                 break;
2453         case CCP_ENGINE_RSA:
2454                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2455                 break;
2456         case CCP_ENGINE_PASSTHRU:
2457                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2458                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2459                 else
2460                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
2461                 break;
2462         case CCP_ENGINE_ECC:
2463                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2464                 break;
2465         default:
2466                 ret = -EINVAL;
2467         }
2468
2469         return ret;
2470 }