drivers/crypto/sa2ul.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * K3 SA2UL crypto accelerator driver
   4  *
   5  * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors:     Keerthy
   8  *              Vitaly Andrianov
   9  *              Tero Kristo
  10  */
  11 #include <linux/clk.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/dmapool.h>
  15 #include <linux/kernel.h>
  16 #include <linux/module.h>
  17 #include <linux/of_device.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/pm_runtime.h>
  20
  21 #include <crypto/aes.h>
  22 #include <crypto/authenc.h>
  23 #include <crypto/des.h>
  24 #include <crypto/internal/aead.h>
  25 #include <crypto/internal/hash.h>
  26 #include <crypto/internal/skcipher.h>
  27 #include <crypto/scatterwalk.h>
  28 #include <crypto/sha1.h>
  29 #include <crypto/sha2.h>
  30
  31 #include "sa2ul.h"
  32
  33 /* Byte offset for key in encryption security context */
  34 #define SC_ENC_KEY_OFFSET (1 + 27 + 4)
  35 /* Byte offset for Aux-1 in encryption security context */
  36 #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
  37
  38 #define SA_CMDL_UPD_ENC         0x0001
  39 #define SA_CMDL_UPD_AUTH        0x0002
  40 #define SA_CMDL_UPD_ENC_IV      0x0004
  41 #define SA_CMDL_UPD_AUTH_IV     0x0008
  42 #define SA_CMDL_UPD_AUX_KEY     0x0010
  43
  44 #define SA_AUTH_SUBKEY_LEN      16
  45 #define SA_CMDL_PAYLOAD_LENGTH_MASK     0xFFFF
  46 #define SA_CMDL_SOP_BYPASS_LEN_MASK     0xFF000000
  47
  48 #define MODE_CONTROL_BYTES      27
  49 #define SA_HASH_PROCESSING      0
  50 #define SA_CRYPTO_PROCESSING    0
  51 #define SA_UPLOAD_HASH_TO_TLR   BIT(6)
  52
  53 #define SA_SW0_FLAGS_MASK       0xF0000
  54 #define SA_SW0_CMDL_INFO_MASK   0x1F00000
  55 #define SA_SW0_CMDL_PRESENT     BIT(4)
  56 #define SA_SW0_ENG_ID_MASK      0x3E000000
  57 #define SA_SW0_DEST_INFO_PRESENT        BIT(30)
  58 #define SA_SW2_EGRESS_LENGTH            0xFF000000
  59 #define SA_BASIC_HASH           0x10
  60
  61 #define SHA256_DIGEST_WORDS    8
  62 /* Make 32-bit word from 4 bytes */
  63 #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
  64                                    ((b2) << 8) | (b3))
  65
  66 /* size of SCCTL structure in bytes */
  67 #define SA_SCCTL_SZ 16
  68
  69 /* Max Authentication tag size */
  70 #define SA_MAX_AUTH_TAG_SZ 64
  71
  72 #define PRIV_ID 0x1
  73 #define PRIV    0x1
  74
  75 static struct device *sa_k3_dev;
  76
  77 /**
  78  * struct sa_cmdl_cfg - Command label configuration descriptor
  79  * @aalg: authentication algorithm ID
  80  * @enc_eng_id: Encryption Engine ID supported by the SA hardware
  81  * @auth_eng_id: Authentication Engine ID
  82  * @iv_size: Initialization Vector size
  83  * @akey: Authentication key
  84  * @akey_len: Authentication key length
  85  * @enc: True, if this is an encode request
  86  */
  87 struct sa_cmdl_cfg {
  88         int aalg;
  89         u8 enc_eng_id;
  90         u8 auth_eng_id;
  91         u8 iv_size;
  92         const u8 *akey;
  93         u16 akey_len;
  94         bool enc;
  95 };
  96
  97 /**
  98  * struct algo_data - Crypto algorithm specific data
  99  * @enc_eng: Encryption engine info structure
 100  * @auth_eng: Authentication engine info structure
 101  * @auth_ctrl: Authentication control word
 102  * @hash_size: Size of digest
 103  * @iv_idx: iv index in psdata
 104  * @iv_out_size: iv out size
 105  * @ealg_id: Encryption Algorithm ID
 106  * @aalg_id: Authentication algorithm ID
 107  * @mci_enc: Mode Control Instruction for Encryption algorithm
 108  * @mci_dec: Mode Control Instruction for Decryption
 109  * @inv_key: Whether the encryption algorithm demands key inversion
 110  * @ctx: Pointer to the algorithm context
 111  * @keyed_mac: Whether the authentication algorithm has key
 112  * @prep_iopad: Function pointer to generate intermediate ipad/opad
 113  */
 114 struct algo_data {
 115         struct sa_eng_info enc_eng;
 116         struct sa_eng_info auth_eng;
 117         u8 auth_ctrl;
 118         u8 hash_size;
 119         u8 iv_idx;
 120         u8 iv_out_size;
 121         u8 ealg_id;
 122         u8 aalg_id;
 123         u8 *mci_enc;
 124         u8 *mci_dec;
 125         bool inv_key;
 126         struct sa_tfm_ctx *ctx;
 127         bool keyed_mac;
 128         void (*prep_iopad)(struct algo_data *algo, const u8 *key,
 129                            u16 key_sz, __be32 *ipad, __be32 *opad);
 130 };
 131
 132 /**
 133  * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
 134  * @type: Type of the crypto algorithm.
 135  * @alg: Union of crypto algorithm definitions.
 136  * @registered: Flag indicating if the crypto algorithm is already registered
 137  */
 138 struct sa_alg_tmpl {
 139         u32 type;               /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
 140         union {
 141                 struct skcipher_alg skcipher;
 142                 struct ahash_alg ahash;
 143                 struct aead_alg aead;
 144         } alg;
 145         bool registered;
 146 };
 147
 148 /**
 149  * struct sa_mapped_sg: scatterlist information for tx and rx
 150  * @mapped: Set to true if the @sgt is mapped
 151  * @dir: mapping direction used for @sgt
 152  * @split_sg: Set if the sg is split and needs to be freed up
 153  * @static_sg: Static scatterlist entry for overriding data
 154  * @sgt: scatterlist table for DMA API use
 155  */
 156 struct sa_mapped_sg {
 157         bool mapped;
 158         enum dma_data_direction dir;
 159         struct scatterlist static_sg;
 160         struct scatterlist *split_sg;
 161         struct sg_table sgt;
 162 };
 163 /**
 164  * struct sa_rx_data: RX Packet miscellaneous data place holder
 165  * @req: crypto request data pointer
 166  * @ddev: pointer to the DMA device
 167  * @tx_in: dma_async_tx_descriptor pointer for rx channel
 168  * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
 169  * @enc: Flag indicating either encryption or decryption
 170  * @enc_iv_size: Initialisation vector size
 171  * @iv_idx: Initialisation vector index
 172  */
 173 struct sa_rx_data {
 174         void *req;
 175         struct device *ddev;
 176         struct dma_async_tx_descriptor *tx_in;
 177         struct sa_mapped_sg mapped_sg[2];
 178         u8 enc;
 179         u8 enc_iv_size;
 180         u8 iv_idx;
 181 };
 182
 183 /**
 184  * struct sa_req: SA request definition
 185  * @dev: device for the request
 186  * @size: total data to the xmitted via DMA
 187  * @enc_offset: offset of cipher data
 188  * @enc_size: data to be passed to cipher engine
 189  * @enc_iv: cipher IV
 190  * @auth_offset: offset of the authentication data
 191  * @auth_size: size of the authentication data
 192  * @auth_iv: authentication IV
 193  * @type: algorithm type for the request
 194  * @cmdl: command label pointer
 195  * @base: pointer to the base request
 196  * @ctx: pointer to the algorithm context data
 197  * @enc: true if this is an encode request
 198  * @src: source data
 199  * @dst: destination data
 200  * @callback: DMA callback for the request
 201  * @mdata_size: metadata size passed to DMA
 202  */
 203 struct sa_req {
 204         struct device *dev;
 205         u16 size;
 206         u8 enc_offset;
 207         u16 enc_size;
 208         u8 *enc_iv;
 209         u8 auth_offset;
 210         u16 auth_size;
 211         u8 *auth_iv;
 212         u32 type;
 213         u32 *cmdl;
 214         struct crypto_async_request *base;
 215         struct sa_tfm_ctx *ctx;
 216         bool enc;
 217         struct scatterlist *src;
 218         struct scatterlist *dst;
 219         dma_async_tx_callback callback;
 220         u16 mdata_size;
 221 };
 222
 223 /*
 224  * Mode Control Instructions for various Key lengths 128, 192, 256
 225  * For CBC (Cipher Block Chaining) mode for encryption
 226  */
 227 static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
 228         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
 229                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 230                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 231         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
 232                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 233                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 234         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
 235                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 236                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 237 };
 238
 239 /*
 240  * Mode Control Instructions for various Key lengths 128, 192, 256
 241  * For CBC (Cipher Block Chaining) mode for decryption
 242  */
 243 static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
 244         {       0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 245                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 246                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 247         {       0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 248                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 249                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 250         {       0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 251                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 252                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 253 };
 254
 255 /*
 256  * Mode Control Instructions for various Key lengths 128, 192, 256
 257  * For CBC (Cipher Block Chaining) mode for encryption
 258  */
 259 static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = {
 260         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
 261                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 262                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 263         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
 264                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 265                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 266         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
 267                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 268                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 269 };
 270
 271 /*
 272  * Mode Control Instructions for various Key lengths 128, 192, 256
 273  * For CBC (Cipher Block Chaining) mode for decryption
 274  */
 275 static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = {
 276         {       0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 277                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 278                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 279         {       0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 280                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 281                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 282         {       0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 283                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 284                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 285 };
 286
 287 /*
 288  * Mode Control Instructions for various Key lengths 128, 192, 256
 289  * For ECB (Electronic Code Book) mode for encryption
 290  */
 291 static u8 mci_ecb_enc_array[3][27] = {
 292         {       0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 293                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 294                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 295         {       0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 296                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 297                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 298         {       0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 299                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 300                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 301 };
 302
 303 /*
 304  * Mode Control Instructions for various Key lengths 128, 192, 256
 305  * For ECB (Electronic Code Book) mode for decryption
 306  */
 307 static u8 mci_ecb_dec_array[3][27] = {
 308         {       0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 309                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 310                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 311         {       0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 312                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 313                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 314         {       0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 315                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 316                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 317 };
 318
 319 /*
 320  * Mode Control Instructions for DES algorithm
 321  * For CBC (Cipher Block Chaining) mode and ECB mode
 322  * encryption and for decryption respectively
 323  */
 324 static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
 325         0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
 326         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 327         0x00, 0x00, 0x00,
 328 };
 329
 330 static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
 331         0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
 332         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 333         0x00, 0x00, 0x00,
 334 };
 335
 336 static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
 337         0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
 338         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 339         0x00, 0x00, 0x00,
 340 };
 341
 342 static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
 343         0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
 344         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 345         0x00, 0x00, 0x00,
 346 };
 347
 348 /*
 349  * Perform 16 byte or 128 bit swizzling
 350  * The SA2UL Expects the security context to
 351  * be in little Endian and the bus width is 128 bits or 16 bytes
 352  * Hence swap 16 bytes at a time from higher to lower address
 353  */
 354 static void sa_swiz_128(u8 *in, u16 len)
 355 {
 356         u8 data[16];
 357         int i, j;
 358
 359         for (i = 0; i < len; i += 16) {
 360                 memcpy(data, &in[i], 16);
 361                 for (j = 0; j < 16; j++)
 362                         in[i + j] = data[15 - j];
 363         }
 364 }
 365
 366 /* Prepare the ipad and opad from key as per SHA algorithm step 1*/
 367 static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz)
 368 {
 369         int i;
 370
 371         for (i = 0; i < key_sz; i++)
 372                 k_ipad[i] = key[i] ^ 0x36;
 373
 374         /* Instead of XOR with 0 */
 375         for (; i < SHA1_BLOCK_SIZE; i++)
 376                 k_ipad[i] = 0x36;
 377 }
 378
 379 static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz)
 380 {
 381         int i;
 382
 383         for (i = 0; i < key_sz; i++)
 384                 k_opad[i] = key[i] ^ 0x5c;
 385
 386         /* Instead of XOR with 0 */
 387         for (; i < SHA1_BLOCK_SIZE; i++)
 388                 k_opad[i] = 0x5c;
 389 }
 390
 391 static void sa_export_shash(void *state, struct shash_desc *hash,
 392                             int digest_size, __be32 *out)
 393 {
 394         struct sha1_state *sha1;
 395         struct sha256_state *sha256;
 396         u32 *result;
 397
 398         switch (digest_size) {
 399         case SHA1_DIGEST_SIZE:
 400                 sha1 = state;
 401                 result = sha1->state;
 402                 break;
 403         case SHA256_DIGEST_SIZE:
 404                 sha256 = state;
 405                 result = sha256->state;
 406                 break;
 407         default:
 408                 dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__,
 409                         digest_size);
 410                 return;
 411         }
 412
 413         crypto_shash_export(hash, state);
 414
 415         cpu_to_be32_array(out, result, digest_size / 4);
 416 }
 417
 418 static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
 419                               u16 key_sz, __be32 *ipad, __be32 *opad)
 420 {
 421         SHASH_DESC_ON_STACK(shash, data->ctx->shash);
 422         int block_size = crypto_shash_blocksize(data->ctx->shash);
 423         int digest_size = crypto_shash_digestsize(data->ctx->shash);
 424         union {
 425                 struct sha1_state sha1;
 426                 struct sha256_state sha256;
 427                 u8 k_pad[SHA1_BLOCK_SIZE];
 428         } sha;
 429
 430         shash->tfm = data->ctx->shash;
 431
 432         prepare_kipad(sha.k_pad, key, key_sz);
 433
 434         crypto_shash_init(shash);
 435         crypto_shash_update(shash, sha.k_pad, block_size);
 436         sa_export_shash(&sha, shash, digest_size, ipad);
 437
 438         prepare_kopad(sha.k_pad, key, key_sz);
 439
 440         crypto_shash_init(shash);
 441         crypto_shash_update(shash, sha.k_pad, block_size);
 442
 443         sa_export_shash(&sha, shash, digest_size, opad);
 444
 445         memzero_explicit(&sha, sizeof(sha));
 446 }
 447
 448 /* Derive the inverse key used in AES-CBC decryption operation */
 449 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
 450 {
 451         struct crypto_aes_ctx ctx;
 452         int key_pos;
 453
 454         if (aes_expandkey(&ctx, key, key_sz)) {
 455                 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
 456                 return -EINVAL;
 457         }
 458
 459         /* work around to get the right inverse for AES_KEYSIZE_192 size keys */
 460         if (key_sz == AES_KEYSIZE_192) {
 461                 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46];
 462                 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47];
 463         }
 464
 465         /* Based crypto_aes_expand_key logic */
 466         switch (key_sz) {
 467         case AES_KEYSIZE_128:
 468         case AES_KEYSIZE_192:
 469                 key_pos = key_sz + 24;
 470                 break;
 471
 472         case AES_KEYSIZE_256:
 473                 key_pos = key_sz + 24 - 4;
 474                 break;
 475
 476         default:
 477                 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
 478                 return -EINVAL;
 479         }
 480
 481         memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
 482         return 0;
 483 }
 484
 485 /* Set Security context for the encryption engine */
 486 static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
 487                          u8 enc, u8 *sc_buf)
 488 {
 489         const u8 *mci = NULL;
 490
 491         /* Set Encryption mode selector to crypto processing */
 492         sc_buf[0] = SA_CRYPTO_PROCESSING;
 493
 494         if (enc)
 495                 mci = ad->mci_enc;
 496         else
 497                 mci = ad->mci_dec;
 498         /* Set the mode control instructions in security context */
 499         if (mci)
 500                 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
 501
 502         /* For AES-CBC decryption get the inverse key */
 503         if (ad->inv_key && !enc) {
 504                 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
 505                         return -EINVAL;
 506         /* For all other cases: key is used */
 507         } else {
 508                 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
 509         }
 510
 511         return 0;
 512 }
 513
 514 /* Set Security context for the authentication engine */
 515 static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
 516                            u8 *sc_buf)
 517 {
 518         __be32 *ipad = (void *)(sc_buf + 32);
 519         __be32 *opad = (void *)(sc_buf + 64);
 520
 521         /* Set Authentication mode selector to hash processing */
 522         sc_buf[0] = SA_HASH_PROCESSING;
 523         /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
 524         sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
 525         sc_buf[1] |= ad->auth_ctrl;
 526
 527         /* Copy the keys or ipad/opad */
 528         if (ad->keyed_mac)
 529                 ad->prep_iopad(ad, key, key_sz, ipad, opad);
 530         else {
 531                 /* basic hash */
 532                 sc_buf[1] |= SA_BASIC_HASH;
 533         }
 534 }
 535
 536 static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16)
 537 {
 538         int j;
 539
 540         for (j = 0; j < ((size16) ? 4 : 2); j++) {
 541                 *out = cpu_to_be32(*((u32 *)iv));
 542                 iv += 4;
 543                 out++;
 544         }
 545 }
 546
 547 /* Format general command label */
 548 static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
 549                               struct sa_cmdl_upd_info *upd_info)
 550 {
 551         u8 enc_offset = 0, auth_offset = 0, total = 0;
 552         u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
 553         u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
 554         u32 *word_ptr = (u32 *)cmdl;
 555         int i;
 556
 557         /* Clear the command label */
 558         memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
 559
 560         /* Iniialize the command update structure */
 561         memzero_explicit(upd_info, sizeof(*upd_info));
 562
 563         if (cfg->enc_eng_id && cfg->auth_eng_id) {
 564                 if (cfg->enc) {
 565                         auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
 566                         enc_next_eng = cfg->auth_eng_id;
 567
 568                         if (cfg->iv_size)
 569                                 auth_offset += cfg->iv_size;
 570                 } else {
 571                         enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
 572                         auth_next_eng = cfg->enc_eng_id;
 573                 }
 574         }
 575
 576         if (cfg->enc_eng_id) {
 577                 upd_info->flags |= SA_CMDL_UPD_ENC;
 578                 upd_info->enc_size.index = enc_offset >> 2;
 579                 upd_info->enc_offset.index = upd_info->enc_size.index + 1;
 580                 /* Encryption command label */
 581                 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
 582
 583                 /* Encryption modes requiring IV */
 584                 if (cfg->iv_size) {
 585                         upd_info->flags |= SA_CMDL_UPD_ENC_IV;
 586                         upd_info->enc_iv.index =
 587                                 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
 588                         upd_info->enc_iv.size = cfg->iv_size;
 589
 590                         cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 591                                 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
 592
 593                         cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
 594                                 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
 595                         total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
 596                 } else {
 597                         cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 598                                                 SA_CMDL_HEADER_SIZE_BYTES;
 599                         total += SA_CMDL_HEADER_SIZE_BYTES;
 600                 }
 601         }
 602
 603         if (cfg->auth_eng_id) {
 604                 upd_info->flags |= SA_CMDL_UPD_AUTH;
 605                 upd_info->auth_size.index = auth_offset >> 2;
 606                 upd_info->auth_offset.index = upd_info->auth_size.index + 1;
 607                 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
 608                 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 609                         SA_CMDL_HEADER_SIZE_BYTES;
 610                 total += SA_CMDL_HEADER_SIZE_BYTES;
 611         }
 612
 613         total = roundup(total, 8);
 614
 615         for (i = 0; i < total / 4; i++)
 616                 word_ptr[i] = swab32(word_ptr[i]);
 617
 618         return total;
 619 }
 620
 621 /* Update Command label */
 622 static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
 623                                   struct sa_cmdl_upd_info *upd_info)
 624 {
 625         int i = 0, j;
 626
 627         if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
 628                 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
 629                 cmdl[upd_info->enc_size.index] |= req->enc_size;
 630                 cmdl[upd_info->enc_offset.index] &=
 631                                                 ~SA_CMDL_SOP_BYPASS_LEN_MASK;
 632                 cmdl[upd_info->enc_offset.index] |=
 633                         ((u32)req->enc_offset <<
 634                          __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
 635
 636                 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
 637                         __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
 638                         u32 *enc_iv = (u32 *)req->enc_iv;
 639
 640                         for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
 641                                 data[j] = cpu_to_be32(*enc_iv);
 642                                 enc_iv++;
 643                         }
 644                 }
 645         }
 646
 647         if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
 648                 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
 649                 cmdl[upd_info->auth_size.index] |= req->auth_size;
 650                 cmdl[upd_info->auth_offset.index] &=
 651                         ~SA_CMDL_SOP_BYPASS_LEN_MASK;
 652                 cmdl[upd_info->auth_offset.index] |=
 653                         ((u32)req->auth_offset <<
 654                          __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
 655                 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
 656                         sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
 657                                    req->auth_iv,
 658                                    (upd_info->auth_iv.size > 8));
 659                 }
 660                 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
 661                         int offset = (req->auth_size & 0xF) ? 4 : 0;
 662
 663                         memcpy(&cmdl[upd_info->aux_key_info.index],
 664                                &upd_info->aux_key[offset], 16);
 665                 }
 666         }
 667 }
 668
 669 /* Format SWINFO words to be sent to SA */
 670 static
 671 void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
 672                    u8 cmdl_present, u8 cmdl_offset, u8 flags,
 673                    u8 hash_size, u32 *swinfo)
 674 {
 675         swinfo[0] = sc_id;
 676         swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK));
 677         if (likely(cmdl_present))
 678                 swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) <<
 679                                                 __ffs(SA_SW0_CMDL_INFO_MASK));
 680         swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK));
 681
 682         swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
 683         swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
 684         swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
 685         swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH));
 686 }
 687
 688 /* Dump the security context */
 689 static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
 690 {
 691 #ifdef DEBUG
 692         dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
 693         print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
 694                        16, 1, buf, SA_CTX_MAX_SZ, false);
 695 #endif
 696 }
 697
 698 static
 699 int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key,
 700                u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz,
 701                struct algo_data *ad, u8 enc, u32 *swinfo)
 702 {
 703         int enc_sc_offset = 0;
 704         int auth_sc_offset = 0;
 705         u8 *sc_buf = ctx->sc;
 706         u16 sc_id = ctx->sc_id;
 707         u8 first_engine = 0;
 708
 709         memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
 710
 711         if (ad->auth_eng.eng_id) {
 712                 if (enc)
 713                         first_engine = ad->enc_eng.eng_id;
 714                 else
 715                         first_engine = ad->auth_eng.eng_id;
 716
 717                 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
 718                 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
 719                 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
 720                 if (!ad->hash_size)
 721                         return -EINVAL;
 722                 ad->hash_size = roundup(ad->hash_size, 8);
 723
 724         } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) {
 725                 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
 726                 first_engine = ad->enc_eng.eng_id;
 727                 sc_buf[1] = SA_SCCTL_FE_ENC;
 728                 ad->hash_size = ad->iv_out_size;
 729         }
 730
 731         /* SCCTL Owner info: 0=host, 1=CP_ACE */
 732         sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
 733         memcpy(&sc_buf[2], &sc_id, 2);
 734         sc_buf[4] = 0x0;
 735         sc_buf[5] = PRIV_ID;
 736         sc_buf[6] = PRIV;
 737         sc_buf[7] = 0x0;
 738
 739         /* Prepare context for encryption engine */
 740         if (ad->enc_eng.sc_size) {
 741                 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc,
 742                                   &sc_buf[enc_sc_offset]))
 743                         return -EINVAL;
 744         }
 745
 746         /* Prepare context for authentication engine */
 747         if (ad->auth_eng.sc_size)
 748                 sa_set_sc_auth(ad, auth_key, auth_key_sz,
 749                                &sc_buf[auth_sc_offset]);
 750
 751         /* Set the ownership of context to CP_ACE */
 752         sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
 753
 754         /* swizzle the security context */
 755         sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
 756
 757         sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
 758                       SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
 759
 760         sa_dump_sc(sc_buf, ctx->sc_phys);
 761
 762         return 0;
 763 }
 764
 765 /* Free the per direction context memory */
 766 static void sa_free_ctx_info(struct sa_ctx_info *ctx,
 767                              struct sa_crypto_data *data)
 768 {
 769         unsigned long bn;
 770
 771         bn = ctx->sc_id - data->sc_id_start;
 772         spin_lock(&data->scid_lock);
 773         __clear_bit(bn, data->ctx_bm);
 774         data->sc_id--;
 775         spin_unlock(&data->scid_lock);
 776
 777         if (ctx->sc) {
 778                 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
 779                 ctx->sc = NULL;
 780         }
 781 }
 782
 783 static int sa_init_ctx_info(struct sa_ctx_info *ctx,
 784                             struct sa_crypto_data *data)
 785 {
 786         unsigned long bn;
 787         int err;
 788
 789         spin_lock(&data->scid_lock);
 790         bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
 791         __set_bit(bn, data->ctx_bm);
 792         data->sc_id++;
 793         spin_unlock(&data->scid_lock);
 794
 795         ctx->sc_id = (u16)(data->sc_id_start + bn);
 796
 797         ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
 798         if (!ctx->sc) {
 799                 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
 800                 err = -ENOMEM;
 801                 goto scid_rollback;
 802         }
 803
 804         return 0;
 805
 806 scid_rollback:
 807         spin_lock(&data->scid_lock);
 808         __clear_bit(bn, data->ctx_bm);
 809         data->sc_id--;
 810         spin_unlock(&data->scid_lock);
 811
 812         return err;
 813 }
 814
 815 static void sa_cipher_cra_exit(struct crypto_skcipher *tfm)
 816 {
 817         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 818         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
 819
 820         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
 821                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
 822                 ctx->dec.sc_id, &ctx->dec.sc_phys);
 823
 824         sa_free_ctx_info(&ctx->enc, data);
 825         sa_free_ctx_info(&ctx->dec, data);
 826
 827         crypto_free_skcipher(ctx->fallback.skcipher);
 828 }
 829
 830 static int sa_cipher_cra_init(struct crypto_skcipher *tfm)
 831 {
 832         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 833         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
 834         const char *name = crypto_tfm_alg_name(&tfm->base);
 835         struct crypto_skcipher *child;
 836         int ret;
 837
 838         memzero_explicit(ctx, sizeof(*ctx));
 839         ctx->dev_data = data;
 840
 841         ret = sa_init_ctx_info(&ctx->enc, data);
 842         if (ret)
 843                 return ret;
 844         ret = sa_init_ctx_info(&ctx->dec, data);
 845         if (ret) {
 846                 sa_free_ctx_info(&ctx->enc, data);
 847                 return ret;
 848         }
 849
 850         child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
 851
 852         if (IS_ERR(child)) {
 853                 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name);
 854                 return PTR_ERR(child);
 855         }
 856
 857         ctx->fallback.skcipher = child;
 858         crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
 859                                          sizeof(struct skcipher_request));
 860
 861         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
 862                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
 863                 ctx->dec.sc_id, &ctx->dec.sc_phys);
 864         return 0;
 865 }
 866
 867 static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 868                             unsigned int keylen, struct algo_data *ad)
 869 {
 870         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 871         struct crypto_skcipher *child = ctx->fallback.skcipher;
 872         int cmdl_len;
 873         struct sa_cmdl_cfg cfg;
 874         int ret;
 875
 876         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 877             keylen != AES_KEYSIZE_256)
 878                 return -EINVAL;
 879
 880         ad->enc_eng.eng_id = SA_ENG_ID_EM1;
 881         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
 882
 883         memzero_explicit(&cfg, sizeof(cfg));
 884         cfg.enc_eng_id = ad->enc_eng.eng_id;
 885         cfg.iv_size = crypto_skcipher_ivsize(tfm);
 886
 887         crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 888         crypto_skcipher_set_flags(child, tfm->base.crt_flags &
 889                                          CRYPTO_TFM_REQ_MASK);
 890         ret = crypto_skcipher_setkey(child, key, keylen);
 891         if (ret)
 892                 return ret;
 893
 894         /* Setup Encryption Security Context & Command label template */
 895         if (sa_init_sc(&ctx->enc, key, keylen, NULL, 0, ad, 1,
 896                        &ctx->enc.epib[1]))
 897                 goto badkey;
 898
 899         cmdl_len = sa_format_cmdl_gen(&cfg,
 900                                       (u8 *)ctx->enc.cmdl,
 901                                       &ctx->enc.cmdl_upd_info);
 902         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
 903                 goto badkey;
 904
 905         ctx->enc.cmdl_size = cmdl_len;
 906
 907         /* Setup Decryption Security Context & Command label template */
 908         if (sa_init_sc(&ctx->dec, key, keylen, NULL, 0, ad, 0,
 909                        &ctx->dec.epib[1]))
 910                 goto badkey;
 911
 912         cfg.enc_eng_id = ad->enc_eng.eng_id;
 913         cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
 914                                       &ctx->dec.cmdl_upd_info);
 915
 916         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
 917                 goto badkey;
 918
 919         ctx->dec.cmdl_size = cmdl_len;
 920         ctx->iv_idx = ad->iv_idx;
 921
 922         return 0;
 923
 924 badkey:
 925         dev_err(sa_k3_dev, "%s: badkey\n", __func__);
 926         return -EINVAL;
 927 }
 928
 929 static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
 930                              unsigned int keylen)
 931 {
 932         struct algo_data ad = { 0 };
 933         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
 934         int key_idx = (keylen >> 3) - 2;
 935
 936         if (key_idx >= 3)
 937                 return -EINVAL;
 938
 939         ad.mci_enc = mci_cbc_enc_array[key_idx];
 940         ad.mci_dec = mci_cbc_dec_array[key_idx];
 941         ad.inv_key = true;
 942         ad.ealg_id = SA_EALG_ID_AES_CBC;
 943         ad.iv_idx = 4;
 944         ad.iv_out_size = 16;
 945
 946         return sa_cipher_setkey(tfm, key, keylen, &ad);
 947 }
 948
 949 static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
 950                              unsigned int keylen)
 951 {
 952         struct algo_data ad = { 0 };
 953         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
 954         int key_idx = (keylen >> 3) - 2;
 955
 956         if (key_idx >= 3)
 957                 return -EINVAL;
 958
 959         ad.mci_enc = mci_ecb_enc_array[key_idx];
 960         ad.mci_dec = mci_ecb_dec_array[key_idx];
 961         ad.inv_key = true;
 962         ad.ealg_id = SA_EALG_ID_AES_ECB;
 963
 964         return sa_cipher_setkey(tfm, key, keylen, &ad);
 965 }
 966
 967 static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
 968                               unsigned int keylen)
 969 {
 970         struct algo_data ad = { 0 };
 971
 972         ad.mci_enc = mci_cbc_3des_enc_array;
 973         ad.mci_dec = mci_cbc_3des_dec_array;
 974         ad.ealg_id = SA_EALG_ID_3DES_CBC;
 975         ad.iv_idx = 6;
 976         ad.iv_out_size = 8;
 977
 978         return sa_cipher_setkey(tfm, key, keylen, &ad);
 979 }
 980
 981 static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
 982                               unsigned int keylen)
 983 {
 984         struct algo_data ad = { 0 };
 985
 986         ad.mci_enc = mci_ecb_3des_enc_array;
 987         ad.mci_dec = mci_ecb_3des_dec_array;
 988
 989         return sa_cipher_setkey(tfm, key, keylen, &ad);
 990 }
 991
 992 static void sa_sync_from_device(struct sa_rx_data *rxd)
 993 {
 994         struct sg_table *sgt;
 995
 996         if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
 997                 sgt = &rxd->mapped_sg[0].sgt;
 998         else
 999                 sgt = &rxd->mapped_sg[1].sgt;
1000
1001         dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
1002 }
1003
1004 static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
1005 {
1006         int i;
1007
1008         for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
1009                 struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
1010
1011                 if (mapped_sg->mapped) {
1012                         dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
1013                                           mapped_sg->dir, 0);
1014                         kfree(mapped_sg->split_sg);
1015                 }
1016         }
1017
1018         kfree(rxd);
1019 }
1020
1021 static void sa_aes_dma_in_callback(void *data)
1022 {
1023         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1024         struct skcipher_request *req;
1025         u32 *result;
1026         __be32 *mdptr;
1027         size_t ml, pl;
1028         int i;
1029
1030         sa_sync_from_device(rxd);
1031         req = container_of(rxd->req, struct skcipher_request, base);
1032
1033         if (req->iv) {
1034                 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
1035                                                                &ml);
1036                 result = (u32 *)req->iv;
1037
1038                 for (i = 0; i < (rxd->enc_iv_size / 4); i++)
1039                         result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
1040         }
1041
1042         sa_free_sa_rx_data(rxd);
1043
1044         skcipher_request_complete(req, 0);
1045 }
1046
1047 static void
1048 sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
1049 {
1050         u32 *out, *in;
1051         int i;
1052
1053         for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
1054                 *out++ = *in++;
1055
1056         mdptr[4] = (0xFFFF << 16);
1057         for (out = &mdptr[5], in = psdata, i = 0;
1058              i < pslen / sizeof(u32); i++)
1059                 *out++ = *in++;
1060 }
1061
1062 static int sa_run(struct sa_req *req)
1063 {
1064         struct sa_rx_data *rxd;
1065         gfp_t gfp_flags;
1066         u32 cmdl[SA_MAX_CMDL_WORDS];
1067         struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
1068         struct device *ddev;
1069         struct dma_chan *dma_rx;
1070         int sg_nents, src_nents, dst_nents;
1071         struct scatterlist *src, *dst;
1072         size_t pl, ml, split_size;
1073         struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
1074         int ret;
1075         struct dma_async_tx_descriptor *tx_out;
1076         u32 *mdptr;
1077         bool diff_dst;
1078         enum dma_data_direction dir_src;
1079         struct sa_mapped_sg *mapped_sg;
1080
1081         gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1082                 GFP_KERNEL : GFP_ATOMIC;
1083
1084         rxd = kzalloc(sizeof(*rxd), gfp_flags);
1085         if (!rxd)
1086                 return -ENOMEM;
1087
1088         if (req->src != req->dst) {
1089                 diff_dst = true;
1090                 dir_src = DMA_TO_DEVICE;
1091         } else {
1092                 diff_dst = false;
1093                 dir_src = DMA_BIDIRECTIONAL;
1094         }
1095
1096         /*
1097          * SA2UL has an interesting feature where the receive DMA channel
1098          * is selected based on the data passed to the engine. Within the
1099          * transition range, there is also a space where it is impossible
1100          * to determine where the data will end up, and this should be
1101          * avoided. This will be handled by the SW fallback mechanism by
1102          * the individual algorithm implementations.
1103          */
1104         if (req->size >= 256)
1105                 dma_rx = pdata->dma_rx2;
1106         else
1107                 dma_rx = pdata->dma_rx1;
1108
1109         ddev = dma_rx->device->dev;
1110         rxd->ddev = ddev;
1111
1112         memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
1113
1114         sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info);
1115
1116         if (req->type != CRYPTO_ALG_TYPE_AHASH) {
1117                 if (req->enc)
1118                         req->type |=
1119                                 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
1120                 else
1121                         req->type |=
1122                                 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
1123         }
1124
1125         cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type;
1126
1127         /*
1128          * Map the packets, first we check if the data fits into a single
1129          * sg entry and use that if possible. If it does not fit, we check
1130          * if we need to do sg_split to align the scatterlist data on the
1131          * actual data size being processed by the crypto engine.
1132          */
1133         src = req->src;
1134         sg_nents = sg_nents_for_len(src, req->size);
1135
1136         split_size = req->size;
1137
1138         mapped_sg = &rxd->mapped_sg[0];
1139         if (sg_nents == 1 && split_size <= req->src->length) {
1140                 src = &mapped_sg->static_sg;
1141                 src_nents = 1;
1142                 sg_init_table(src, 1);
1143                 sg_set_page(src, sg_page(req->src), split_size,
1144                             req->src->offset);
1145
1146                 mapped_sg->sgt.sgl = src;
1147                 mapped_sg->sgt.orig_nents = src_nents;
1148                 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1149                 if (ret)
1150                         return ret;
1151
1152                 mapped_sg->dir = dir_src;
1153                 mapped_sg->mapped = true;
1154         } else {
1155                 mapped_sg->sgt.sgl = req->src;
1156                 mapped_sg->sgt.orig_nents = sg_nents;
1157                 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1158                 if (ret)
1159                         return ret;
1160
1161                 mapped_sg->dir = dir_src;
1162                 mapped_sg->mapped = true;
1163
1164                 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
1165                                &split_size, &src, &src_nents, gfp_flags);
1166                 if (ret) {
1167                         src_nents = mapped_sg->sgt.nents;
1168                         src = mapped_sg->sgt.sgl;
1169                 } else {
1170                         mapped_sg->split_sg = src;
1171                 }
1172         }
1173
1174         dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
1175
1176         if (!diff_dst) {
1177                 dst_nents = src_nents;
1178                 dst = src;
1179         } else {
1180                 dst_nents = sg_nents_for_len(req->dst, req->size);
1181                 mapped_sg = &rxd->mapped_sg[1];
1182
1183                 if (dst_nents == 1 && split_size <= req->dst->length) {
1184                         dst = &mapped_sg->static_sg;
1185                         dst_nents = 1;
1186                         sg_init_table(dst, 1);
1187                         sg_set_page(dst, sg_page(req->dst), split_size,
1188                                     req->dst->offset);
1189
1190                         mapped_sg->sgt.sgl = dst;
1191                         mapped_sg->sgt.orig_nents = dst_nents;
1192                         ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
1193                                               DMA_FROM_DEVICE, 0);
1194                         if (ret)
1195                                 goto err_cleanup;
1196
1197                         mapped_sg->dir = DMA_FROM_DEVICE;
1198                         mapped_sg->mapped = true;
1199                 } else {
1200                         mapped_sg->sgt.sgl = req->dst;
1201                         mapped_sg->sgt.orig_nents = dst_nents;
1202                         ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
1203                                               DMA_FROM_DEVICE, 0);
1204                         if (ret)
1205                                 goto err_cleanup;
1206
1207                         mapped_sg->dir = DMA_FROM_DEVICE;
1208                         mapped_sg->mapped = true;
1209
1210                         ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
1211                                        0, 1, &split_size, &dst, &dst_nents,
1212                                        gfp_flags);
1213                         if (ret) {
1214                                 dst_nents = mapped_sg->sgt.nents;
1215                                 dst = mapped_sg->sgt.sgl;
1216                         } else {
1217                                 mapped_sg->split_sg = dst;
1218                         }
1219                 }
1220         }
1221
1222         rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
1223                                              DMA_DEV_TO_MEM,
1224                                              DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1225         if (!rxd->tx_in) {
1226                 dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
1227                 ret = -EINVAL;
1228                 goto err_cleanup;
1229         }
1230
1231         rxd->req = (void *)req->base;
1232         rxd->enc = req->enc;
1233         rxd->iv_idx = req->ctx->iv_idx;
1234         rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
1235         rxd->tx_in->callback = req->callback;
1236         rxd->tx_in->callback_param = rxd;
1237
1238         tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src,
1239                                          src_nents, DMA_MEM_TO_DEV,
1240                                          DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1241
1242         if (!tx_out) {
1243                 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
1244                 ret = -EINVAL;
1245                 goto err_cleanup;
1246         }
1247
1248         /*
1249          * Prepare metadata for DMA engine. This essentially describes the
1250          * crypto algorithm to be used, data sizes, different keys etc.
1251          */
1252         mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
1253
1254         sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
1255                                    sizeof(u32))), cmdl, sizeof(sa_ctx->epib),
1256                            sa_ctx->epib);
1257
1258         ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
1259         dmaengine_desc_set_metadata_len(tx_out, req->mdata_size);
1260
1261         dmaengine_submit(tx_out);
1262         dmaengine_submit(rxd->tx_in);
1263
1264         dma_async_issue_pending(dma_rx);
1265         dma_async_issue_pending(pdata->dma_tx);
1266
1267         return -EINPROGRESS;
1268
1269 err_cleanup:
1270         sa_free_sa_rx_data(rxd);
1271
1272         return ret;
1273 }
1274
1275 static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
1276 {
1277         struct sa_tfm_ctx *ctx =
1278             crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1279         struct crypto_alg *alg = req->base.tfm->__crt_alg;
1280         struct sa_req sa_req = { 0 };
1281
1282         if (!req->cryptlen)
1283                 return 0;
1284
1285         if (req->cryptlen % alg->cra_blocksize)
1286                 return -EINVAL;
1287
1288         /* Use SW fallback if the data size is not supported */
1289         if (req->cryptlen > SA_MAX_DATA_SZ ||
1290             (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN &&
1291              req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) {
1292                 struct skcipher_request *subreq = skcipher_request_ctx(req);
1293
1294                 skcipher_request_set_tfm(subreq, ctx->fallback.skcipher);
1295                 skcipher_request_set_callback(subreq, req->base.flags,
1296                                               req->base.complete,
1297                                               req->base.data);
1298                 skcipher_request_set_crypt(subreq, req->src, req->dst,
1299                                            req->cryptlen, req->iv);
1300                 if (enc)
1301                         return crypto_skcipher_encrypt(subreq);
1302                 else
1303                         return crypto_skcipher_decrypt(subreq);
1304         }
1305
1306         sa_req.size = req->cryptlen;
1307         sa_req.enc_size = req->cryptlen;
1308         sa_req.src = req->src;
1309         sa_req.dst = req->dst;
1310         sa_req.enc_iv = iv;
1311         sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
1312         sa_req.enc = enc;
1313         sa_req.callback = sa_aes_dma_in_callback;
1314         sa_req.mdata_size = 44;
1315         sa_req.base = &req->base;
1316         sa_req.ctx = ctx;
1317
1318         return sa_run(&sa_req);
1319 }
1320
1321 static int sa_encrypt(struct skcipher_request *req)
1322 {
1323         return sa_cipher_run(req, req->iv, 1);
1324 }
1325
1326 static int sa_decrypt(struct skcipher_request *req)
1327 {
1328         return sa_cipher_run(req, req->iv, 0);
1329 }
1330
1331 static void sa_sha_dma_in_callback(void *data)
1332 {
1333         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1334         struct ahash_request *req;
1335         struct crypto_ahash *tfm;
1336         unsigned int authsize;
1337         int i;
1338         size_t ml, pl;
1339         u32 *result;
1340         __be32 *mdptr;
1341
1342         sa_sync_from_device(rxd);
1343         req = container_of(rxd->req, struct ahash_request, base);
1344         tfm = crypto_ahash_reqtfm(req);
1345         authsize = crypto_ahash_digestsize(tfm);
1346
1347         mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1348         result = (u32 *)req->result;
1349
1350         for (i = 0; i < (authsize / 4); i++)
1351                 result[i] = be32_to_cpu(mdptr[i + 4]);
1352
1353         sa_free_sa_rx_data(rxd);
1354
1355         ahash_request_complete(req, 0);
1356 }
1357
1358 static int zero_message_process(struct ahash_request *req)
1359 {
1360         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1361         int sa_digest_size = crypto_ahash_digestsize(tfm);
1362
1363         switch (sa_digest_size) {
1364         case SHA1_DIGEST_SIZE:
1365                 memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
1366                 break;
1367         case SHA256_DIGEST_SIZE:
1368                 memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
1369                 break;
1370         case SHA512_DIGEST_SIZE:
1371                 memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
1372                 break;
1373         default:
1374                 return -EINVAL;
1375         }
1376
1377         return 0;
1378 }
1379
1380 static int sa_sha_run(struct ahash_request *req)
1381 {
1382         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1383         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1384         struct sa_req sa_req = { 0 };
1385         size_t auth_len;
1386
1387         auth_len = req->nbytes;
1388
1389         if (!auth_len)
1390                 return zero_message_process(req);
1391
1392         if (auth_len > SA_MAX_DATA_SZ ||
1393             (auth_len >= SA_UNSAFE_DATA_SZ_MIN &&
1394              auth_len <= SA_UNSAFE_DATA_SZ_MAX)) {
1395                 struct ahash_request *subreq = &rctx->fallback_req;
1396                 int ret = 0;
1397
1398                 ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1399                 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1400
1401                 crypto_ahash_init(subreq);
1402
1403                 subreq->nbytes = auth_len;
1404                 subreq->src = req->src;
1405                 subreq->result = req->result;
1406
1407                 ret |= crypto_ahash_update(subreq);
1408
1409                 subreq->nbytes = 0;
1410
1411                 ret |= crypto_ahash_final(subreq);
1412
1413                 return ret;
1414         }
1415
1416         sa_req.size = auth_len;
1417         sa_req.auth_size = auth_len;
1418         sa_req.src = req->src;
1419         sa_req.dst = req->src;
1420         sa_req.enc = true;
1421         sa_req.type = CRYPTO_ALG_TYPE_AHASH;
1422         sa_req.callback = sa_sha_dma_in_callback;
1423         sa_req.mdata_size = 28;
1424         sa_req.ctx = ctx;
1425         sa_req.base = &req->base;
1426
1427         return sa_run(&sa_req);
1428 }
1429
1430 static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct  algo_data *ad)
1431 {
1432         int bs = crypto_shash_blocksize(ctx->shash);
1433         int cmdl_len;
1434         struct sa_cmdl_cfg cfg;
1435
1436         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1437         ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1438         ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1439
1440         memset(ctx->authkey, 0, bs);
1441         memset(&cfg, 0, sizeof(cfg));
1442         cfg.aalg = ad->aalg_id;
1443         cfg.enc_eng_id = ad->enc_eng.eng_id;
1444         cfg.auth_eng_id = ad->auth_eng.eng_id;
1445         cfg.iv_size = 0;
1446         cfg.akey = NULL;
1447         cfg.akey_len = 0;
1448
1449         /* Setup Encryption Security Context & Command label template */
1450         if (sa_init_sc(&ctx->enc, NULL, 0, NULL, 0, ad, 0,
1451                        &ctx->enc.epib[1]))
1452                 goto badkey;
1453
1454         cmdl_len = sa_format_cmdl_gen(&cfg,
1455                                       (u8 *)ctx->enc.cmdl,
1456                                       &ctx->enc.cmdl_upd_info);
1457         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1458                 goto badkey;
1459
1460         ctx->enc.cmdl_size = cmdl_len;
1461
1462         return 0;
1463
1464 badkey:
1465         dev_err(sa_k3_dev, "%s: badkey\n", __func__);
1466         return -EINVAL;
1467 }
1468
1469 static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
1470 {
1471         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1472         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1473         int ret;
1474
1475         memset(ctx, 0, sizeof(*ctx));
1476         ctx->dev_data = data;
1477         ret = sa_init_ctx_info(&ctx->enc, data);
1478         if (ret)
1479                 return ret;
1480
1481         if (alg_base) {
1482                 ctx->shash = crypto_alloc_shash(alg_base, 0,
1483                                                 CRYPTO_ALG_NEED_FALLBACK);
1484                 if (IS_ERR(ctx->shash)) {
1485                         dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n",
1486                                 alg_base);
1487                         return PTR_ERR(ctx->shash);
1488                 }
1489                 /* for fallback */
1490                 ctx->fallback.ahash =
1491                         crypto_alloc_ahash(alg_base, 0,
1492                                            CRYPTO_ALG_NEED_FALLBACK);
1493                 if (IS_ERR(ctx->fallback.ahash)) {
1494                         dev_err(ctx->dev_data->dev,
1495                                 "Could not load fallback driver\n");
1496                         return PTR_ERR(ctx->fallback.ahash);
1497                 }
1498         }
1499
1500         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1501                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1502                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1503
1504         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1505                                  sizeof(struct sa_sha_req_ctx) +
1506                                  crypto_ahash_reqsize(ctx->fallback.ahash));
1507
1508         return 0;
1509 }
1510
1511 static int sa_sha_digest(struct ahash_request *req)
1512 {
1513         return sa_sha_run(req);
1514 }
1515
1516 static int sa_sha_init(struct ahash_request *req)
1517 {
1518         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1519         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1520         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1521
1522         dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
1523                 crypto_ahash_digestsize(tfm), rctx);
1524
1525         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1526         rctx->fallback_req.base.flags =
1527                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1528
1529         return crypto_ahash_init(&rctx->fallback_req);
1530 }
1531
1532 static int sa_sha_update(struct ahash_request *req)
1533 {
1534         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1535         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1536         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1537
1538         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1539         rctx->fallback_req.base.flags =
1540                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1541         rctx->fallback_req.nbytes = req->nbytes;
1542         rctx->fallback_req.src = req->src;
1543
1544         return crypto_ahash_update(&rctx->fallback_req);
1545 }
1546
1547 static int sa_sha_final(struct ahash_request *req)
1548 {
1549         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1550         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1551         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1552
1553         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1554         rctx->fallback_req.base.flags =
1555                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1556         rctx->fallback_req.result = req->result;
1557
1558         return crypto_ahash_final(&rctx->fallback_req);
1559 }
1560
1561 static int sa_sha_finup(struct ahash_request *req)
1562 {
1563         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1564         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1565         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1566
1567         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1568         rctx->fallback_req.base.flags =
1569                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1570
1571         rctx->fallback_req.nbytes = req->nbytes;
1572         rctx->fallback_req.src = req->src;
1573         rctx->fallback_req.result = req->result;
1574
1575         return crypto_ahash_finup(&rctx->fallback_req);
1576 }
1577
1578 static int sa_sha_import(struct ahash_request *req, const void *in)
1579 {
1580         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1581         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1582         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1583
1584         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1585         rctx->fallback_req.base.flags = req->base.flags &
1586                 CRYPTO_TFM_REQ_MAY_SLEEP;
1587
1588         return crypto_ahash_import(&rctx->fallback_req, in);
1589 }
1590
1591 static int sa_sha_export(struct ahash_request *req, void *out)
1592 {
1593         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1594         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1595         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1596         struct ahash_request *subreq = &rctx->fallback_req;
1597
1598         ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1599         subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1600
1601         return crypto_ahash_export(subreq, out);
1602 }
1603
1604 static int sa_sha1_cra_init(struct crypto_tfm *tfm)
1605 {
1606         struct algo_data ad = { 0 };
1607         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1608
1609         sa_sha_cra_init_alg(tfm, "sha1");
1610
1611         ad.aalg_id = SA_AALG_ID_SHA1;
1612         ad.hash_size = SHA1_DIGEST_SIZE;
1613         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1614
1615         sa_sha_setup(ctx, &ad);
1616
1617         return 0;
1618 }
1619
1620 static int sa_sha256_cra_init(struct crypto_tfm *tfm)
1621 {
1622         struct algo_data ad = { 0 };
1623         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1624
1625         sa_sha_cra_init_alg(tfm, "sha256");
1626
1627         ad.aalg_id = SA_AALG_ID_SHA2_256;
1628         ad.hash_size = SHA256_DIGEST_SIZE;
1629         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1630
1631         sa_sha_setup(ctx, &ad);
1632
1633         return 0;
1634 }
1635
1636 static int sa_sha512_cra_init(struct crypto_tfm *tfm)
1637 {
1638         struct algo_data ad = { 0 };
1639         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1640
1641         sa_sha_cra_init_alg(tfm, "sha512");
1642
1643         ad.aalg_id = SA_AALG_ID_SHA2_512;
1644         ad.hash_size = SHA512_DIGEST_SIZE;
1645         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512;
1646
1647         sa_sha_setup(ctx, &ad);
1648
1649         return 0;
1650 }
1651
1652 static void sa_sha_cra_exit(struct crypto_tfm *tfm)
1653 {
1654         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1655         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1656
1657         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1658                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1659                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1660
1661         if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH)
1662                 sa_free_ctx_info(&ctx->enc, data);
1663
1664         crypto_free_shash(ctx->shash);
1665         crypto_free_ahash(ctx->fallback.ahash);
1666 }
1667
1668 static void sa_aead_dma_in_callback(void *data)
1669 {
1670         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1671         struct aead_request *req;
1672         struct crypto_aead *tfm;
1673         unsigned int start;
1674         unsigned int authsize;
1675         u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
1676         size_t pl, ml;
1677         int i;
1678         int err = 0;
1679         u16 auth_len;
1680         u32 *mdptr;
1681
1682         sa_sync_from_device(rxd);
1683         req = container_of(rxd->req, struct aead_request, base);
1684         tfm = crypto_aead_reqtfm(req);
1685         start = req->assoclen + req->cryptlen;
1686         authsize = crypto_aead_authsize(tfm);
1687
1688         mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1689         for (i = 0; i < (authsize / 4); i++)
1690                 mdptr[i + 4] = swab32(mdptr[i + 4]);
1691
1692         auth_len = req->assoclen + req->cryptlen;
1693
1694         if (rxd->enc) {
1695                 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1696                                          1);
1697         } else {
1698                 auth_len -= authsize;
1699                 start -= authsize;
1700                 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
1701                                          0);
1702
1703                 err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
1704         }
1705
1706         sa_free_sa_rx_data(rxd);
1707
1708         aead_request_complete(req, err);
1709 }
1710
1711 static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash,
1712                             const char *fallback)
1713 {
1714         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1715         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1716         int ret;
1717
1718         memzero_explicit(ctx, sizeof(*ctx));
1719
1720         ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK);
1721         if (IS_ERR(ctx->shash)) {
1722                 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash);
1723                 return PTR_ERR(ctx->shash);
1724         }
1725
1726         ctx->fallback.aead = crypto_alloc_aead(fallback, 0,
1727                                                CRYPTO_ALG_NEED_FALLBACK);
1728
1729         if (IS_ERR(ctx->fallback.aead)) {
1730                 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n",
1731                         fallback);
1732                 return PTR_ERR(ctx->fallback.aead);
1733         }
1734
1735         crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
1736                                 crypto_aead_reqsize(ctx->fallback.aead));
1737
1738         ret = sa_init_ctx_info(&ctx->enc, data);
1739         if (ret)
1740                 return ret;
1741
1742         ret = sa_init_ctx_info(&ctx->dec, data);
1743         if (ret) {
1744                 sa_free_ctx_info(&ctx->enc, data);
1745                 return ret;
1746         }
1747
1748         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1749                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1750                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1751
1752         return ret;
1753 }
1754
1755 static int sa_cra_init_aead_sha1(struct crypto_aead *tfm)
1756 {
1757         return sa_cra_init_aead(tfm, "sha1",
1758                                 "authenc(hmac(sha1-ce),cbc(aes-ce))");
1759 }
1760
1761 static int sa_cra_init_aead_sha256(struct crypto_aead *tfm)
1762 {
1763         return sa_cra_init_aead(tfm, "sha256",
1764                                 "authenc(hmac(sha256-ce),cbc(aes-ce))");
1765 }
1766
1767 static void sa_exit_tfm_aead(struct crypto_aead *tfm)
1768 {
1769         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1770         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1771
1772         crypto_free_shash(ctx->shash);
1773         crypto_free_aead(ctx->fallback.aead);
1774
1775         sa_free_ctx_info(&ctx->enc, data);
1776         sa_free_ctx_info(&ctx->dec, data);
1777 }
1778
1779 /* AEAD algorithm configuration interface function */
1780 static int sa_aead_setkey(struct crypto_aead *authenc,
1781                           const u8 *key, unsigned int keylen,
1782                           struct algo_data *ad)
1783 {
1784         struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
1785         struct crypto_authenc_keys keys;
1786         int cmdl_len;
1787         struct sa_cmdl_cfg cfg;
1788         int key_idx;
1789
1790         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1791                 return -EINVAL;
1792
1793         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
1794         key_idx = (keys.enckeylen >> 3) - 2;
1795         if (key_idx >= 3)
1796                 return -EINVAL;
1797
1798         ad->ctx = ctx;
1799         ad->enc_eng.eng_id = SA_ENG_ID_EM1;
1800         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1801         ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1802         ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1803         ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx];
1804         ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx];
1805         ad->inv_key = true;
1806         ad->keyed_mac = true;
1807         ad->ealg_id = SA_EALG_ID_AES_CBC;
1808         ad->prep_iopad = sa_prepare_iopads;
1809
1810         memset(&cfg, 0, sizeof(cfg));
1811         cfg.enc = true;
1812         cfg.aalg = ad->aalg_id;
1813         cfg.enc_eng_id = ad->enc_eng.eng_id;
1814         cfg.auth_eng_id = ad->auth_eng.eng_id;
1815         cfg.iv_size = crypto_aead_ivsize(authenc);
1816         cfg.akey = keys.authkey;
1817         cfg.akey_len = keys.authkeylen;
1818
1819         /* Setup Encryption Security Context & Command label template */
1820         if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen,
1821                        keys.authkey, keys.authkeylen,
1822                        ad, 1, &ctx->enc.epib[1]))
1823                 return -EINVAL;
1824
1825         cmdl_len = sa_format_cmdl_gen(&cfg,
1826                                       (u8 *)ctx->enc.cmdl,
1827                                       &ctx->enc.cmdl_upd_info);
1828         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1829                 return -EINVAL;
1830
1831         ctx->enc.cmdl_size = cmdl_len;
1832
1833         /* Setup Decryption Security Context & Command label template */
1834         if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen,
1835                        keys.authkey, keys.authkeylen,
1836                        ad, 0, &ctx->dec.epib[1]))
1837                 return -EINVAL;
1838
1839         cfg.enc = false;
1840         cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
1841                                       &ctx->dec.cmdl_upd_info);
1842
1843         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1844                 return -EINVAL;
1845
1846         ctx->dec.cmdl_size = cmdl_len;
1847
1848         crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK);
1849         crypto_aead_set_flags(ctx->fallback.aead,
1850                               crypto_aead_get_flags(authenc) &
1851                               CRYPTO_TFM_REQ_MASK);
1852         crypto_aead_setkey(ctx->fallback.aead, key, keylen);
1853
1854         return 0;
1855 }
1856
1857 static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1858 {
1859         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
1860
1861         return crypto_aead_setauthsize(ctx->fallback.aead, authsize);
1862 }
1863
1864 static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
1865                                    const u8 *key, unsigned int keylen)
1866 {
1867         struct algo_data ad = { 0 };
1868
1869         ad.ealg_id = SA_EALG_ID_AES_CBC;
1870         ad.aalg_id = SA_AALG_ID_HMAC_SHA1;
1871         ad.hash_size = SHA1_DIGEST_SIZE;
1872         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1873
1874         return sa_aead_setkey(authenc, key, keylen, &ad);
1875 }
1876
1877 static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
1878                                      const u8 *key, unsigned int keylen)
1879 {
1880         struct algo_data ad = { 0 };
1881
1882         ad.ealg_id = SA_EALG_ID_AES_CBC;
1883         ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256;
1884         ad.hash_size = SHA256_DIGEST_SIZE;
1885         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1886
1887         return sa_aead_setkey(authenc, key, keylen, &ad);
1888 }
1889
1890 static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
1891 {
1892         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1893         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1894         struct sa_req sa_req = { 0 };
1895         size_t auth_size, enc_size;
1896
1897         enc_size = req->cryptlen;
1898         auth_size = req->assoclen + req->cryptlen;
1899
1900         if (!enc) {
1901                 enc_size -= crypto_aead_authsize(tfm);
1902                 auth_size -= crypto_aead_authsize(tfm);
1903         }
1904
1905         if (auth_size > SA_MAX_DATA_SZ ||
1906             (auth_size >= SA_UNSAFE_DATA_SZ_MIN &&
1907              auth_size <= SA_UNSAFE_DATA_SZ_MAX)) {
1908                 struct aead_request *subreq = aead_request_ctx(req);
1909                 int ret;
1910
1911                 aead_request_set_tfm(subreq, ctx->fallback.aead);
1912                 aead_request_set_callback(subreq, req->base.flags,
1913                                           req->base.complete, req->base.data);
1914                 aead_request_set_crypt(subreq, req->src, req->dst,
1915                                        req->cryptlen, req->iv);
1916                 aead_request_set_ad(subreq, req->assoclen);
1917
1918                 ret = enc ? crypto_aead_encrypt(subreq) :
1919                         crypto_aead_decrypt(subreq);
1920                 return ret;
1921         }
1922
1923         sa_req.enc_offset = req->assoclen;
1924         sa_req.enc_size = enc_size;
1925         sa_req.auth_size = auth_size;
1926         sa_req.size = auth_size;
1927         sa_req.enc_iv = iv;
1928         sa_req.type = CRYPTO_ALG_TYPE_AEAD;
1929         sa_req.enc = enc;
1930         sa_req.callback = sa_aead_dma_in_callback;
1931         sa_req.mdata_size = 52;
1932         sa_req.base = &req->base;
1933         sa_req.ctx = ctx;
1934         sa_req.src = req->src;
1935         sa_req.dst = req->dst;
1936
1937         return sa_run(&sa_req);
1938 }
1939
1940 /* AEAD algorithm encrypt interface function */
1941 static int sa_aead_encrypt(struct aead_request *req)
1942 {
1943         return sa_aead_run(req, req->iv, 1);
1944 }
1945
1946 /* AEAD algorithm decrypt interface function */
1947 static int sa_aead_decrypt(struct aead_request *req)
1948 {
1949         return sa_aead_run(req, req->iv, 0);
1950 }
1951
1952 static struct sa_alg_tmpl sa_algs[] = {
1953         {
1954                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1955                 .alg.skcipher = {
1956                         .base.cra_name          = "cbc(aes)",
1957                         .base.cra_driver_name   = "cbc-aes-sa2ul",
1958                         .base.cra_priority      = 30000,
1959                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
1960                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
1961                                                   CRYPTO_ALG_ASYNC |
1962                                                   CRYPTO_ALG_NEED_FALLBACK,
1963                         .base.cra_blocksize     = AES_BLOCK_SIZE,
1964                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
1965                         .base.cra_module        = THIS_MODULE,
1966                         .init                   = sa_cipher_cra_init,
1967                         .exit                   = sa_cipher_cra_exit,
1968                         .min_keysize            = AES_MIN_KEY_SIZE,
1969                         .max_keysize            = AES_MAX_KEY_SIZE,
1970                         .ivsize                 = AES_BLOCK_SIZE,
1971                         .setkey                 = sa_aes_cbc_setkey,
1972                         .encrypt                = sa_encrypt,
1973                         .decrypt                = sa_decrypt,
1974                 }
1975         },
1976         {
1977                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1978                 .alg.skcipher = {
1979                         .base.cra_name          = "ecb(aes)",
1980                         .base.cra_driver_name   = "ecb-aes-sa2ul",
1981                         .base.cra_priority      = 30000,
1982                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
1983                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
1984                                                   CRYPTO_ALG_ASYNC |
1985                                                   CRYPTO_ALG_NEED_FALLBACK,
1986                         .base.cra_blocksize     = AES_BLOCK_SIZE,
1987                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
1988                         .base.cra_module        = THIS_MODULE,
1989                         .init                   = sa_cipher_cra_init,
1990                         .exit                   = sa_cipher_cra_exit,
1991                         .min_keysize            = AES_MIN_KEY_SIZE,
1992                         .max_keysize            = AES_MAX_KEY_SIZE,
1993                         .setkey                 = sa_aes_ecb_setkey,
1994                         .encrypt                = sa_encrypt,
1995                         .decrypt                = sa_decrypt,
1996                 }
1997         },
1998         {
1999                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2000                 .alg.skcipher = {
2001                         .base.cra_name          = "cbc(des3_ede)",
2002                         .base.cra_driver_name   = "cbc-des3-sa2ul",
2003                         .base.cra_priority      = 30000,
2004                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
2005                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2006                                                   CRYPTO_ALG_ASYNC |
2007                                                   CRYPTO_ALG_NEED_FALLBACK,
2008                         .base.cra_blocksize     = DES_BLOCK_SIZE,
2009                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
2010                         .base.cra_module        = THIS_MODULE,
2011                         .init                   = sa_cipher_cra_init,
2012                         .exit                   = sa_cipher_cra_exit,
2013                         .min_keysize            = 3 * DES_KEY_SIZE,
2014                         .max_keysize            = 3 * DES_KEY_SIZE,
2015                         .ivsize                 = DES_BLOCK_SIZE,
2016                         .setkey                 = sa_3des_cbc_setkey,
2017                         .encrypt                = sa_encrypt,
2018                         .decrypt                = sa_decrypt,
2019                 }
2020         },
2021         {
2022                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2023                 .alg.skcipher = {
2024                         .base.cra_name          = "ecb(des3_ede)",
2025                         .base.cra_driver_name   = "ecb-des3-sa2ul",
2026                         .base.cra_priority      = 30000,
2027                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
2028                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2029                                                   CRYPTO_ALG_ASYNC |
2030                                                   CRYPTO_ALG_NEED_FALLBACK,
2031                         .base.cra_blocksize     = DES_BLOCK_SIZE,
2032                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
2033                         .base.cra_module        = THIS_MODULE,
2034                         .init                   = sa_cipher_cra_init,
2035                         .exit                   = sa_cipher_cra_exit,
2036                         .min_keysize            = 3 * DES_KEY_SIZE,
2037                         .max_keysize            = 3 * DES_KEY_SIZE,
2038                         .setkey                 = sa_3des_ecb_setkey,
2039                         .encrypt                = sa_encrypt,
2040                         .decrypt                = sa_decrypt,
2041                 }
2042         },
2043         {
2044                 .type = CRYPTO_ALG_TYPE_AHASH,
2045                 .alg.ahash = {
2046                         .halg.base = {
2047                                 .cra_name       = "sha1",
2048                                 .cra_driver_name        = "sha1-sa2ul",
2049                                 .cra_priority   = 400,
2050                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2051                                                   CRYPTO_ALG_ASYNC |
2052                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2053                                                   CRYPTO_ALG_NEED_FALLBACK,
2054                                 .cra_blocksize  = SHA1_BLOCK_SIZE,
2055                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2056                                 .cra_module     = THIS_MODULE,
2057                                 .cra_init       = sa_sha1_cra_init,
2058                                 .cra_exit       = sa_sha_cra_exit,
2059                         },
2060                         .halg.digestsize        = SHA1_DIGEST_SIZE,
2061                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2062                                                   sizeof(struct sha1_state),
2063                         .init                   = sa_sha_init,
2064                         .update                 = sa_sha_update,
2065                         .final                  = sa_sha_final,
2066                         .finup                  = sa_sha_finup,
2067                         .digest                 = sa_sha_digest,
2068                         .export                 = sa_sha_export,
2069                         .import                 = sa_sha_import,
2070                 },
2071         },
2072         {
2073                 .type = CRYPTO_ALG_TYPE_AHASH,
2074                 .alg.ahash = {
2075                         .halg.base = {
2076                                 .cra_name       = "sha256",
2077                                 .cra_driver_name        = "sha256-sa2ul",
2078                                 .cra_priority   = 400,
2079                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2080                                                   CRYPTO_ALG_ASYNC |
2081                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2082                                                   CRYPTO_ALG_NEED_FALLBACK,
2083                                 .cra_blocksize  = SHA256_BLOCK_SIZE,
2084                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2085                                 .cra_module     = THIS_MODULE,
2086                                 .cra_init       = sa_sha256_cra_init,
2087                                 .cra_exit       = sa_sha_cra_exit,
2088                         },
2089                         .halg.digestsize        = SHA256_DIGEST_SIZE,
2090                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2091                                                   sizeof(struct sha256_state),
2092                         .init                   = sa_sha_init,
2093                         .update                 = sa_sha_update,
2094                         .final                  = sa_sha_final,
2095                         .finup                  = sa_sha_finup,
2096                         .digest                 = sa_sha_digest,
2097                         .export                 = sa_sha_export,
2098                         .import                 = sa_sha_import,
2099                 },
2100         },
2101         {
2102                 .type = CRYPTO_ALG_TYPE_AHASH,
2103                 .alg.ahash = {
2104                         .halg.base = {
2105                                 .cra_name       = "sha512",
2106                                 .cra_driver_name        = "sha512-sa2ul",
2107                                 .cra_priority   = 400,
2108                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2109                                                   CRYPTO_ALG_ASYNC |
2110                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2111                                                   CRYPTO_ALG_NEED_FALLBACK,
2112                                 .cra_blocksize  = SHA512_BLOCK_SIZE,
2113                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2114                                 .cra_module     = THIS_MODULE,
2115                                 .cra_init       = sa_sha512_cra_init,
2116                                 .cra_exit       = sa_sha_cra_exit,
2117                         },
2118                         .halg.digestsize        = SHA512_DIGEST_SIZE,
2119                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2120                                                   sizeof(struct sha512_state),
2121                         .init                   = sa_sha_init,
2122                         .update                 = sa_sha_update,
2123                         .final                  = sa_sha_final,
2124                         .finup                  = sa_sha_finup,
2125                         .digest                 = sa_sha_digest,
2126                         .export                 = sa_sha_export,
2127                         .import                 = sa_sha_import,
2128                 },
2129         },
2130         {
2131                 .type   = CRYPTO_ALG_TYPE_AEAD,
2132                 .alg.aead = {
2133                         .base = {
2134                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2135                                 .cra_driver_name =
2136                                         "authenc(hmac(sha1),cbc(aes))-sa2ul",
2137                                 .cra_blocksize = AES_BLOCK_SIZE,
2138                                 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2139                                         CRYPTO_ALG_KERN_DRIVER_ONLY |
2140                                         CRYPTO_ALG_ASYNC |
2141                                         CRYPTO_ALG_NEED_FALLBACK,
2142                                 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2143                                 .cra_module = THIS_MODULE,
2144                                 .cra_priority = 3000,
2145                         },
2146                         .ivsize = AES_BLOCK_SIZE,
2147                         .maxauthsize = SHA1_DIGEST_SIZE,
2148
2149                         .init = sa_cra_init_aead_sha1,
2150                         .exit = sa_exit_tfm_aead,
2151                         .setkey = sa_aead_cbc_sha1_setkey,
2152                         .setauthsize = sa_aead_setauthsize,
2153                         .encrypt = sa_aead_encrypt,
2154                         .decrypt = sa_aead_decrypt,
2155                 },
2156         },
2157         {
2158                 .type   = CRYPTO_ALG_TYPE_AEAD,
2159                 .alg.aead = {
2160                         .base = {
2161                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2162                                 .cra_driver_name =
2163                                         "authenc(hmac(sha256),cbc(aes))-sa2ul",
2164                                 .cra_blocksize = AES_BLOCK_SIZE,
2165                                 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2166                                         CRYPTO_ALG_KERN_DRIVER_ONLY |
2167                                         CRYPTO_ALG_ASYNC |
2168                                         CRYPTO_ALG_NEED_FALLBACK,
2169                                 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2170                                 .cra_module = THIS_MODULE,
2171                                 .cra_alignmask = 0,
2172                                 .cra_priority = 3000,
2173                         },
2174                         .ivsize = AES_BLOCK_SIZE,
2175                         .maxauthsize = SHA256_DIGEST_SIZE,
2176
2177                         .init = sa_cra_init_aead_sha256,
2178                         .exit = sa_exit_tfm_aead,
2179                         .setkey = sa_aead_cbc_sha256_setkey,
2180                         .setauthsize = sa_aead_setauthsize,
2181                         .encrypt = sa_aead_encrypt,
2182                         .decrypt = sa_aead_decrypt,
2183                 },
2184         },
2185 };
2186
2187 /* Register the algorithms in crypto framework */
2188 static void sa_register_algos(const struct device *dev)
2189 {
2190         char *alg_name;
2191         u32 type;
2192         int i, err;
2193
2194         for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2195                 type = sa_algs[i].type;
2196                 if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
2197                         alg_name = sa_algs[i].alg.skcipher.base.cra_name;
2198                         err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
2199                 } else if (type == CRYPTO_ALG_TYPE_AHASH) {
2200                         alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
2201                         err = crypto_register_ahash(&sa_algs[i].alg.ahash);
2202                 } else if (type == CRYPTO_ALG_TYPE_AEAD) {
2203                         alg_name = sa_algs[i].alg.aead.base.cra_name;
2204                         err = crypto_register_aead(&sa_algs[i].alg.aead);
2205                 } else {
2206                         dev_err(dev,
2207                                 "un-supported crypto algorithm (%d)",
2208                                 sa_algs[i].type);
2209                         continue;
2210                 }
2211
2212                 if (err)
2213                         dev_err(dev, "Failed to register '%s'\n", alg_name);
2214                 else
2215                         sa_algs[i].registered = true;
2216         }
2217 }
2218
2219 /* Unregister the algorithms in crypto framework */
2220 static void sa_unregister_algos(const struct device *dev)
2221 {
2222         u32 type;
2223         int i;
2224
2225         for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2226                 type = sa_algs[i].type;
2227                 if (!sa_algs[i].registered)
2228                         continue;
2229                 if (type == CRYPTO_ALG_TYPE_SKCIPHER)
2230                         crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
2231                 else if (type == CRYPTO_ALG_TYPE_AHASH)
2232                         crypto_unregister_ahash(&sa_algs[i].alg.ahash);
2233                 else if (type == CRYPTO_ALG_TYPE_AEAD)
2234                         crypto_unregister_aead(&sa_algs[i].alg.aead);
2235
2236                 sa_algs[i].registered = false;
2237         }
2238 }
2239
2240 static int sa_init_mem(struct sa_crypto_data *dev_data)
2241 {
2242         struct device *dev = &dev_data->pdev->dev;
2243         /* Setup dma pool for security context buffers */
2244         dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
2245                                             SA_CTX_MAX_SZ, 64, 0);
2246         if (!dev_data->sc_pool) {
2247                 dev_err(dev, "Failed to create dma pool");
2248                 return -ENOMEM;
2249         }
2250
2251         return 0;
2252 }
2253
2254 static int sa_dma_init(struct sa_crypto_data *dd)
2255 {
2256         int ret;
2257         struct dma_slave_config cfg;
2258
2259         dd->dma_rx1 = NULL;
2260         dd->dma_tx = NULL;
2261         dd->dma_rx2 = NULL;
2262
2263         ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
2264         if (ret)
2265                 return ret;
2266
2267         dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
2268         if (IS_ERR(dd->dma_rx1))
2269                 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
2270                                      "Unable to request rx1 DMA channel\n");
2271
2272         dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
2273         if (IS_ERR(dd->dma_rx2)) {
2274                 dma_release_channel(dd->dma_rx1);
2275                 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
2276                                      "Unable to request rx2 DMA channel\n");
2277         }
2278
2279         dd->dma_tx = dma_request_chan(dd->dev, "tx");
2280         if (IS_ERR(dd->dma_tx)) {
2281                 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
2282                                     "Unable to request tx DMA channel\n");
2283                 goto err_dma_tx;
2284         }
2285
2286         memzero_explicit(&cfg, sizeof(cfg));
2287
2288         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2289         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2290         cfg.src_maxburst = 4;
2291         cfg.dst_maxburst = 4;
2292
2293         ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
2294         if (ret) {
2295                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2296                         ret);
2297                 return ret;
2298         }
2299
2300         ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
2301         if (ret) {
2302                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2303                         ret);
2304                 return ret;
2305         }
2306
2307         ret = dmaengine_slave_config(dd->dma_tx, &cfg);
2308         if (ret) {
2309                 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
2310                         ret);
2311                 return ret;
2312         }
2313
2314         return 0;
2315
2316 err_dma_tx:
2317         dma_release_channel(dd->dma_rx1);
2318         dma_release_channel(dd->dma_rx2);
2319
2320         return ret;
2321 }
2322
2323 static int sa_link_child(struct device *dev, void *data)
2324 {
2325         struct device *parent = data;
2326
2327         device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER);
2328
2329         return 0;
2330 }
2331
2332 static int sa_ul_probe(struct platform_device *pdev)
2333 {
2334         struct device *dev = &pdev->dev;
2335         struct device_node *node = dev->of_node;
2336         struct resource *res;
2337         static void __iomem *saul_base;
2338         struct sa_crypto_data *dev_data;
2339         u32 val;
2340         int ret;
2341
2342         dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
2343         if (!dev_data)
2344                 return -ENOMEM;
2345
2346         sa_k3_dev = dev;
2347         dev_data->dev = dev;
2348         dev_data->pdev = pdev;
2349         platform_set_drvdata(pdev, dev_data);
2350         dev_set_drvdata(sa_k3_dev, dev_data);
2351
2352         pm_runtime_enable(dev);
2353         ret = pm_runtime_get_sync(dev);
2354         if (ret < 0) {
2355                 dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__,
2356                         ret);
2357                 return ret;
2358         }
2359
2360         sa_init_mem(dev_data);
2361         ret = sa_dma_init(dev_data);
2362         if (ret)
2363                 goto disable_pm_runtime;
2364
2365         spin_lock_init(&dev_data->scid_lock);
2366         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2367         saul_base = devm_ioremap_resource(dev, res);
2368
2369         dev_data->base = saul_base;
2370         val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
2371             SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN |
2372             SA_EEC_TRNG_EN;
2373
2374         writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
2375
2376         sa_register_algos(dev);
2377
2378         ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
2379         if (ret)
2380                 goto release_dma;
2381
2382         device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child);
2383
2384         return 0;
2385
2386 release_dma:
2387         sa_unregister_algos(&pdev->dev);
2388
2389         dma_release_channel(dev_data->dma_rx2);
2390         dma_release_channel(dev_data->dma_rx1);
2391         dma_release_channel(dev_data->dma_tx);
2392
2393         dma_pool_destroy(dev_data->sc_pool);
2394
2395 disable_pm_runtime:
2396         pm_runtime_put_sync(&pdev->dev);
2397         pm_runtime_disable(&pdev->dev);
2398
2399         return ret;
2400 }
2401
2402 static int sa_ul_remove(struct platform_device *pdev)
2403 {
2404         struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
2405
2406         sa_unregister_algos(&pdev->dev);
2407
2408         dma_release_channel(dev_data->dma_rx2);
2409         dma_release_channel(dev_data->dma_rx1);
2410         dma_release_channel(dev_data->dma_tx);
2411
2412         dma_pool_destroy(dev_data->sc_pool);
2413
2414         platform_set_drvdata(pdev, NULL);
2415
2416         pm_runtime_put_sync(&pdev->dev);
2417         pm_runtime_disable(&pdev->dev);
2418
2419         return 0;
2420 }
2421
2422 static const struct of_device_id of_match[] = {
2423         {.compatible = "ti,j721e-sa2ul",},
2424         {.compatible = "ti,am654-sa2ul",},
2425         {},
2426 };
2427 MODULE_DEVICE_TABLE(of, of_match);
2428
2429 static struct platform_driver sa_ul_driver = {
2430         .probe = sa_ul_probe,
2431         .remove = sa_ul_remove,
2432         .driver = {
2433                    .name = "saul-crypto",
2434                    .of_match_table = of_match,
2435                    },
2436 };
2437 module_platform_driver(sa_ul_driver);
2438 MODULE_LICENSE("GPL v2");