drivers/crypto/sa2ul.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * K3 SA2UL crypto accelerator driver
   4  *
   5  * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors:     Keerthy
   8  *              Vitaly Andrianov
   9  *              Tero Kristo
  10  */
  11 #include <linux/clk.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/dmapool.h>
  14 #include <linux/module.h>
  15 #include <linux/of_device.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/pm_runtime.h>
  18
  19 #include <crypto/aes.h>
  20 #include <crypto/authenc.h>
  21 #include <crypto/des.h>
  22 #include <crypto/internal/aead.h>
  23 #include <crypto/internal/hash.h>
  24 #include <crypto/internal/skcipher.h>
  25 #include <crypto/scatterwalk.h>
  26 #include <crypto/sha.h>
  27
  28 #include "sa2ul.h"
  29
  30 /* Byte offset for key in encryption security context */
  31 #define SC_ENC_KEY_OFFSET (1 + 27 + 4)
  32 /* Byte offset for Aux-1 in encryption security context */
  33 #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
  34
  35 #define SA_CMDL_UPD_ENC         0x0001
  36 #define SA_CMDL_UPD_AUTH        0x0002
  37 #define SA_CMDL_UPD_ENC_IV      0x0004
  38 #define SA_CMDL_UPD_AUTH_IV     0x0008
  39 #define SA_CMDL_UPD_AUX_KEY     0x0010
  40
  41 #define SA_AUTH_SUBKEY_LEN      16
  42 #define SA_CMDL_PAYLOAD_LENGTH_MASK     0xFFFF
  43 #define SA_CMDL_SOP_BYPASS_LEN_MASK     0xFF000000
  44
  45 #define MODE_CONTROL_BYTES      27
  46 #define SA_HASH_PROCESSING      0
  47 #define SA_CRYPTO_PROCESSING    0
  48 #define SA_UPLOAD_HASH_TO_TLR   BIT(6)
  49
  50 #define SA_SW0_FLAGS_MASK       0xF0000
  51 #define SA_SW0_CMDL_INFO_MASK   0x1F00000
  52 #define SA_SW0_CMDL_PRESENT     BIT(4)
  53 #define SA_SW0_ENG_ID_MASK      0x3E000000
  54 #define SA_SW0_DEST_INFO_PRESENT        BIT(30)
  55 #define SA_SW2_EGRESS_LENGTH            0xFF000000
  56 #define SA_BASIC_HASH           0x10
  57
  58 #define SHA256_DIGEST_WORDS    8
  59 /* Make 32-bit word from 4 bytes */
  60 #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
  61                                    ((b2) << 8) | (b3))
  62
  63 /* size of SCCTL structure in bytes */
  64 #define SA_SCCTL_SZ 16
  65
  66 /* Max Authentication tag size */
  67 #define SA_MAX_AUTH_TAG_SZ 64
  68
  69 #define PRIV_ID 0x1
  70 #define PRIV    0x1
  71
  72 static struct device *sa_k3_dev;
  73
  74 /**
  75  * struct sa_cmdl_cfg - Command label configuration descriptor
  76  * @aalg: authentication algorithm ID
  77  * @enc_eng_id: Encryption Engine ID supported by the SA hardware
  78  * @auth_eng_id: Authentication Engine ID
  79  * @iv_size: Initialization Vector size
  80  * @akey: Authentication key
  81  * @akey_len: Authentication key length
  82  * @enc: True, if this is an encode request
  83  */
  84 struct sa_cmdl_cfg {
  85         int aalg;
  86         u8 enc_eng_id;
  87         u8 auth_eng_id;
  88         u8 iv_size;
  89         const u8 *akey;
  90         u16 akey_len;
  91         bool enc;
  92 };
  93
  94 /**
  95  * struct algo_data - Crypto algorithm specific data
  96  * @enc_eng: Encryption engine info structure
  97  * @auth_eng: Authentication engine info structure
  98  * @auth_ctrl: Authentication control word
  99  * @hash_size: Size of digest
 100  * @iv_idx: iv index in psdata
 101  * @iv_out_size: iv out size
 102  * @ealg_id: Encryption Algorithm ID
 103  * @aalg_id: Authentication algorithm ID
 104  * @mci_enc: Mode Control Instruction for Encryption algorithm
 105  * @mci_dec: Mode Control Instruction for Decryption
 106  * @inv_key: Whether the encryption algorithm demands key inversion
 107  * @ctx: Pointer to the algorithm context
 108  * @keyed_mac: Whether the authentication algorithm has key
 109  * @prep_iopad: Function pointer to generate intermediate ipad/opad
 110  */
 111 struct algo_data {
 112         struct sa_eng_info enc_eng;
 113         struct sa_eng_info auth_eng;
 114         u8 auth_ctrl;
 115         u8 hash_size;
 116         u8 iv_idx;
 117         u8 iv_out_size;
 118         u8 ealg_id;
 119         u8 aalg_id;
 120         u8 *mci_enc;
 121         u8 *mci_dec;
 122         bool inv_key;
 123         struct sa_tfm_ctx *ctx;
 124         bool keyed_mac;
 125         void (*prep_iopad)(struct algo_data *algo, const u8 *key,
 126                            u16 key_sz, __be32 *ipad, __be32 *opad);
 127 };
 128
 129 /**
 130  * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
 131  * @type: Type of the crypto algorithm.
 132  * @alg: Union of crypto algorithm definitions.
 133  * @registered: Flag indicating if the crypto algorithm is already registered
 134  */
 135 struct sa_alg_tmpl {
 136         u32 type;               /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
 137         union {
 138                 struct skcipher_alg skcipher;
 139                 struct ahash_alg ahash;
 140                 struct aead_alg aead;
 141         } alg;
 142         bool registered;
 143 };
 144
 145 /**
 146  * struct sa_rx_data: RX Packet miscellaneous data place holder
 147  * @req: crypto request data pointer
 148  * @ddev: pointer to the DMA device
 149  * @tx_in: dma_async_tx_descriptor pointer for rx channel
 150  * @split_src_sg: Set if the src sg is split and needs to be freed up
 151  * @split_dst_sg: Set if the dst sg is split and needs to be freed up
 152  * @enc: Flag indicating either encryption or decryption
 153  * @enc_iv_size: Initialisation vector size
 154  * @iv_idx: Initialisation vector index
 155  * @rx_sg: Static scatterlist entry for overriding RX data
 156  * @tx_sg: Static scatterlist entry for overriding TX data
 157  * @src: Source data pointer
 158  * @dst: Destination data pointer
 159  */
 160 struct sa_rx_data {
 161         void *req;
 162         struct device *ddev;
 163         struct dma_async_tx_descriptor *tx_in;
 164         struct scatterlist *split_src_sg;
 165         struct scatterlist *split_dst_sg;
 166         u8 enc;
 167         u8 enc_iv_size;
 168         u8 iv_idx;
 169         struct scatterlist rx_sg;
 170         struct scatterlist tx_sg;
 171         struct scatterlist *src;
 172         struct scatterlist *dst;
 173 };
 174
 175 /**
 176  * struct sa_req: SA request definition
 177  * @dev: device for the request
 178  * @size: total data to the xmitted via DMA
 179  * @enc_offset: offset of cipher data
 180  * @enc_size: data to be passed to cipher engine
 181  * @enc_iv: cipher IV
 182  * @auth_offset: offset of the authentication data
 183  * @auth_size: size of the authentication data
 184  * @auth_iv: authentication IV
 185  * @type: algorithm type for the request
 186  * @cmdl: command label pointer
 187  * @base: pointer to the base request
 188  * @ctx: pointer to the algorithm context data
 189  * @enc: true if this is an encode request
 190  * @src: source data
 191  * @dst: destination data
 192  * @callback: DMA callback for the request
 193  * @mdata_size: metadata size passed to DMA
 194  */
 195 struct sa_req {
 196         struct device *dev;
 197         u16 size;
 198         u8 enc_offset;
 199         u16 enc_size;
 200         u8 *enc_iv;
 201         u8 auth_offset;
 202         u16 auth_size;
 203         u8 *auth_iv;
 204         u32 type;
 205         u32 *cmdl;
 206         struct crypto_async_request *base;
 207         struct sa_tfm_ctx *ctx;
 208         bool enc;
 209         struct scatterlist *src;
 210         struct scatterlist *dst;
 211         dma_async_tx_callback callback;
 212         u16 mdata_size;
 213 };
 214
 215 /*
 216  * Mode Control Instructions for various Key lengths 128, 192, 256
 217  * For CBC (Cipher Block Chaining) mode for encryption
 218  */
 219 static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
 220         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
 221                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 222                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 223         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
 224                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 225                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 226         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
 227                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 228                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 229 };
 230
 231 /*
 232  * Mode Control Instructions for various Key lengths 128, 192, 256
 233  * For CBC (Cipher Block Chaining) mode for decryption
 234  */
 235 static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
 236         {       0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 237                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 238                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 239         {       0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 240                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 241                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 242         {       0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 243                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 244                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 245 };
 246
 247 /*
 248  * Mode Control Instructions for various Key lengths 128, 192, 256
 249  * For CBC (Cipher Block Chaining) mode for encryption
 250  */
 251 static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = {
 252         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
 253                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 254                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 255         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
 256                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 257                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 258         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
 259                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 260                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 261 };
 262
 263 /*
 264  * Mode Control Instructions for various Key lengths 128, 192, 256
 265  * For CBC (Cipher Block Chaining) mode for decryption
 266  */
 267 static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = {
 268         {       0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 269                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 270                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 271         {       0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 272                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 273                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 274         {       0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 275                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 276                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 277 };
 278
 279 /*
 280  * Mode Control Instructions for various Key lengths 128, 192, 256
 281  * For ECB (Electronic Code Book) mode for encryption
 282  */
 283 static u8 mci_ecb_enc_array[3][27] = {
 284         {       0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 285                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 286                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 287         {       0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 288                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 289                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 290         {       0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 291                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 292                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 293 };
 294
 295 /*
 296  * Mode Control Instructions for various Key lengths 128, 192, 256
 297  * For ECB (Electronic Code Book) mode for decryption
 298  */
 299 static u8 mci_ecb_dec_array[3][27] = {
 300         {       0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 301                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 302                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 303         {       0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 304                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 305                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 306         {       0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 307                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 308                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 309 };
 310
 311 /*
 312  * Mode Control Instructions for DES algorithm
 313  * For CBC (Cipher Block Chaining) mode and ECB mode
 314  * encryption and for decryption respectively
 315  */
 316 static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
 317         0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
 318         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 319         0x00, 0x00, 0x00,
 320 };
 321
 322 static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
 323         0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
 324         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 325         0x00, 0x00, 0x00,
 326 };
 327
 328 static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
 329         0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
 330         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 331         0x00, 0x00, 0x00,
 332 };
 333
 334 static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
 335         0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
 336         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 337         0x00, 0x00, 0x00,
 338 };
 339
 340 /*
 341  * Perform 16 byte or 128 bit swizzling
 342  * The SA2UL Expects the security context to
 343  * be in little Endian and the bus width is 128 bits or 16 bytes
 344  * Hence swap 16 bytes at a time from higher to lower address
 345  */
 346 static void sa_swiz_128(u8 *in, u16 len)
 347 {
 348         u8 data[16];
 349         int i, j;
 350
 351         for (i = 0; i < len; i += 16) {
 352                 memcpy(data, &in[i], 16);
 353                 for (j = 0; j < 16; j++)
 354                         in[i + j] = data[15 - j];
 355         }
 356 }
 357
 358 /* Prepare the ipad and opad from key as per SHA algorithm step 1*/
 359 static void prepare_kiopad(u8 *k_ipad, u8 *k_opad, const u8 *key, u16 key_sz)
 360 {
 361         int i;
 362
 363         for (i = 0; i < key_sz; i++) {
 364                 k_ipad[i] = key[i] ^ 0x36;
 365                 k_opad[i] = key[i] ^ 0x5c;
 366         }
 367
 368         /* Instead of XOR with 0 */
 369         for (; i < SHA1_BLOCK_SIZE; i++) {
 370                 k_ipad[i] = 0x36;
 371                 k_opad[i] = 0x5c;
 372         }
 373 }
 374
 375 static void sa_export_shash(struct shash_desc *hash, int block_size,
 376                             int digest_size, __be32 *out)
 377 {
 378         union {
 379                 struct sha1_state sha1;
 380                 struct sha256_state sha256;
 381                 struct sha512_state sha512;
 382         } sha;
 383         void *state;
 384         u32 *result;
 385         int i;
 386
 387         switch (digest_size) {
 388         case SHA1_DIGEST_SIZE:
 389                 state = &sha.sha1;
 390                 result = sha.sha1.state;
 391                 break;
 392         case SHA256_DIGEST_SIZE:
 393                 state = &sha.sha256;
 394                 result = sha.sha256.state;
 395                 break;
 396         default:
 397                 dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__,
 398                         digest_size);
 399                 return;
 400         }
 401
 402         crypto_shash_export(hash, state);
 403
 404         for (i = 0; i < digest_size >> 2; i++)
 405                 out[i] = cpu_to_be32(result[i]);
 406 }
 407
 408 static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
 409                               u16 key_sz, __be32 *ipad, __be32 *opad)
 410 {
 411         SHASH_DESC_ON_STACK(shash, data->ctx->shash);
 412         int block_size = crypto_shash_blocksize(data->ctx->shash);
 413         int digest_size = crypto_shash_digestsize(data->ctx->shash);
 414         u8 k_ipad[SHA1_BLOCK_SIZE];
 415         u8 k_opad[SHA1_BLOCK_SIZE];
 416
 417         shash->tfm = data->ctx->shash;
 418
 419         prepare_kiopad(k_ipad, k_opad, key, key_sz);
 420
 421         memzero_explicit(ipad, block_size);
 422         memzero_explicit(opad, block_size);
 423
 424         crypto_shash_init(shash);
 425         crypto_shash_update(shash, k_ipad, block_size);
 426         sa_export_shash(shash, block_size, digest_size, ipad);
 427
 428         crypto_shash_init(shash);
 429         crypto_shash_update(shash, k_opad, block_size);
 430
 431         sa_export_shash(shash, block_size, digest_size, opad);
 432 }
 433
 434 /* Derive the inverse key used in AES-CBC decryption operation */
 435 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
 436 {
 437         struct crypto_aes_ctx ctx;
 438         int key_pos;
 439
 440         if (aes_expandkey(&ctx, key, key_sz)) {
 441                 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
 442                 return -EINVAL;
 443         }
 444
 445         /* work around to get the right inverse for AES_KEYSIZE_192 size keys */
 446         if (key_sz == AES_KEYSIZE_192) {
 447                 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46];
 448                 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47];
 449         }
 450
 451         /* Based crypto_aes_expand_key logic */
 452         switch (key_sz) {
 453         case AES_KEYSIZE_128:
 454         case AES_KEYSIZE_192:
 455                 key_pos = key_sz + 24;
 456                 break;
 457
 458         case AES_KEYSIZE_256:
 459                 key_pos = key_sz + 24 - 4;
 460                 break;
 461
 462         default:
 463                 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
 464                 return -EINVAL;
 465         }
 466
 467         memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
 468         return 0;
 469 }
 470
 471 /* Set Security context for the encryption engine */
 472 static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
 473                          u8 enc, u8 *sc_buf)
 474 {
 475         const u8 *mci = NULL;
 476
 477         /* Set Encryption mode selector to crypto processing */
 478         sc_buf[0] = SA_CRYPTO_PROCESSING;
 479
 480         if (enc)
 481                 mci = ad->mci_enc;
 482         else
 483                 mci = ad->mci_dec;
 484         /* Set the mode control instructions in security context */
 485         if (mci)
 486                 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
 487
 488         /* For AES-CBC decryption get the inverse key */
 489         if (ad->inv_key && !enc) {
 490                 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
 491                         return -EINVAL;
 492         /* For all other cases: key is used */
 493         } else {
 494                 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
 495         }
 496
 497         return 0;
 498 }
 499
 500 /* Set Security context for the authentication engine */
 501 static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
 502                            u8 *sc_buf)
 503 {
 504         __be32 ipad[64], opad[64];
 505
 506         /* Set Authentication mode selector to hash processing */
 507         sc_buf[0] = SA_HASH_PROCESSING;
 508         /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
 509         sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
 510         sc_buf[1] |= ad->auth_ctrl;
 511
 512         /* Copy the keys or ipad/opad */
 513         if (ad->keyed_mac) {
 514                 ad->prep_iopad(ad, key, key_sz, ipad, opad);
 515
 516                 /* Copy ipad to AuthKey */
 517                 memcpy(&sc_buf[32], ipad, ad->hash_size);
 518                 /* Copy opad to Aux-1 */
 519                 memcpy(&sc_buf[64], opad, ad->hash_size);
 520         } else {
 521                 /* basic hash */
 522                 sc_buf[1] |= SA_BASIC_HASH;
 523         }
 524 }
 525
 526 static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16)
 527 {
 528         int j;
 529
 530         for (j = 0; j < ((size16) ? 4 : 2); j++) {
 531                 *out = cpu_to_be32(*((u32 *)iv));
 532                 iv += 4;
 533                 out++;
 534         }
 535 }
 536
 537 /* Format general command label */
 538 static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
 539                               struct sa_cmdl_upd_info *upd_info)
 540 {
 541         u8 enc_offset = 0, auth_offset = 0, total = 0;
 542         u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
 543         u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
 544         u32 *word_ptr = (u32 *)cmdl;
 545         int i;
 546
 547         /* Clear the command label */
 548         memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
 549
 550         /* Iniialize the command update structure */
 551         memzero_explicit(upd_info, sizeof(*upd_info));
 552
 553         if (cfg->enc_eng_id && cfg->auth_eng_id) {
 554                 if (cfg->enc) {
 555                         auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
 556                         enc_next_eng = cfg->auth_eng_id;
 557
 558                         if (cfg->iv_size)
 559                                 auth_offset += cfg->iv_size;
 560                 } else {
 561                         enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
 562                         auth_next_eng = cfg->enc_eng_id;
 563                 }
 564         }
 565
 566         if (cfg->enc_eng_id) {
 567                 upd_info->flags |= SA_CMDL_UPD_ENC;
 568                 upd_info->enc_size.index = enc_offset >> 2;
 569                 upd_info->enc_offset.index = upd_info->enc_size.index + 1;
 570                 /* Encryption command label */
 571                 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
 572
 573                 /* Encryption modes requiring IV */
 574                 if (cfg->iv_size) {
 575                         upd_info->flags |= SA_CMDL_UPD_ENC_IV;
 576                         upd_info->enc_iv.index =
 577                                 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
 578                         upd_info->enc_iv.size = cfg->iv_size;
 579
 580                         cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 581                                 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
 582
 583                         cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
 584                                 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
 585                         total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
 586                 } else {
 587                         cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 588                                                 SA_CMDL_HEADER_SIZE_BYTES;
 589                         total += SA_CMDL_HEADER_SIZE_BYTES;
 590                 }
 591         }
 592
 593         if (cfg->auth_eng_id) {
 594                 upd_info->flags |= SA_CMDL_UPD_AUTH;
 595                 upd_info->auth_size.index = auth_offset >> 2;
 596                 upd_info->auth_offset.index = upd_info->auth_size.index + 1;
 597                 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
 598                 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 599                         SA_CMDL_HEADER_SIZE_BYTES;
 600                 total += SA_CMDL_HEADER_SIZE_BYTES;
 601         }
 602
 603         total = roundup(total, 8);
 604
 605         for (i = 0; i < total / 4; i++)
 606                 word_ptr[i] = swab32(word_ptr[i]);
 607
 608         return total;
 609 }
 610
 611 /* Update Command label */
 612 static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
 613                                   struct sa_cmdl_upd_info *upd_info)
 614 {
 615         int i = 0, j;
 616
 617         if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
 618                 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
 619                 cmdl[upd_info->enc_size.index] |= req->enc_size;
 620                 cmdl[upd_info->enc_offset.index] &=
 621                                                 ~SA_CMDL_SOP_BYPASS_LEN_MASK;
 622                 cmdl[upd_info->enc_offset.index] |=
 623                         ((u32)req->enc_offset <<
 624                          __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
 625
 626                 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
 627                         __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
 628                         u32 *enc_iv = (u32 *)req->enc_iv;
 629
 630                         for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
 631                                 data[j] = cpu_to_be32(*enc_iv);
 632                                 enc_iv++;
 633                         }
 634                 }
 635         }
 636
 637         if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
 638                 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
 639                 cmdl[upd_info->auth_size.index] |= req->auth_size;
 640                 cmdl[upd_info->auth_offset.index] &=
 641                         ~SA_CMDL_SOP_BYPASS_LEN_MASK;
 642                 cmdl[upd_info->auth_offset.index] |=
 643                         ((u32)req->auth_offset <<
 644                          __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
 645                 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
 646                         sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
 647                                    req->auth_iv,
 648                                    (upd_info->auth_iv.size > 8));
 649                 }
 650                 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
 651                         int offset = (req->auth_size & 0xF) ? 4 : 0;
 652
 653                         memcpy(&cmdl[upd_info->aux_key_info.index],
 654                                &upd_info->aux_key[offset], 16);
 655                 }
 656         }
 657 }
 658
 659 /* Format SWINFO words to be sent to SA */
 660 static
 661 void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
 662                    u8 cmdl_present, u8 cmdl_offset, u8 flags,
 663                    u8 hash_size, u32 *swinfo)
 664 {
 665         swinfo[0] = sc_id;
 666         swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK));
 667         if (likely(cmdl_present))
 668                 swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) <<
 669                                                 __ffs(SA_SW0_CMDL_INFO_MASK));
 670         swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK));
 671
 672         swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
 673         swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
 674         swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
 675         swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH));
 676 }
 677
 678 /* Dump the security context */
 679 static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
 680 {
 681 #ifdef DEBUG
 682         dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
 683         print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
 684                        16, 1, buf, SA_CTX_MAX_SZ, false);
 685 #endif
 686 }
 687
 688 static
 689 int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key,
 690                u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz,
 691                struct algo_data *ad, u8 enc, u32 *swinfo)
 692 {
 693         int enc_sc_offset = 0;
 694         int auth_sc_offset = 0;
 695         u8 *sc_buf = ctx->sc;
 696         u16 sc_id = ctx->sc_id;
 697         u8 first_engine = 0;
 698
 699         memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
 700
 701         if (ad->auth_eng.eng_id) {
 702                 if (enc)
 703                         first_engine = ad->enc_eng.eng_id;
 704                 else
 705                         first_engine = ad->auth_eng.eng_id;
 706
 707                 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
 708                 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
 709                 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
 710                 if (!ad->hash_size)
 711                         return -EINVAL;
 712                 ad->hash_size = roundup(ad->hash_size, 8);
 713
 714         } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) {
 715                 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
 716                 first_engine = ad->enc_eng.eng_id;
 717                 sc_buf[1] = SA_SCCTL_FE_ENC;
 718                 ad->hash_size = ad->iv_out_size;
 719         }
 720
 721         /* SCCTL Owner info: 0=host, 1=CP_ACE */
 722         sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
 723         memcpy(&sc_buf[2], &sc_id, 2);
 724         sc_buf[4] = 0x0;
 725         sc_buf[5] = PRIV_ID;
 726         sc_buf[6] = PRIV;
 727         sc_buf[7] = 0x0;
 728
 729         /* Prepare context for encryption engine */
 730         if (ad->enc_eng.sc_size) {
 731                 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc,
 732                                   &sc_buf[enc_sc_offset]))
 733                         return -EINVAL;
 734         }
 735
 736         /* Prepare context for authentication engine */
 737         if (ad->auth_eng.sc_size)
 738                 sa_set_sc_auth(ad, auth_key, auth_key_sz,
 739                                &sc_buf[auth_sc_offset]);
 740
 741         /* Set the ownership of context to CP_ACE */
 742         sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
 743
 744         /* swizzle the security context */
 745         sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
 746
 747         sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
 748                       SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
 749
 750         sa_dump_sc(sc_buf, ctx->sc_phys);
 751
 752         return 0;
 753 }
 754
 755 /* Free the per direction context memory */
 756 static void sa_free_ctx_info(struct sa_ctx_info *ctx,
 757                              struct sa_crypto_data *data)
 758 {
 759         unsigned long bn;
 760
 761         bn = ctx->sc_id - data->sc_id_start;
 762         spin_lock(&data->scid_lock);
 763         __clear_bit(bn, data->ctx_bm);
 764         data->sc_id--;
 765         spin_unlock(&data->scid_lock);
 766
 767         if (ctx->sc) {
 768                 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
 769                 ctx->sc = NULL;
 770         }
 771 }
 772
 773 static int sa_init_ctx_info(struct sa_ctx_info *ctx,
 774                             struct sa_crypto_data *data)
 775 {
 776         unsigned long bn;
 777         int err;
 778
 779         spin_lock(&data->scid_lock);
 780         bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
 781         __set_bit(bn, data->ctx_bm);
 782         data->sc_id++;
 783         spin_unlock(&data->scid_lock);
 784
 785         ctx->sc_id = (u16)(data->sc_id_start + bn);
 786
 787         ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
 788         if (!ctx->sc) {
 789                 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
 790                 err = -ENOMEM;
 791                 goto scid_rollback;
 792         }
 793
 794         return 0;
 795
 796 scid_rollback:
 797         spin_lock(&data->scid_lock);
 798         __clear_bit(bn, data->ctx_bm);
 799         data->sc_id--;
 800         spin_unlock(&data->scid_lock);
 801
 802         return err;
 803 }
 804
 805 static void sa_cipher_cra_exit(struct crypto_skcipher *tfm)
 806 {
 807         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 808         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
 809
 810         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
 811                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
 812                 ctx->dec.sc_id, &ctx->dec.sc_phys);
 813
 814         sa_free_ctx_info(&ctx->enc, data);
 815         sa_free_ctx_info(&ctx->dec, data);
 816
 817         crypto_free_sync_skcipher(ctx->fallback.skcipher);
 818 }
 819
 820 static int sa_cipher_cra_init(struct crypto_skcipher *tfm)
 821 {
 822         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 823         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
 824         const char *name = crypto_tfm_alg_name(&tfm->base);
 825         int ret;
 826
 827         memzero_explicit(ctx, sizeof(*ctx));
 828         ctx->dev_data = data;
 829
 830         ret = sa_init_ctx_info(&ctx->enc, data);
 831         if (ret)
 832                 return ret;
 833         ret = sa_init_ctx_info(&ctx->dec, data);
 834         if (ret) {
 835                 sa_free_ctx_info(&ctx->enc, data);
 836                 return ret;
 837         }
 838
 839         ctx->fallback.skcipher =
 840                 crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
 841
 842         if (IS_ERR(ctx->fallback.skcipher)) {
 843                 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name);
 844                 return PTR_ERR(ctx->fallback.skcipher);
 845         }
 846
 847         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
 848                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
 849                 ctx->dec.sc_id, &ctx->dec.sc_phys);
 850         return 0;
 851 }
 852
 853 static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 854                             unsigned int keylen, struct algo_data *ad)
 855 {
 856         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 857         int cmdl_len;
 858         struct sa_cmdl_cfg cfg;
 859         int ret;
 860
 861         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 862             keylen != AES_KEYSIZE_256)
 863                 return -EINVAL;
 864
 865         ad->enc_eng.eng_id = SA_ENG_ID_EM1;
 866         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
 867
 868         memzero_explicit(&cfg, sizeof(cfg));
 869         cfg.enc_eng_id = ad->enc_eng.eng_id;
 870         cfg.iv_size = crypto_skcipher_ivsize(tfm);
 871
 872         crypto_sync_skcipher_clear_flags(ctx->fallback.skcipher,
 873                                          CRYPTO_TFM_REQ_MASK);
 874         crypto_sync_skcipher_set_flags(ctx->fallback.skcipher,
 875                                        tfm->base.crt_flags &
 876                                        CRYPTO_TFM_REQ_MASK);
 877         ret = crypto_sync_skcipher_setkey(ctx->fallback.skcipher, key, keylen);
 878         if (ret)
 879                 return ret;
 880
 881         /* Setup Encryption Security Context & Command label template */
 882         if (sa_init_sc(&ctx->enc, key, keylen, NULL, 0, ad, 1,
 883                        &ctx->enc.epib[1]))
 884                 goto badkey;
 885
 886         cmdl_len = sa_format_cmdl_gen(&cfg,
 887                                       (u8 *)ctx->enc.cmdl,
 888                                       &ctx->enc.cmdl_upd_info);
 889         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
 890                 goto badkey;
 891
 892         ctx->enc.cmdl_size = cmdl_len;
 893
 894         /* Setup Decryption Security Context & Command label template */
 895         if (sa_init_sc(&ctx->dec, key, keylen, NULL, 0, ad, 0,
 896                        &ctx->dec.epib[1]))
 897                 goto badkey;
 898
 899         cfg.enc_eng_id = ad->enc_eng.eng_id;
 900         cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
 901                                       &ctx->dec.cmdl_upd_info);
 902
 903         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
 904                 goto badkey;
 905
 906         ctx->dec.cmdl_size = cmdl_len;
 907         ctx->iv_idx = ad->iv_idx;
 908
 909         return 0;
 910
 911 badkey:
 912         dev_err(sa_k3_dev, "%s: badkey\n", __func__);
 913         return -EINVAL;
 914 }
 915
 916 static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
 917                              unsigned int keylen)
 918 {
 919         struct algo_data ad = { 0 };
 920         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
 921         int key_idx = (keylen >> 3) - 2;
 922
 923         if (key_idx >= 3)
 924                 return -EINVAL;
 925
 926         ad.mci_enc = mci_cbc_enc_array[key_idx];
 927         ad.mci_dec = mci_cbc_dec_array[key_idx];
 928         ad.inv_key = true;
 929         ad.ealg_id = SA_EALG_ID_AES_CBC;
 930         ad.iv_idx = 4;
 931         ad.iv_out_size = 16;
 932
 933         return sa_cipher_setkey(tfm, key, keylen, &ad);
 934 }
 935
 936 static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
 937                              unsigned int keylen)
 938 {
 939         struct algo_data ad = { 0 };
 940         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
 941         int key_idx = (keylen >> 3) - 2;
 942
 943         if (key_idx >= 3)
 944                 return -EINVAL;
 945
 946         ad.mci_enc = mci_ecb_enc_array[key_idx];
 947         ad.mci_dec = mci_ecb_dec_array[key_idx];
 948         ad.inv_key = true;
 949         ad.ealg_id = SA_EALG_ID_AES_ECB;
 950
 951         return sa_cipher_setkey(tfm, key, keylen, &ad);
 952 }
 953
 954 static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
 955                               unsigned int keylen)
 956 {
 957         struct algo_data ad = { 0 };
 958
 959         ad.mci_enc = mci_cbc_3des_enc_array;
 960         ad.mci_dec = mci_cbc_3des_dec_array;
 961         ad.ealg_id = SA_EALG_ID_3DES_CBC;
 962         ad.iv_idx = 6;
 963         ad.iv_out_size = 8;
 964
 965         return sa_cipher_setkey(tfm, key, keylen, &ad);
 966 }
 967
 968 static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
 969                               unsigned int keylen)
 970 {
 971         struct algo_data ad = { 0 };
 972
 973         ad.mci_enc = mci_ecb_3des_enc_array;
 974         ad.mci_dec = mci_ecb_3des_dec_array;
 975
 976         return sa_cipher_setkey(tfm, key, keylen, &ad);
 977 }
 978
 979 static void sa_aes_dma_in_callback(void *data)
 980 {
 981         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
 982         struct skcipher_request *req;
 983         int sglen;
 984         u32 *result;
 985         __be32 *mdptr;
 986         size_t ml, pl;
 987         int i;
 988         enum dma_data_direction dir_src;
 989         bool diff_dst;
 990
 991         req = container_of(rxd->req, struct skcipher_request, base);
 992         sglen = sg_nents_for_len(req->src, req->cryptlen);
 993
 994         diff_dst = (req->src != req->dst) ? true : false;
 995         dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
 996
 997         if (req->iv) {
 998                 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
 999                                                                &ml);
1000                 result = (u32 *)req->iv;
1001
1002                 for (i = 0; i < (rxd->enc_iv_size / 4); i++)
1003                         result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
1004         }
1005
1006         dma_unmap_sg(rxd->ddev, req->src, sglen, dir_src);
1007         kfree(rxd->split_src_sg);
1008
1009         if (diff_dst) {
1010                 sglen = sg_nents_for_len(req->dst, req->cryptlen);
1011
1012                 dma_unmap_sg(rxd->ddev, req->dst, sglen,
1013                              DMA_FROM_DEVICE);
1014                 kfree(rxd->split_dst_sg);
1015         }
1016
1017         kfree(rxd);
1018
1019         skcipher_request_complete(req, 0);
1020 }
1021
1022 static void
1023 sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
1024 {
1025         u32 *out, *in;
1026         int i;
1027
1028         for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
1029                 *out++ = *in++;
1030
1031         mdptr[4] = (0xFFFF << 16);
1032         for (out = &mdptr[5], in = psdata, i = 0;
1033              i < pslen / sizeof(u32); i++)
1034                 *out++ = *in++;
1035 }
1036
1037 static int sa_run(struct sa_req *req)
1038 {
1039         struct sa_rx_data *rxd;
1040         gfp_t gfp_flags;
1041         u32 cmdl[SA_MAX_CMDL_WORDS];
1042         struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
1043         struct device *ddev;
1044         struct dma_chan *dma_rx;
1045         int sg_nents, src_nents, dst_nents;
1046         int mapped_src_nents, mapped_dst_nents;
1047         struct scatterlist *src, *dst;
1048         size_t pl, ml, split_size;
1049         struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
1050         int ret;
1051         struct dma_async_tx_descriptor *tx_out;
1052         u32 *mdptr;
1053         bool diff_dst;
1054         enum dma_data_direction dir_src;
1055
1056         gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1057                 GFP_KERNEL : GFP_ATOMIC;
1058
1059         rxd = kzalloc(sizeof(*rxd), gfp_flags);
1060         if (!rxd)
1061                 return -ENOMEM;
1062
1063         if (req->src != req->dst) {
1064                 diff_dst = true;
1065                 dir_src = DMA_TO_DEVICE;
1066         } else {
1067                 diff_dst = false;
1068                 dir_src = DMA_BIDIRECTIONAL;
1069         }
1070
1071         /*
1072          * SA2UL has an interesting feature where the receive DMA channel
1073          * is selected based on the data passed to the engine. Within the
1074          * transition range, there is also a space where it is impossible
1075          * to determine where the data will end up, and this should be
1076          * avoided. This will be handled by the SW fallback mechanism by
1077          * the individual algorithm implementations.
1078          */
1079         if (req->size >= 256)
1080                 dma_rx = pdata->dma_rx2;
1081         else
1082                 dma_rx = pdata->dma_rx1;
1083
1084         ddev = dma_rx->device->dev;
1085
1086         memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
1087
1088         sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info);
1089
1090         if (req->type != CRYPTO_ALG_TYPE_AHASH) {
1091                 if (req->enc)
1092                         req->type |=
1093                                 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
1094                 else
1095                         req->type |=
1096                                 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
1097         }
1098
1099         cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type;
1100
1101         /*
1102          * Map the packets, first we check if the data fits into a single
1103          * sg entry and use that if possible. If it does not fit, we check
1104          * if we need to do sg_split to align the scatterlist data on the
1105          * actual data size being processed by the crypto engine.
1106          */
1107         src = req->src;
1108         sg_nents = sg_nents_for_len(src, req->size);
1109
1110         split_size = req->size;
1111
1112         if (sg_nents == 1 && split_size <= req->src->length) {
1113                 src = &rxd->rx_sg;
1114                 sg_init_table(src, 1);
1115                 sg_set_page(src, sg_page(req->src), split_size,
1116                             req->src->offset);
1117                 src_nents = 1;
1118                 dma_map_sg(ddev, src, sg_nents, dir_src);
1119         } else {
1120                 mapped_src_nents = dma_map_sg(ddev, req->src, sg_nents,
1121                                               dir_src);
1122                 ret = sg_split(req->src, mapped_src_nents, 0, 1, &split_size,
1123                                &src, &src_nents, gfp_flags);
1124                 if (ret) {
1125                         src_nents = sg_nents;
1126                         src = req->src;
1127                 } else {
1128                         rxd->split_src_sg = src;
1129                 }
1130         }
1131
1132         if (!diff_dst) {
1133                 dst_nents = src_nents;
1134                 dst = src;
1135         } else {
1136                 dst_nents = sg_nents_for_len(req->dst, req->size);
1137
1138                 if (dst_nents == 1 && split_size <= req->dst->length) {
1139                         dst = &rxd->tx_sg;
1140                         sg_init_table(dst, 1);
1141                         sg_set_page(dst, sg_page(req->dst), split_size,
1142                                     req->dst->offset);
1143                         dst_nents = 1;
1144                         dma_map_sg(ddev, dst, dst_nents, DMA_FROM_DEVICE);
1145                 } else {
1146                         mapped_dst_nents = dma_map_sg(ddev, req->dst, dst_nents,
1147                                                       DMA_FROM_DEVICE);
1148                         ret = sg_split(req->dst, mapped_dst_nents, 0, 1,
1149                                        &split_size, &dst, &dst_nents,
1150                                        gfp_flags);
1151                         if (ret) {
1152                                 dst_nents = dst_nents;
1153                                 dst = req->dst;
1154                         } else {
1155                                 rxd->split_dst_sg = dst;
1156                         }
1157                 }
1158         }
1159
1160         if (unlikely(src_nents != sg_nents)) {
1161                 dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n");
1162                 ret = -EIO;
1163                 goto err_cleanup;
1164         }
1165
1166         rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
1167                                              DMA_DEV_TO_MEM,
1168                                              DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1169         if (!rxd->tx_in) {
1170                 dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
1171                 ret = -EINVAL;
1172                 goto err_cleanup;
1173         }
1174
1175         rxd->req = (void *)req->base;
1176         rxd->enc = req->enc;
1177         rxd->ddev = ddev;
1178         rxd->src = src;
1179         rxd->dst = dst;
1180         rxd->iv_idx = req->ctx->iv_idx;
1181         rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
1182         rxd->tx_in->callback = req->callback;
1183         rxd->tx_in->callback_param = rxd;
1184
1185         tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src,
1186                                          src_nents, DMA_MEM_TO_DEV,
1187                                          DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1188
1189         if (!tx_out) {
1190                 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
1191                 ret = -EINVAL;
1192                 goto err_cleanup;
1193         }
1194
1195         /*
1196          * Prepare metadata for DMA engine. This essentially describes the
1197          * crypto algorithm to be used, data sizes, different keys etc.
1198          */
1199         mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
1200
1201         sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
1202                                    sizeof(u32))), cmdl, sizeof(sa_ctx->epib),
1203                            sa_ctx->epib);
1204
1205         ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
1206         dmaengine_desc_set_metadata_len(tx_out, req->mdata_size);
1207
1208         dmaengine_submit(tx_out);
1209         dmaengine_submit(rxd->tx_in);
1210
1211         dma_async_issue_pending(dma_rx);
1212         dma_async_issue_pending(pdata->dma_tx);
1213
1214         return -EINPROGRESS;
1215
1216 err_cleanup:
1217         dma_unmap_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE);
1218         kfree(rxd->split_src_sg);
1219
1220         if (req->src != req->dst) {
1221                 dst_nents = sg_nents_for_len(req->dst, req->size);
1222                 dma_unmap_sg(ddev, req->dst, dst_nents, DMA_FROM_DEVICE);
1223                 kfree(rxd->split_dst_sg);
1224         }
1225
1226         kfree(rxd);
1227
1228         return ret;
1229 }
1230
1231 static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
1232 {
1233         struct sa_tfm_ctx *ctx =
1234             crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1235         struct crypto_alg *alg = req->base.tfm->__crt_alg;
1236         struct sa_req sa_req = { 0 };
1237         int ret;
1238
1239         if (!req->cryptlen)
1240                 return 0;
1241
1242         if (req->cryptlen % alg->cra_blocksize)
1243                 return -EINVAL;
1244
1245         /* Use SW fallback if the data size is not supported */
1246         if (req->cryptlen > SA_MAX_DATA_SZ ||
1247             (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN &&
1248              req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) {
1249                 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback.skcipher);
1250
1251                 skcipher_request_set_sync_tfm(subreq, ctx->fallback.skcipher);
1252                 skcipher_request_set_callback(subreq, req->base.flags,
1253                                               NULL, NULL);
1254                 skcipher_request_set_crypt(subreq, req->src, req->dst,
1255                                            req->cryptlen, req->iv);
1256                 if (enc)
1257                         ret = crypto_skcipher_encrypt(subreq);
1258                 else
1259                         ret = crypto_skcipher_decrypt(subreq);
1260
1261                 skcipher_request_zero(subreq);
1262                 return ret;
1263         }
1264
1265         sa_req.size = req->cryptlen;
1266         sa_req.enc_size = req->cryptlen;
1267         sa_req.src = req->src;
1268         sa_req.dst = req->dst;
1269         sa_req.enc_iv = iv;
1270         sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
1271         sa_req.enc = enc;
1272         sa_req.callback = sa_aes_dma_in_callback;
1273         sa_req.mdata_size = 44;
1274         sa_req.base = &req->base;
1275         sa_req.ctx = ctx;
1276
1277         return sa_run(&sa_req);
1278 }
1279
1280 static int sa_encrypt(struct skcipher_request *req)
1281 {
1282         return sa_cipher_run(req, req->iv, 1);
1283 }
1284
1285 static int sa_decrypt(struct skcipher_request *req)
1286 {
1287         return sa_cipher_run(req, req->iv, 0);
1288 }
1289
1290 static void sa_sha_dma_in_callback(void *data)
1291 {
1292         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1293         struct ahash_request *req;
1294         struct crypto_ahash *tfm;
1295         unsigned int authsize;
1296         int i, sg_nents;
1297         size_t ml, pl;
1298         u32 *result;
1299         __be32 *mdptr;
1300
1301         req = container_of(rxd->req, struct ahash_request, base);
1302         tfm = crypto_ahash_reqtfm(req);
1303         authsize = crypto_ahash_digestsize(tfm);
1304
1305         mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1306         result = (u32 *)req->result;
1307
1308         for (i = 0; i < (authsize / 4); i++)
1309                 result[i] = be32_to_cpu(mdptr[i + 4]);
1310
1311         sg_nents = sg_nents_for_len(req->src, req->nbytes);
1312         dma_unmap_sg(rxd->ddev, req->src, sg_nents, DMA_FROM_DEVICE);
1313
1314         kfree(rxd->split_src_sg);
1315
1316         kfree(rxd);
1317
1318         ahash_request_complete(req, 0);
1319 }
1320
1321 static int zero_message_process(struct ahash_request *req)
1322 {
1323         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1324         int sa_digest_size = crypto_ahash_digestsize(tfm);
1325
1326         switch (sa_digest_size) {
1327         case SHA1_DIGEST_SIZE:
1328                 memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
1329                 break;
1330         case SHA256_DIGEST_SIZE:
1331                 memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
1332                 break;
1333         case SHA512_DIGEST_SIZE:
1334                 memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
1335                 break;
1336         default:
1337                 return -EINVAL;
1338         }
1339
1340         return 0;
1341 }
1342
1343 static int sa_sha_run(struct ahash_request *req)
1344 {
1345         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1346         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1347         struct sa_req sa_req = { 0 };
1348         size_t auth_len;
1349
1350         auth_len = req->nbytes;
1351
1352         if (!auth_len)
1353                 return zero_message_process(req);
1354
1355         if (auth_len > SA_MAX_DATA_SZ ||
1356             (auth_len >= SA_UNSAFE_DATA_SZ_MIN &&
1357              auth_len <= SA_UNSAFE_DATA_SZ_MAX)) {
1358                 struct ahash_request *subreq = &rctx->fallback_req;
1359                 int ret = 0;
1360
1361                 ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1362                 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1363
1364                 crypto_ahash_init(subreq);
1365
1366                 subreq->nbytes = auth_len;
1367                 subreq->src = req->src;
1368                 subreq->result = req->result;
1369
1370                 ret |= crypto_ahash_update(subreq);
1371
1372                 subreq->nbytes = 0;
1373
1374                 ret |= crypto_ahash_final(subreq);
1375
1376                 return ret;
1377         }
1378
1379         sa_req.size = auth_len;
1380         sa_req.auth_size = auth_len;
1381         sa_req.src = req->src;
1382         sa_req.dst = req->src;
1383         sa_req.enc = true;
1384         sa_req.type = CRYPTO_ALG_TYPE_AHASH;
1385         sa_req.callback = sa_sha_dma_in_callback;
1386         sa_req.mdata_size = 28;
1387         sa_req.ctx = ctx;
1388         sa_req.base = &req->base;
1389
1390         return sa_run(&sa_req);
1391 }
1392
1393 static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct  algo_data *ad)
1394 {
1395         int bs = crypto_shash_blocksize(ctx->shash);
1396         int cmdl_len;
1397         struct sa_cmdl_cfg cfg;
1398
1399         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1400         ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1401         ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1402
1403         memset(ctx->authkey, 0, bs);
1404         memset(&cfg, 0, sizeof(cfg));
1405         cfg.aalg = ad->aalg_id;
1406         cfg.enc_eng_id = ad->enc_eng.eng_id;
1407         cfg.auth_eng_id = ad->auth_eng.eng_id;
1408         cfg.iv_size = 0;
1409         cfg.akey = NULL;
1410         cfg.akey_len = 0;
1411
1412         /* Setup Encryption Security Context & Command label template */
1413         if (sa_init_sc(&ctx->enc, NULL, 0, NULL, 0, ad, 0,
1414                        &ctx->enc.epib[1]))
1415                 goto badkey;
1416
1417         cmdl_len = sa_format_cmdl_gen(&cfg,
1418                                       (u8 *)ctx->enc.cmdl,
1419                                       &ctx->enc.cmdl_upd_info);
1420         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1421                 goto badkey;
1422
1423         ctx->enc.cmdl_size = cmdl_len;
1424
1425         return 0;
1426
1427 badkey:
1428         dev_err(sa_k3_dev, "%s: badkey\n", __func__);
1429         return -EINVAL;
1430 }
1431
1432 static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
1433 {
1434         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1435         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1436         int ret;
1437
1438         memset(ctx, 0, sizeof(*ctx));
1439         ctx->dev_data = data;
1440         ret = sa_init_ctx_info(&ctx->enc, data);
1441         if (ret)
1442                 return ret;
1443
1444         if (alg_base) {
1445                 ctx->shash = crypto_alloc_shash(alg_base, 0,
1446                                                 CRYPTO_ALG_NEED_FALLBACK);
1447                 if (IS_ERR(ctx->shash)) {
1448                         dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n",
1449                                 alg_base);
1450                         return PTR_ERR(ctx->shash);
1451                 }
1452                 /* for fallback */
1453                 ctx->fallback.ahash =
1454                         crypto_alloc_ahash(alg_base, 0,
1455                                            CRYPTO_ALG_NEED_FALLBACK);
1456                 if (IS_ERR(ctx->fallback.ahash)) {
1457                         dev_err(ctx->dev_data->dev,
1458                                 "Could not load fallback driver\n");
1459                         return PTR_ERR(ctx->fallback.ahash);
1460                 }
1461         }
1462
1463         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1464                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1465                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1466
1467         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1468                                  sizeof(struct sa_sha_req_ctx) +
1469                                  crypto_ahash_reqsize(ctx->fallback.ahash));
1470
1471         return 0;
1472 }
1473
1474 static int sa_sha_digest(struct ahash_request *req)
1475 {
1476         return sa_sha_run(req);
1477 }
1478
1479 static int sa_sha_init(struct ahash_request *req)
1480 {
1481         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1482         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1483         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1484
1485         dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
1486                 crypto_ahash_digestsize(tfm), rctx);
1487
1488         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1489         rctx->fallback_req.base.flags =
1490                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1491
1492         return crypto_ahash_init(&rctx->fallback_req);
1493 }
1494
1495 static int sa_sha_update(struct ahash_request *req)
1496 {
1497         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1498         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1499         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1500
1501         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1502         rctx->fallback_req.base.flags =
1503                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1504         rctx->fallback_req.nbytes = req->nbytes;
1505         rctx->fallback_req.src = req->src;
1506
1507         return crypto_ahash_update(&rctx->fallback_req);
1508 }
1509
1510 static int sa_sha_final(struct ahash_request *req)
1511 {
1512         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1513         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1514         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1515
1516         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1517         rctx->fallback_req.base.flags =
1518                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1519         rctx->fallback_req.result = req->result;
1520
1521         return crypto_ahash_final(&rctx->fallback_req);
1522 }
1523
1524 static int sa_sha_finup(struct ahash_request *req)
1525 {
1526         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1527         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1528         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1529
1530         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1531         rctx->fallback_req.base.flags =
1532                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1533
1534         rctx->fallback_req.nbytes = req->nbytes;
1535         rctx->fallback_req.src = req->src;
1536         rctx->fallback_req.result = req->result;
1537
1538         return crypto_ahash_finup(&rctx->fallback_req);
1539 }
1540
1541 static int sa_sha_import(struct ahash_request *req, const void *in)
1542 {
1543         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1544         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1545         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1546
1547         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1548         rctx->fallback_req.base.flags = req->base.flags &
1549                 CRYPTO_TFM_REQ_MAY_SLEEP;
1550
1551         return crypto_ahash_import(&rctx->fallback_req, in);
1552 }
1553
1554 static int sa_sha_export(struct ahash_request *req, void *out)
1555 {
1556         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1557         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1558         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1559         struct ahash_request *subreq = &rctx->fallback_req;
1560
1561         ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1562         subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1563
1564         return crypto_ahash_export(subreq, out);
1565 }
1566
1567 static int sa_sha1_cra_init(struct crypto_tfm *tfm)
1568 {
1569         struct algo_data ad = { 0 };
1570         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1571
1572         sa_sha_cra_init_alg(tfm, "sha1");
1573
1574         ad.aalg_id = SA_AALG_ID_SHA1;
1575         ad.hash_size = SHA1_DIGEST_SIZE;
1576         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1577
1578         sa_sha_setup(ctx, &ad);
1579
1580         return 0;
1581 }
1582
1583 static int sa_sha256_cra_init(struct crypto_tfm *tfm)
1584 {
1585         struct algo_data ad = { 0 };
1586         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1587
1588         sa_sha_cra_init_alg(tfm, "sha256");
1589
1590         ad.aalg_id = SA_AALG_ID_SHA2_256;
1591         ad.hash_size = SHA256_DIGEST_SIZE;
1592         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1593
1594         sa_sha_setup(ctx, &ad);
1595
1596         return 0;
1597 }
1598
1599 static int sa_sha512_cra_init(struct crypto_tfm *tfm)
1600 {
1601         struct algo_data ad = { 0 };
1602         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1603
1604         sa_sha_cra_init_alg(tfm, "sha512");
1605
1606         ad.aalg_id = SA_AALG_ID_SHA2_512;
1607         ad.hash_size = SHA512_DIGEST_SIZE;
1608         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512;
1609
1610         sa_sha_setup(ctx, &ad);
1611
1612         return 0;
1613 }
1614
1615 static void sa_sha_cra_exit(struct crypto_tfm *tfm)
1616 {
1617         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1618         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1619
1620         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1621                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1622                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1623
1624         if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH)
1625                 sa_free_ctx_info(&ctx->enc, data);
1626
1627         crypto_free_shash(ctx->shash);
1628         crypto_free_ahash(ctx->fallback.ahash);
1629 }
1630
1631 static void sa_aead_dma_in_callback(void *data)
1632 {
1633         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1634         struct aead_request *req;
1635         struct crypto_aead *tfm;
1636         unsigned int start;
1637         unsigned int authsize;
1638         u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
1639         size_t pl, ml;
1640         int i, sglen;
1641         int err = 0;
1642         u16 auth_len;
1643         u32 *mdptr;
1644         bool diff_dst;
1645         enum dma_data_direction dir_src;
1646
1647         req = container_of(rxd->req, struct aead_request, base);
1648         tfm = crypto_aead_reqtfm(req);
1649         start = req->assoclen + req->cryptlen;
1650         authsize = crypto_aead_authsize(tfm);
1651
1652         diff_dst = (req->src != req->dst) ? true : false;
1653         dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
1654
1655         mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1656         for (i = 0; i < (authsize / 4); i++)
1657                 mdptr[i + 4] = swab32(mdptr[i + 4]);
1658
1659         auth_len = req->assoclen + req->cryptlen;
1660         if (!rxd->enc)
1661                 auth_len -= authsize;
1662
1663         sglen =  sg_nents_for_len(rxd->src, auth_len);
1664         dma_unmap_sg(rxd->ddev, rxd->src, sglen, dir_src);
1665         kfree(rxd->split_src_sg);
1666
1667         if (diff_dst) {
1668                 sglen = sg_nents_for_len(rxd->dst, auth_len);
1669                 dma_unmap_sg(rxd->ddev, rxd->dst, sglen, DMA_FROM_DEVICE);
1670                 kfree(rxd->split_dst_sg);
1671         }
1672
1673         if (rxd->enc) {
1674                 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1675                                          1);
1676         } else {
1677                 start -= authsize;
1678                 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
1679                                          0);
1680
1681                 err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
1682         }
1683
1684         kfree(rxd);
1685
1686         aead_request_complete(req, err);
1687 }
1688
1689 static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash,
1690                             const char *fallback)
1691 {
1692         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1693         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1694         int ret;
1695
1696         memzero_explicit(ctx, sizeof(*ctx));
1697
1698         ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK);
1699         if (IS_ERR(ctx->shash)) {
1700                 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash);
1701                 return PTR_ERR(ctx->shash);
1702         }
1703
1704         ctx->fallback.aead = crypto_alloc_aead(fallback, 0,
1705                                                CRYPTO_ALG_NEED_FALLBACK);
1706
1707         if (IS_ERR(ctx->fallback.aead)) {
1708                 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n",
1709                         fallback);
1710                 return PTR_ERR(ctx->fallback.aead);
1711         }
1712
1713         crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
1714                                 crypto_aead_reqsize(ctx->fallback.aead));
1715
1716         ret = sa_init_ctx_info(&ctx->enc, data);
1717         if (ret)
1718                 return ret;
1719
1720         ret = sa_init_ctx_info(&ctx->dec, data);
1721         if (ret) {
1722                 sa_free_ctx_info(&ctx->enc, data);
1723                 return ret;
1724         }
1725
1726         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1727                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1728                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1729
1730         return ret;
1731 }
1732
1733 static int sa_cra_init_aead_sha1(struct crypto_aead *tfm)
1734 {
1735         return sa_cra_init_aead(tfm, "sha1",
1736                                 "authenc(hmac(sha1-ce),cbc(aes-ce))");
1737 }
1738
1739 static int sa_cra_init_aead_sha256(struct crypto_aead *tfm)
1740 {
1741         return sa_cra_init_aead(tfm, "sha256",
1742                                 "authenc(hmac(sha256-ce),cbc(aes-ce))");
1743 }
1744
1745 static void sa_exit_tfm_aead(struct crypto_aead *tfm)
1746 {
1747         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1748         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1749
1750         crypto_free_shash(ctx->shash);
1751         crypto_free_aead(ctx->fallback.aead);
1752
1753         sa_free_ctx_info(&ctx->enc, data);
1754         sa_free_ctx_info(&ctx->dec, data);
1755 }
1756
1757 /* AEAD algorithm configuration interface function */
1758 static int sa_aead_setkey(struct crypto_aead *authenc,
1759                           const u8 *key, unsigned int keylen,
1760                           struct algo_data *ad)
1761 {
1762         struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
1763         struct crypto_authenc_keys keys;
1764         int cmdl_len;
1765         struct sa_cmdl_cfg cfg;
1766         int key_idx;
1767
1768         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1769                 return -EINVAL;
1770
1771         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
1772         key_idx = (keys.enckeylen >> 3) - 2;
1773         if (key_idx >= 3)
1774                 return -EINVAL;
1775
1776         ad->ctx = ctx;
1777         ad->enc_eng.eng_id = SA_ENG_ID_EM1;
1778         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1779         ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1780         ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1781         ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx];
1782         ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx];
1783         ad->inv_key = true;
1784         ad->keyed_mac = true;
1785         ad->ealg_id = SA_EALG_ID_AES_CBC;
1786         ad->prep_iopad = sa_prepare_iopads;
1787
1788         memset(&cfg, 0, sizeof(cfg));
1789         cfg.enc = true;
1790         cfg.aalg = ad->aalg_id;
1791         cfg.enc_eng_id = ad->enc_eng.eng_id;
1792         cfg.auth_eng_id = ad->auth_eng.eng_id;
1793         cfg.iv_size = crypto_aead_ivsize(authenc);
1794         cfg.akey = keys.authkey;
1795         cfg.akey_len = keys.authkeylen;
1796
1797         /* Setup Encryption Security Context & Command label template */
1798         if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen,
1799                        keys.authkey, keys.authkeylen,
1800                        ad, 1, &ctx->enc.epib[1]))
1801                 return -EINVAL;
1802
1803         cmdl_len = sa_format_cmdl_gen(&cfg,
1804                                       (u8 *)ctx->enc.cmdl,
1805                                       &ctx->enc.cmdl_upd_info);
1806         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1807                 return -EINVAL;
1808
1809         ctx->enc.cmdl_size = cmdl_len;
1810
1811         /* Setup Decryption Security Context & Command label template */
1812         if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen,
1813                        keys.authkey, keys.authkeylen,
1814                        ad, 0, &ctx->dec.epib[1]))
1815                 return -EINVAL;
1816
1817         cfg.enc = false;
1818         cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
1819                                       &ctx->dec.cmdl_upd_info);
1820
1821         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1822                 return -EINVAL;
1823
1824         ctx->dec.cmdl_size = cmdl_len;
1825
1826         crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK);
1827         crypto_aead_set_flags(ctx->fallback.aead,
1828                               crypto_aead_get_flags(authenc) &
1829                               CRYPTO_TFM_REQ_MASK);
1830         crypto_aead_setkey(ctx->fallback.aead, key, keylen);
1831
1832         return 0;
1833 }
1834
1835 static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1836 {
1837         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
1838
1839         return crypto_aead_setauthsize(ctx->fallback.aead, authsize);
1840 }
1841
1842 static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
1843                                    const u8 *key, unsigned int keylen)
1844 {
1845         struct algo_data ad = { 0 };
1846
1847         ad.ealg_id = SA_EALG_ID_AES_CBC;
1848         ad.aalg_id = SA_AALG_ID_HMAC_SHA1;
1849         ad.hash_size = SHA1_DIGEST_SIZE;
1850         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1851
1852         return sa_aead_setkey(authenc, key, keylen, &ad);
1853 }
1854
1855 static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
1856                                      const u8 *key, unsigned int keylen)
1857 {
1858         struct algo_data ad = { 0 };
1859
1860         ad.ealg_id = SA_EALG_ID_AES_CBC;
1861         ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256;
1862         ad.hash_size = SHA256_DIGEST_SIZE;
1863         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1864
1865         return sa_aead_setkey(authenc, key, keylen, &ad);
1866 }
1867
1868 static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
1869 {
1870         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1871         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1872         struct sa_req sa_req = { 0 };
1873         size_t auth_size, enc_size;
1874
1875         enc_size = req->cryptlen;
1876         auth_size = req->assoclen + req->cryptlen;
1877
1878         if (!enc) {
1879                 enc_size -= crypto_aead_authsize(tfm);
1880                 auth_size -= crypto_aead_authsize(tfm);
1881         }
1882
1883         if (auth_size > SA_MAX_DATA_SZ ||
1884             (auth_size >= SA_UNSAFE_DATA_SZ_MIN &&
1885              auth_size <= SA_UNSAFE_DATA_SZ_MAX)) {
1886                 struct aead_request *subreq = aead_request_ctx(req);
1887                 int ret;
1888
1889                 aead_request_set_tfm(subreq, ctx->fallback.aead);
1890                 aead_request_set_callback(subreq, req->base.flags,
1891                                           req->base.complete, req->base.data);
1892                 aead_request_set_crypt(subreq, req->src, req->dst,
1893                                        req->cryptlen, req->iv);
1894                 aead_request_set_ad(subreq, req->assoclen);
1895
1896                 ret = enc ? crypto_aead_encrypt(subreq) :
1897                         crypto_aead_decrypt(subreq);
1898                 return ret;
1899         }
1900
1901         sa_req.enc_offset = req->assoclen;
1902         sa_req.enc_size = enc_size;
1903         sa_req.auth_size = auth_size;
1904         sa_req.size = auth_size;
1905         sa_req.enc_iv = iv;
1906         sa_req.type = CRYPTO_ALG_TYPE_AEAD;
1907         sa_req.enc = enc;
1908         sa_req.callback = sa_aead_dma_in_callback;
1909         sa_req.mdata_size = 52;
1910         sa_req.base = &req->base;
1911         sa_req.ctx = ctx;
1912         sa_req.src = req->src;
1913         sa_req.dst = req->dst;
1914
1915         return sa_run(&sa_req);
1916 }
1917
1918 /* AEAD algorithm encrypt interface function */
1919 static int sa_aead_encrypt(struct aead_request *req)
1920 {
1921         return sa_aead_run(req, req->iv, 1);
1922 }
1923
1924 /* AEAD algorithm decrypt interface function */
1925 static int sa_aead_decrypt(struct aead_request *req)
1926 {
1927         return sa_aead_run(req, req->iv, 0);
1928 }
1929
1930 static struct sa_alg_tmpl sa_algs[] = {
1931         {
1932                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1933                 .alg.skcipher = {
1934                         .base.cra_name          = "cbc(aes)",
1935                         .base.cra_driver_name   = "cbc-aes-sa2ul",
1936                         .base.cra_priority      = 30000,
1937                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
1938                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
1939                                                   CRYPTO_ALG_ASYNC |
1940                                                   CRYPTO_ALG_NEED_FALLBACK,
1941                         .base.cra_blocksize     = AES_BLOCK_SIZE,
1942                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
1943                         .base.cra_module        = THIS_MODULE,
1944                         .init                   = sa_cipher_cra_init,
1945                         .exit                   = sa_cipher_cra_exit,
1946                         .min_keysize            = AES_MIN_KEY_SIZE,
1947                         .max_keysize            = AES_MAX_KEY_SIZE,
1948                         .ivsize                 = AES_BLOCK_SIZE,
1949                         .setkey                 = sa_aes_cbc_setkey,
1950                         .encrypt                = sa_encrypt,
1951                         .decrypt                = sa_decrypt,
1952                 }
1953         },
1954         {
1955                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1956                 .alg.skcipher = {
1957                         .base.cra_name          = "ecb(aes)",
1958                         .base.cra_driver_name   = "ecb-aes-sa2ul",
1959                         .base.cra_priority      = 30000,
1960                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
1961                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
1962                                                   CRYPTO_ALG_ASYNC |
1963                                                   CRYPTO_ALG_NEED_FALLBACK,
1964                         .base.cra_blocksize     = AES_BLOCK_SIZE,
1965                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
1966                         .base.cra_module        = THIS_MODULE,
1967                         .init                   = sa_cipher_cra_init,
1968                         .exit                   = sa_cipher_cra_exit,
1969                         .min_keysize            = AES_MIN_KEY_SIZE,
1970                         .max_keysize            = AES_MAX_KEY_SIZE,
1971                         .setkey                 = sa_aes_ecb_setkey,
1972                         .encrypt                = sa_encrypt,
1973                         .decrypt                = sa_decrypt,
1974                 }
1975         },
1976         {
1977                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1978                 .alg.skcipher = {
1979                         .base.cra_name          = "cbc(des3_ede)",
1980                         .base.cra_driver_name   = "cbc-des3-sa2ul",
1981                         .base.cra_priority      = 30000,
1982                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
1983                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
1984                                                   CRYPTO_ALG_ASYNC |
1985                                                   CRYPTO_ALG_NEED_FALLBACK,
1986                         .base.cra_blocksize     = DES_BLOCK_SIZE,
1987                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
1988                         .base.cra_module        = THIS_MODULE,
1989                         .init                   = sa_cipher_cra_init,
1990                         .exit                   = sa_cipher_cra_exit,
1991                         .min_keysize            = 3 * DES_KEY_SIZE,
1992                         .max_keysize            = 3 * DES_KEY_SIZE,
1993                         .ivsize                 = DES_BLOCK_SIZE,
1994                         .setkey                 = sa_3des_cbc_setkey,
1995                         .encrypt                = sa_encrypt,
1996                         .decrypt                = sa_decrypt,
1997                 }
1998         },
1999         {
2000                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2001                 .alg.skcipher = {
2002                         .base.cra_name          = "ecb(des3_ede)",
2003                         .base.cra_driver_name   = "ecb-des3-sa2ul",
2004                         .base.cra_priority      = 30000,
2005                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
2006                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2007                                                   CRYPTO_ALG_ASYNC |
2008                                                   CRYPTO_ALG_NEED_FALLBACK,
2009                         .base.cra_blocksize     = DES_BLOCK_SIZE,
2010                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
2011                         .base.cra_module        = THIS_MODULE,
2012                         .init                   = sa_cipher_cra_init,
2013                         .exit                   = sa_cipher_cra_exit,
2014                         .min_keysize            = 3 * DES_KEY_SIZE,
2015                         .max_keysize            = 3 * DES_KEY_SIZE,
2016                         .setkey                 = sa_3des_ecb_setkey,
2017                         .encrypt                = sa_encrypt,
2018                         .decrypt                = sa_decrypt,
2019                 }
2020         },
2021         {
2022                 .type = CRYPTO_ALG_TYPE_AHASH,
2023                 .alg.ahash = {
2024                         .halg.base = {
2025                                 .cra_name       = "sha1",
2026                                 .cra_driver_name        = "sha1-sa2ul",
2027                                 .cra_priority   = 400,
2028                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2029                                                   CRYPTO_ALG_ASYNC |
2030                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2031                                                   CRYPTO_ALG_NEED_FALLBACK,
2032                                 .cra_blocksize  = SHA1_BLOCK_SIZE,
2033                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2034                                 .cra_module     = THIS_MODULE,
2035                                 .cra_init       = sa_sha1_cra_init,
2036                                 .cra_exit       = sa_sha_cra_exit,
2037                         },
2038                         .halg.digestsize        = SHA1_DIGEST_SIZE,
2039                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2040                                                   sizeof(struct sha1_state),
2041                         .init                   = sa_sha_init,
2042                         .update                 = sa_sha_update,
2043                         .final                  = sa_sha_final,
2044                         .finup                  = sa_sha_finup,
2045                         .digest                 = sa_sha_digest,
2046                         .export                 = sa_sha_export,
2047                         .import                 = sa_sha_import,
2048                 },
2049         },
2050         {
2051                 .type = CRYPTO_ALG_TYPE_AHASH,
2052                 .alg.ahash = {
2053                         .halg.base = {
2054                                 .cra_name       = "sha256",
2055                                 .cra_driver_name        = "sha256-sa2ul",
2056                                 .cra_priority   = 400,
2057                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2058                                                   CRYPTO_ALG_ASYNC |
2059                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2060                                                   CRYPTO_ALG_NEED_FALLBACK,
2061                                 .cra_blocksize  = SHA256_BLOCK_SIZE,
2062                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2063                                 .cra_module     = THIS_MODULE,
2064                                 .cra_init       = sa_sha256_cra_init,
2065                                 .cra_exit       = sa_sha_cra_exit,
2066                         },
2067                         .halg.digestsize        = SHA256_DIGEST_SIZE,
2068                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2069                                                   sizeof(struct sha256_state),
2070                         .init                   = sa_sha_init,
2071                         .update                 = sa_sha_update,
2072                         .final                  = sa_sha_final,
2073                         .finup                  = sa_sha_finup,
2074                         .digest                 = sa_sha_digest,
2075                         .export                 = sa_sha_export,
2076                         .import                 = sa_sha_import,
2077                 },
2078         },
2079         {
2080                 .type = CRYPTO_ALG_TYPE_AHASH,
2081                 .alg.ahash = {
2082                         .halg.base = {
2083                                 .cra_name       = "sha512",
2084                                 .cra_driver_name        = "sha512-sa2ul",
2085                                 .cra_priority   = 400,
2086                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2087                                                   CRYPTO_ALG_ASYNC |
2088                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2089                                                   CRYPTO_ALG_NEED_FALLBACK,
2090                                 .cra_blocksize  = SHA512_BLOCK_SIZE,
2091                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2092                                 .cra_module     = THIS_MODULE,
2093                                 .cra_init       = sa_sha512_cra_init,
2094                                 .cra_exit       = sa_sha_cra_exit,
2095                         },
2096                         .halg.digestsize        = SHA512_DIGEST_SIZE,
2097                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2098                                                   sizeof(struct sha512_state),
2099                         .init                   = sa_sha_init,
2100                         .update                 = sa_sha_update,
2101                         .final                  = sa_sha_final,
2102                         .finup                  = sa_sha_finup,
2103                         .digest                 = sa_sha_digest,
2104                         .export                 = sa_sha_export,
2105                         .import                 = sa_sha_import,
2106                 },
2107         },
2108         {
2109                 .type   = CRYPTO_ALG_TYPE_AEAD,
2110                 .alg.aead = {
2111                         .base = {
2112                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2113                                 .cra_driver_name =
2114                                         "authenc(hmac(sha1),cbc(aes))-sa2ul",
2115                                 .cra_blocksize = AES_BLOCK_SIZE,
2116                                 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2117                                         CRYPTO_ALG_KERN_DRIVER_ONLY |
2118                                         CRYPTO_ALG_ASYNC |
2119                                         CRYPTO_ALG_NEED_FALLBACK,
2120                                 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2121                                 .cra_module = THIS_MODULE,
2122                                 .cra_priority = 3000,
2123                         },
2124                         .ivsize = AES_BLOCK_SIZE,
2125                         .maxauthsize = SHA1_DIGEST_SIZE,
2126
2127                         .init = sa_cra_init_aead_sha1,
2128                         .exit = sa_exit_tfm_aead,
2129                         .setkey = sa_aead_cbc_sha1_setkey,
2130                         .setauthsize = sa_aead_setauthsize,
2131                         .encrypt = sa_aead_encrypt,
2132                         .decrypt = sa_aead_decrypt,
2133                 },
2134         },
2135         {
2136                 .type   = CRYPTO_ALG_TYPE_AEAD,
2137                 .alg.aead = {
2138                         .base = {
2139                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2140                                 .cra_driver_name =
2141                                         "authenc(hmac(sha256),cbc(aes))-sa2ul",
2142                                 .cra_blocksize = AES_BLOCK_SIZE,
2143                                 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2144                                         CRYPTO_ALG_KERN_DRIVER_ONLY |
2145                                         CRYPTO_ALG_ASYNC |
2146                                         CRYPTO_ALG_NEED_FALLBACK,
2147                                 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2148                                 .cra_module = THIS_MODULE,
2149                                 .cra_alignmask = 0,
2150                                 .cra_priority = 3000,
2151                         },
2152                         .ivsize = AES_BLOCK_SIZE,
2153                         .maxauthsize = SHA256_DIGEST_SIZE,
2154
2155                         .init = sa_cra_init_aead_sha256,
2156                         .exit = sa_exit_tfm_aead,
2157                         .setkey = sa_aead_cbc_sha256_setkey,
2158                         .setauthsize = sa_aead_setauthsize,
2159                         .encrypt = sa_aead_encrypt,
2160                         .decrypt = sa_aead_decrypt,
2161                 },
2162         },
2163 };
2164
2165 /* Register the algorithms in crypto framework */
2166 static void sa_register_algos(const struct device *dev)
2167 {
2168         char *alg_name;
2169         u32 type;
2170         int i, err;
2171
2172         for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2173                 type = sa_algs[i].type;
2174                 if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
2175                         alg_name = sa_algs[i].alg.skcipher.base.cra_name;
2176                         err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
2177                 } else if (type == CRYPTO_ALG_TYPE_AHASH) {
2178                         alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
2179                         err = crypto_register_ahash(&sa_algs[i].alg.ahash);
2180                 } else if (type == CRYPTO_ALG_TYPE_AEAD) {
2181                         alg_name = sa_algs[i].alg.aead.base.cra_name;
2182                         err = crypto_register_aead(&sa_algs[i].alg.aead);
2183                 } else {
2184                         dev_err(dev,
2185                                 "un-supported crypto algorithm (%d)",
2186                                 sa_algs[i].type);
2187                         continue;
2188                 }
2189
2190                 if (err)
2191                         dev_err(dev, "Failed to register '%s'\n", alg_name);
2192                 else
2193                         sa_algs[i].registered = true;
2194         }
2195 }
2196
2197 /* Unregister the algorithms in crypto framework */
2198 static void sa_unregister_algos(const struct device *dev)
2199 {
2200         u32 type;
2201         int i;
2202
2203         for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2204                 type = sa_algs[i].type;
2205                 if (!sa_algs[i].registered)
2206                         continue;
2207                 if (type == CRYPTO_ALG_TYPE_SKCIPHER)
2208                         crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
2209                 else if (type == CRYPTO_ALG_TYPE_AHASH)
2210                         crypto_unregister_ahash(&sa_algs[i].alg.ahash);
2211                 else if (type == CRYPTO_ALG_TYPE_AEAD)
2212                         crypto_unregister_aead(&sa_algs[i].alg.aead);
2213
2214                 sa_algs[i].registered = false;
2215         }
2216 }
2217
2218 static int sa_init_mem(struct sa_crypto_data *dev_data)
2219 {
2220         struct device *dev = &dev_data->pdev->dev;
2221         /* Setup dma pool for security context buffers */
2222         dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
2223                                             SA_CTX_MAX_SZ, 64, 0);
2224         if (!dev_data->sc_pool) {
2225                 dev_err(dev, "Failed to create dma pool");
2226                 return -ENOMEM;
2227         }
2228
2229         return 0;
2230 }
2231
2232 static int sa_dma_init(struct sa_crypto_data *dd)
2233 {
2234         int ret;
2235         struct dma_slave_config cfg;
2236
2237         dd->dma_rx1 = NULL;
2238         dd->dma_tx = NULL;
2239         dd->dma_rx2 = NULL;
2240
2241         ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
2242         if (ret)
2243                 return ret;
2244
2245         dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
2246         if (IS_ERR(dd->dma_rx1)) {
2247                 if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER)
2248                         dev_err(dd->dev, "Unable to request rx1 DMA channel\n");
2249                 return PTR_ERR(dd->dma_rx1);
2250         }
2251
2252         dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
2253         if (IS_ERR(dd->dma_rx2)) {
2254                 dma_release_channel(dd->dma_rx1);
2255                 if (PTR_ERR(dd->dma_rx2) != -EPROBE_DEFER)
2256                         dev_err(dd->dev, "Unable to request rx2 DMA channel\n");
2257                 return PTR_ERR(dd->dma_rx2);
2258         }
2259
2260         dd->dma_tx = dma_request_chan(dd->dev, "tx");
2261         if (IS_ERR(dd->dma_tx)) {
2262                 if (PTR_ERR(dd->dma_tx) != -EPROBE_DEFER)
2263                         dev_err(dd->dev, "Unable to request tx DMA channel\n");
2264                 ret = PTR_ERR(dd->dma_tx);
2265                 goto err_dma_tx;
2266         }
2267
2268         memzero_explicit(&cfg, sizeof(cfg));
2269
2270         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2271         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2272         cfg.src_maxburst = 4;
2273         cfg.dst_maxburst = 4;
2274
2275         ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
2276         if (ret) {
2277                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2278                         ret);
2279                 return ret;
2280         }
2281
2282         ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
2283         if (ret) {
2284                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2285                         ret);
2286                 return ret;
2287         }
2288
2289         ret = dmaengine_slave_config(dd->dma_tx, &cfg);
2290         if (ret) {
2291                 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
2292                         ret);
2293                 return ret;
2294         }
2295
2296         return 0;
2297
2298 err_dma_tx:
2299         dma_release_channel(dd->dma_rx1);
2300         dma_release_channel(dd->dma_rx2);
2301
2302         return ret;
2303 }
2304
2305 static int sa_link_child(struct device *dev, void *data)
2306 {
2307         struct device *parent = data;
2308
2309         device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER);
2310
2311         return 0;
2312 }
2313
2314 static int sa_ul_probe(struct platform_device *pdev)
2315 {
2316         struct device *dev = &pdev->dev;
2317         struct device_node *node = dev->of_node;
2318         struct resource *res;
2319         static void __iomem *saul_base;
2320         struct sa_crypto_data *dev_data;
2321         u32 val;
2322         int ret;
2323
2324         dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
2325         if (!dev_data)
2326                 return -ENOMEM;
2327
2328         sa_k3_dev = dev;
2329         dev_data->dev = dev;
2330         dev_data->pdev = pdev;
2331         platform_set_drvdata(pdev, dev_data);
2332         dev_set_drvdata(sa_k3_dev, dev_data);
2333
2334         pm_runtime_enable(dev);
2335         ret = pm_runtime_get_sync(dev);
2336         if (ret) {
2337                 dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__,
2338                         ret);
2339                 return ret;
2340         }
2341
2342         sa_init_mem(dev_data);
2343         ret = sa_dma_init(dev_data);
2344         if (ret)
2345                 goto disable_pm_runtime;
2346
2347         spin_lock_init(&dev_data->scid_lock);
2348         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2349         saul_base = devm_ioremap_resource(dev, res);
2350
2351         dev_data->base = saul_base;
2352         val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
2353             SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN |
2354             SA_EEC_TRNG_EN;
2355
2356         writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
2357
2358         sa_register_algos(dev);
2359
2360         ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
2361         if (ret)
2362                 goto release_dma;
2363
2364         device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child);
2365
2366         return 0;
2367
2368 release_dma:
2369         sa_unregister_algos(&pdev->dev);
2370
2371         dma_release_channel(dev_data->dma_rx2);
2372         dma_release_channel(dev_data->dma_rx1);
2373         dma_release_channel(dev_data->dma_tx);
2374
2375         dma_pool_destroy(dev_data->sc_pool);
2376
2377 disable_pm_runtime:
2378         pm_runtime_put_sync(&pdev->dev);
2379         pm_runtime_disable(&pdev->dev);
2380
2381         return ret;
2382 }
2383
2384 static int sa_ul_remove(struct platform_device *pdev)
2385 {
2386         struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
2387
2388         sa_unregister_algos(&pdev->dev);
2389
2390         dma_release_channel(dev_data->dma_rx2);
2391         dma_release_channel(dev_data->dma_rx1);
2392         dma_release_channel(dev_data->dma_tx);
2393
2394         dma_pool_destroy(dev_data->sc_pool);
2395
2396         platform_set_drvdata(pdev, NULL);
2397
2398         pm_runtime_put_sync(&pdev->dev);
2399         pm_runtime_disable(&pdev->dev);
2400
2401         return 0;
2402 }
2403
2404 static const struct of_device_id of_match[] = {
2405         {.compatible = "ti,j721e-sa2ul",},
2406         {.compatible = "ti,am654-sa2ul",},
2407         {},
2408 };
2409 MODULE_DEVICE_TABLE(of, of_match);
2410
2411 static struct platform_driver sa_ul_driver = {
2412         .probe = sa_ul_probe,
2413         .remove = sa_ul_remove,
2414         .driver = {
2415                    .name = "saul-crypto",
2416                    .of_match_table = of_match,
2417                    },
2418 };
2419 module_platform_driver(sa_ul_driver);
2420 MODULE_LICENSE("GPL v2");