drivers/crypto/sa2ul.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * K3 SA2UL crypto accelerator driver
   4  *
   5  * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors:     Keerthy
   8  *              Vitaly Andrianov
   9  *              Tero Kristo
  10  */
  11 #include <linux/clk.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/dmapool.h>
  15 #include <linux/kernel.h>
  16 #include <linux/module.h>
  17 #include <linux/of_device.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/pm_runtime.h>
  20
  21 #include <crypto/aes.h>
  22 #include <crypto/authenc.h>
  23 #include <crypto/des.h>
  24 #include <crypto/internal/aead.h>
  25 #include <crypto/internal/hash.h>
  26 #include <crypto/internal/skcipher.h>
  27 #include <crypto/scatterwalk.h>
  28 #include <crypto/sha1.h>
  29 #include <crypto/sha2.h>
  30
  31 #include "sa2ul.h"
  32
  33 /* Byte offset for key in encryption security context */
  34 #define SC_ENC_KEY_OFFSET (1 + 27 + 4)
  35 /* Byte offset for Aux-1 in encryption security context */
  36 #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
  37
  38 #define SA_CMDL_UPD_ENC         0x0001
  39 #define SA_CMDL_UPD_AUTH        0x0002
  40 #define SA_CMDL_UPD_ENC_IV      0x0004
  41 #define SA_CMDL_UPD_AUTH_IV     0x0008
  42 #define SA_CMDL_UPD_AUX_KEY     0x0010
  43
  44 #define SA_AUTH_SUBKEY_LEN      16
  45 #define SA_CMDL_PAYLOAD_LENGTH_MASK     0xFFFF
  46 #define SA_CMDL_SOP_BYPASS_LEN_MASK     0xFF000000
  47
  48 #define MODE_CONTROL_BYTES      27
  49 #define SA_HASH_PROCESSING      0
  50 #define SA_CRYPTO_PROCESSING    0
  51 #define SA_UPLOAD_HASH_TO_TLR   BIT(6)
  52
  53 #define SA_SW0_FLAGS_MASK       0xF0000
  54 #define SA_SW0_CMDL_INFO_MASK   0x1F00000
  55 #define SA_SW0_CMDL_PRESENT     BIT(4)
  56 #define SA_SW0_ENG_ID_MASK      0x3E000000
  57 #define SA_SW0_DEST_INFO_PRESENT        BIT(30)
  58 #define SA_SW2_EGRESS_LENGTH            0xFF000000
  59 #define SA_BASIC_HASH           0x10
  60
  61 #define SHA256_DIGEST_WORDS    8
  62 /* Make 32-bit word from 4 bytes */
  63 #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
  64                                    ((b2) << 8) | (b3))
  65
  66 /* size of SCCTL structure in bytes */
  67 #define SA_SCCTL_SZ 16
  68
  69 /* Max Authentication tag size */
  70 #define SA_MAX_AUTH_TAG_SZ 64
  71
  72 enum sa_algo_id {
  73         SA_ALG_CBC_AES = 0,
  74         SA_ALG_EBC_AES,
  75         SA_ALG_CBC_DES3,
  76         SA_ALG_ECB_DES3,
  77         SA_ALG_SHA1,
  78         SA_ALG_SHA256,
  79         SA_ALG_SHA512,
  80         SA_ALG_AUTHENC_SHA1_AES,
  81         SA_ALG_AUTHENC_SHA256_AES,
  82 };
  83
  84 struct sa_match_data {
  85         u8 priv;
  86         u8 priv_id;
  87         u32 supported_algos;
  88         bool skip_engine_control;
  89 };
  90
  91 static struct device *sa_k3_dev;
  92
  93 /**
  94  * struct sa_cmdl_cfg - Command label configuration descriptor
  95  * @aalg: authentication algorithm ID
  96  * @enc_eng_id: Encryption Engine ID supported by the SA hardware
  97  * @auth_eng_id: Authentication Engine ID
  98  * @iv_size: Initialization Vector size
  99  * @akey: Authentication key
 100  * @akey_len: Authentication key length
 101  * @enc: True, if this is an encode request
 102  */
 103 struct sa_cmdl_cfg {
 104         int aalg;
 105         u8 enc_eng_id;
 106         u8 auth_eng_id;
 107         u8 iv_size;
 108         const u8 *akey;
 109         u16 akey_len;
 110         bool enc;
 111 };
 112
 113 /**
 114  * struct algo_data - Crypto algorithm specific data
 115  * @enc_eng: Encryption engine info structure
 116  * @auth_eng: Authentication engine info structure
 117  * @auth_ctrl: Authentication control word
 118  * @hash_size: Size of digest
 119  * @iv_idx: iv index in psdata
 120  * @iv_out_size: iv out size
 121  * @ealg_id: Encryption Algorithm ID
 122  * @aalg_id: Authentication algorithm ID
 123  * @mci_enc: Mode Control Instruction for Encryption algorithm
 124  * @mci_dec: Mode Control Instruction for Decryption
 125  * @inv_key: Whether the encryption algorithm demands key inversion
 126  * @ctx: Pointer to the algorithm context
 127  * @keyed_mac: Whether the authentication algorithm has key
 128  * @prep_iopad: Function pointer to generate intermediate ipad/opad
 129  */
 130 struct algo_data {
 131         struct sa_eng_info enc_eng;
 132         struct sa_eng_info auth_eng;
 133         u8 auth_ctrl;
 134         u8 hash_size;
 135         u8 iv_idx;
 136         u8 iv_out_size;
 137         u8 ealg_id;
 138         u8 aalg_id;
 139         u8 *mci_enc;
 140         u8 *mci_dec;
 141         bool inv_key;
 142         struct sa_tfm_ctx *ctx;
 143         bool keyed_mac;
 144         void (*prep_iopad)(struct algo_data *algo, const u8 *key,
 145                            u16 key_sz, __be32 *ipad, __be32 *opad);
 146 };
 147
 148 /**
 149  * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
 150  * @type: Type of the crypto algorithm.
 151  * @alg: Union of crypto algorithm definitions.
 152  * @registered: Flag indicating if the crypto algorithm is already registered
 153  */
 154 struct sa_alg_tmpl {
 155         u32 type;               /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
 156         union {
 157                 struct skcipher_alg skcipher;
 158                 struct ahash_alg ahash;
 159                 struct aead_alg aead;
 160         } alg;
 161         bool registered;
 162 };
 163
 164 /**
 165  * struct sa_mapped_sg: scatterlist information for tx and rx
 166  * @mapped: Set to true if the @sgt is mapped
 167  * @dir: mapping direction used for @sgt
 168  * @split_sg: Set if the sg is split and needs to be freed up
 169  * @static_sg: Static scatterlist entry for overriding data
 170  * @sgt: scatterlist table for DMA API use
 171  */
 172 struct sa_mapped_sg {
 173         bool mapped;
 174         enum dma_data_direction dir;
 175         struct scatterlist static_sg;
 176         struct scatterlist *split_sg;
 177         struct sg_table sgt;
 178 };
 179 /**
 180  * struct sa_rx_data: RX Packet miscellaneous data place holder
 181  * @req: crypto request data pointer
 182  * @ddev: pointer to the DMA device
 183  * @tx_in: dma_async_tx_descriptor pointer for rx channel
 184  * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
 185  * @enc: Flag indicating either encryption or decryption
 186  * @enc_iv_size: Initialisation vector size
 187  * @iv_idx: Initialisation vector index
 188  */
 189 struct sa_rx_data {
 190         void *req;
 191         struct device *ddev;
 192         struct dma_async_tx_descriptor *tx_in;
 193         struct sa_mapped_sg mapped_sg[2];
 194         u8 enc;
 195         u8 enc_iv_size;
 196         u8 iv_idx;
 197 };
 198
 199 /**
 200  * struct sa_req: SA request definition
 201  * @dev: device for the request
 202  * @size: total data to the xmitted via DMA
 203  * @enc_offset: offset of cipher data
 204  * @enc_size: data to be passed to cipher engine
 205  * @enc_iv: cipher IV
 206  * @auth_offset: offset of the authentication data
 207  * @auth_size: size of the authentication data
 208  * @auth_iv: authentication IV
 209  * @type: algorithm type for the request
 210  * @cmdl: command label pointer
 211  * @base: pointer to the base request
 212  * @ctx: pointer to the algorithm context data
 213  * @enc: true if this is an encode request
 214  * @src: source data
 215  * @dst: destination data
 216  * @callback: DMA callback for the request
 217  * @mdata_size: metadata size passed to DMA
 218  */
 219 struct sa_req {
 220         struct device *dev;
 221         u16 size;
 222         u8 enc_offset;
 223         u16 enc_size;
 224         u8 *enc_iv;
 225         u8 auth_offset;
 226         u16 auth_size;
 227         u8 *auth_iv;
 228         u32 type;
 229         u32 *cmdl;
 230         struct crypto_async_request *base;
 231         struct sa_tfm_ctx *ctx;
 232         bool enc;
 233         struct scatterlist *src;
 234         struct scatterlist *dst;
 235         dma_async_tx_callback callback;
 236         u16 mdata_size;
 237 };
 238
 239 /*
 240  * Mode Control Instructions for various Key lengths 128, 192, 256
 241  * For CBC (Cipher Block Chaining) mode for encryption
 242  */
 243 static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
 244         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
 245                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 246                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 247         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
 248                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 249                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 250         {       0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
 251                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 252                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 253 };
 254
 255 /*
 256  * Mode Control Instructions for various Key lengths 128, 192, 256
 257  * For CBC (Cipher Block Chaining) mode for decryption
 258  */
 259 static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
 260         {       0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 261                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 262                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 263         {       0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 264                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 265                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 266         {       0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 267                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 268                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 269 };
 270
 271 /*
 272  * Mode Control Instructions for various Key lengths 128, 192, 256
 273  * For CBC (Cipher Block Chaining) mode for encryption
 274  */
 275 static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = {
 276         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
 277                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 278                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 279         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
 280                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 281                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 282         {       0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
 283                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 284                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 285 };
 286
 287 /*
 288  * Mode Control Instructions for various Key lengths 128, 192, 256
 289  * For CBC (Cipher Block Chaining) mode for decryption
 290  */
 291 static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = {
 292         {       0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 293                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 294                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 295         {       0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 296                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 297                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 298         {       0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
 299                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 300                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 301 };
 302
 303 /*
 304  * Mode Control Instructions for various Key lengths 128, 192, 256
 305  * For ECB (Electronic Code Book) mode for encryption
 306  */
 307 static u8 mci_ecb_enc_array[3][27] = {
 308         {       0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 309                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 310                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 311         {       0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 312                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 313                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 314         {       0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 315                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 316                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 317 };
 318
 319 /*
 320  * Mode Control Instructions for various Key lengths 128, 192, 256
 321  * For ECB (Electronic Code Book) mode for decryption
 322  */
 323 static u8 mci_ecb_dec_array[3][27] = {
 324         {       0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 325                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 326                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 327         {       0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 328                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 329                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 330         {       0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
 331                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 332                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00        },
 333 };
 334
 335 /*
 336  * Mode Control Instructions for DES algorithm
 337  * For CBC (Cipher Block Chaining) mode and ECB mode
 338  * encryption and for decryption respectively
 339  */
 340 static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
 341         0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
 342         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 343         0x00, 0x00, 0x00,
 344 };
 345
 346 static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
 347         0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
 348         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 349         0x00, 0x00, 0x00,
 350 };
 351
 352 static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
 353         0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
 354         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 355         0x00, 0x00, 0x00,
 356 };
 357
 358 static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
 359         0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
 360         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 361         0x00, 0x00, 0x00,
 362 };
 363
 364 /*
 365  * Perform 16 byte or 128 bit swizzling
 366  * The SA2UL Expects the security context to
 367  * be in little Endian and the bus width is 128 bits or 16 bytes
 368  * Hence swap 16 bytes at a time from higher to lower address
 369  */
 370 static void sa_swiz_128(u8 *in, u16 len)
 371 {
 372         u8 data[16];
 373         int i, j;
 374
 375         for (i = 0; i < len; i += 16) {
 376                 memcpy(data, &in[i], 16);
 377                 for (j = 0; j < 16; j++)
 378                         in[i + j] = data[15 - j];
 379         }
 380 }
 381
 382 /* Prepare the ipad and opad from key as per SHA algorithm step 1*/
 383 static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz)
 384 {
 385         int i;
 386
 387         for (i = 0; i < key_sz; i++)
 388                 k_ipad[i] = key[i] ^ 0x36;
 389
 390         /* Instead of XOR with 0 */
 391         for (; i < SHA1_BLOCK_SIZE; i++)
 392                 k_ipad[i] = 0x36;
 393 }
 394
 395 static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz)
 396 {
 397         int i;
 398
 399         for (i = 0; i < key_sz; i++)
 400                 k_opad[i] = key[i] ^ 0x5c;
 401
 402         /* Instead of XOR with 0 */
 403         for (; i < SHA1_BLOCK_SIZE; i++)
 404                 k_opad[i] = 0x5c;
 405 }
 406
 407 static void sa_export_shash(void *state, struct shash_desc *hash,
 408                             int digest_size, __be32 *out)
 409 {
 410         struct sha1_state *sha1;
 411         struct sha256_state *sha256;
 412         u32 *result;
 413
 414         switch (digest_size) {
 415         case SHA1_DIGEST_SIZE:
 416                 sha1 = state;
 417                 result = sha1->state;
 418                 break;
 419         case SHA256_DIGEST_SIZE:
 420                 sha256 = state;
 421                 result = sha256->state;
 422                 break;
 423         default:
 424                 dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__,
 425                         digest_size);
 426                 return;
 427         }
 428
 429         crypto_shash_export(hash, state);
 430
 431         cpu_to_be32_array(out, result, digest_size / 4);
 432 }
 433
 434 static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
 435                               u16 key_sz, __be32 *ipad, __be32 *opad)
 436 {
 437         SHASH_DESC_ON_STACK(shash, data->ctx->shash);
 438         int block_size = crypto_shash_blocksize(data->ctx->shash);
 439         int digest_size = crypto_shash_digestsize(data->ctx->shash);
 440         union {
 441                 struct sha1_state sha1;
 442                 struct sha256_state sha256;
 443                 u8 k_pad[SHA1_BLOCK_SIZE];
 444         } sha;
 445
 446         shash->tfm = data->ctx->shash;
 447
 448         prepare_kipad(sha.k_pad, key, key_sz);
 449
 450         crypto_shash_init(shash);
 451         crypto_shash_update(shash, sha.k_pad, block_size);
 452         sa_export_shash(&sha, shash, digest_size, ipad);
 453
 454         prepare_kopad(sha.k_pad, key, key_sz);
 455
 456         crypto_shash_init(shash);
 457         crypto_shash_update(shash, sha.k_pad, block_size);
 458
 459         sa_export_shash(&sha, shash, digest_size, opad);
 460
 461         memzero_explicit(&sha, sizeof(sha));
 462 }
 463
 464 /* Derive the inverse key used in AES-CBC decryption operation */
 465 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
 466 {
 467         struct crypto_aes_ctx ctx;
 468         int key_pos;
 469
 470         if (aes_expandkey(&ctx, key, key_sz)) {
 471                 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
 472                 return -EINVAL;
 473         }
 474
 475         /* work around to get the right inverse for AES_KEYSIZE_192 size keys */
 476         if (key_sz == AES_KEYSIZE_192) {
 477                 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46];
 478                 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47];
 479         }
 480
 481         /* Based crypto_aes_expand_key logic */
 482         switch (key_sz) {
 483         case AES_KEYSIZE_128:
 484         case AES_KEYSIZE_192:
 485                 key_pos = key_sz + 24;
 486                 break;
 487
 488         case AES_KEYSIZE_256:
 489                 key_pos = key_sz + 24 - 4;
 490                 break;
 491
 492         default:
 493                 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
 494                 return -EINVAL;
 495         }
 496
 497         memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
 498         return 0;
 499 }
 500
 501 /* Set Security context for the encryption engine */
 502 static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
 503                          u8 enc, u8 *sc_buf)
 504 {
 505         const u8 *mci = NULL;
 506
 507         /* Set Encryption mode selector to crypto processing */
 508         sc_buf[0] = SA_CRYPTO_PROCESSING;
 509
 510         if (enc)
 511                 mci = ad->mci_enc;
 512         else
 513                 mci = ad->mci_dec;
 514         /* Set the mode control instructions in security context */
 515         if (mci)
 516                 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
 517
 518         /* For AES-CBC decryption get the inverse key */
 519         if (ad->inv_key && !enc) {
 520                 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
 521                         return -EINVAL;
 522         /* For all other cases: key is used */
 523         } else {
 524                 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
 525         }
 526
 527         return 0;
 528 }
 529
 530 /* Set Security context for the authentication engine */
 531 static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
 532                            u8 *sc_buf)
 533 {
 534         __be32 *ipad = (void *)(sc_buf + 32);
 535         __be32 *opad = (void *)(sc_buf + 64);
 536
 537         /* Set Authentication mode selector to hash processing */
 538         sc_buf[0] = SA_HASH_PROCESSING;
 539         /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
 540         sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
 541         sc_buf[1] |= ad->auth_ctrl;
 542
 543         /* Copy the keys or ipad/opad */
 544         if (ad->keyed_mac)
 545                 ad->prep_iopad(ad, key, key_sz, ipad, opad);
 546         else {
 547                 /* basic hash */
 548                 sc_buf[1] |= SA_BASIC_HASH;
 549         }
 550 }
 551
 552 static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16)
 553 {
 554         int j;
 555
 556         for (j = 0; j < ((size16) ? 4 : 2); j++) {
 557                 *out = cpu_to_be32(*((u32 *)iv));
 558                 iv += 4;
 559                 out++;
 560         }
 561 }
 562
 563 /* Format general command label */
 564 static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
 565                               struct sa_cmdl_upd_info *upd_info)
 566 {
 567         u8 enc_offset = 0, auth_offset = 0, total = 0;
 568         u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
 569         u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
 570         u32 *word_ptr = (u32 *)cmdl;
 571         int i;
 572
 573         /* Clear the command label */
 574         memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
 575
 576         /* Iniialize the command update structure */
 577         memzero_explicit(upd_info, sizeof(*upd_info));
 578
 579         if (cfg->enc_eng_id && cfg->auth_eng_id) {
 580                 if (cfg->enc) {
 581                         auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
 582                         enc_next_eng = cfg->auth_eng_id;
 583
 584                         if (cfg->iv_size)
 585                                 auth_offset += cfg->iv_size;
 586                 } else {
 587                         enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
 588                         auth_next_eng = cfg->enc_eng_id;
 589                 }
 590         }
 591
 592         if (cfg->enc_eng_id) {
 593                 upd_info->flags |= SA_CMDL_UPD_ENC;
 594                 upd_info->enc_size.index = enc_offset >> 2;
 595                 upd_info->enc_offset.index = upd_info->enc_size.index + 1;
 596                 /* Encryption command label */
 597                 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
 598
 599                 /* Encryption modes requiring IV */
 600                 if (cfg->iv_size) {
 601                         upd_info->flags |= SA_CMDL_UPD_ENC_IV;
 602                         upd_info->enc_iv.index =
 603                                 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
 604                         upd_info->enc_iv.size = cfg->iv_size;
 605
 606                         cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 607                                 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
 608
 609                         cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
 610                                 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
 611                         total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
 612                 } else {
 613                         cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 614                                                 SA_CMDL_HEADER_SIZE_BYTES;
 615                         total += SA_CMDL_HEADER_SIZE_BYTES;
 616                 }
 617         }
 618
 619         if (cfg->auth_eng_id) {
 620                 upd_info->flags |= SA_CMDL_UPD_AUTH;
 621                 upd_info->auth_size.index = auth_offset >> 2;
 622                 upd_info->auth_offset.index = upd_info->auth_size.index + 1;
 623                 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
 624                 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
 625                         SA_CMDL_HEADER_SIZE_BYTES;
 626                 total += SA_CMDL_HEADER_SIZE_BYTES;
 627         }
 628
 629         total = roundup(total, 8);
 630
 631         for (i = 0; i < total / 4; i++)
 632                 word_ptr[i] = swab32(word_ptr[i]);
 633
 634         return total;
 635 }
 636
 637 /* Update Command label */
 638 static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
 639                                   struct sa_cmdl_upd_info *upd_info)
 640 {
 641         int i = 0, j;
 642
 643         if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
 644                 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
 645                 cmdl[upd_info->enc_size.index] |= req->enc_size;
 646                 cmdl[upd_info->enc_offset.index] &=
 647                                                 ~SA_CMDL_SOP_BYPASS_LEN_MASK;
 648                 cmdl[upd_info->enc_offset.index] |=
 649                         ((u32)req->enc_offset <<
 650                          __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
 651
 652                 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
 653                         __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
 654                         u32 *enc_iv = (u32 *)req->enc_iv;
 655
 656                         for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
 657                                 data[j] = cpu_to_be32(*enc_iv);
 658                                 enc_iv++;
 659                         }
 660                 }
 661         }
 662
 663         if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
 664                 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
 665                 cmdl[upd_info->auth_size.index] |= req->auth_size;
 666                 cmdl[upd_info->auth_offset.index] &=
 667                         ~SA_CMDL_SOP_BYPASS_LEN_MASK;
 668                 cmdl[upd_info->auth_offset.index] |=
 669                         ((u32)req->auth_offset <<
 670                          __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
 671                 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
 672                         sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
 673                                    req->auth_iv,
 674                                    (upd_info->auth_iv.size > 8));
 675                 }
 676                 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
 677                         int offset = (req->auth_size & 0xF) ? 4 : 0;
 678
 679                         memcpy(&cmdl[upd_info->aux_key_info.index],
 680                                &upd_info->aux_key[offset], 16);
 681                 }
 682         }
 683 }
 684
 685 /* Format SWINFO words to be sent to SA */
 686 static
 687 void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
 688                    u8 cmdl_present, u8 cmdl_offset, u8 flags,
 689                    u8 hash_size, u32 *swinfo)
 690 {
 691         swinfo[0] = sc_id;
 692         swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK));
 693         if (likely(cmdl_present))
 694                 swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) <<
 695                                                 __ffs(SA_SW0_CMDL_INFO_MASK));
 696         swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK));
 697
 698         swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
 699         swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
 700         swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
 701         swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH));
 702 }
 703
 704 /* Dump the security context */
 705 static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
 706 {
 707 #ifdef DEBUG
 708         dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
 709         print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
 710                        16, 1, buf, SA_CTX_MAX_SZ, false);
 711 #endif
 712 }
 713
 714 static
 715 int sa_init_sc(struct sa_ctx_info *ctx, const struct sa_match_data *match_data,
 716                const u8 *enc_key, u16 enc_key_sz,
 717                const u8 *auth_key, u16 auth_key_sz,
 718                struct algo_data *ad, u8 enc, u32 *swinfo)
 719 {
 720         int enc_sc_offset = 0;
 721         int auth_sc_offset = 0;
 722         u8 *sc_buf = ctx->sc;
 723         u16 sc_id = ctx->sc_id;
 724         u8 first_engine = 0;
 725
 726         memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
 727
 728         if (ad->auth_eng.eng_id) {
 729                 if (enc)
 730                         first_engine = ad->enc_eng.eng_id;
 731                 else
 732                         first_engine = ad->auth_eng.eng_id;
 733
 734                 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
 735                 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
 736                 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
 737                 if (!ad->hash_size)
 738                         return -EINVAL;
 739                 ad->hash_size = roundup(ad->hash_size, 8);
 740
 741         } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) {
 742                 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
 743                 first_engine = ad->enc_eng.eng_id;
 744                 sc_buf[1] = SA_SCCTL_FE_ENC;
 745                 ad->hash_size = ad->iv_out_size;
 746         }
 747
 748         /* SCCTL Owner info: 0=host, 1=CP_ACE */
 749         sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
 750         memcpy(&sc_buf[2], &sc_id, 2);
 751         sc_buf[4] = 0x0;
 752         sc_buf[5] = match_data->priv_id;
 753         sc_buf[6] = match_data->priv;
 754         sc_buf[7] = 0x0;
 755
 756         /* Prepare context for encryption engine */
 757         if (ad->enc_eng.sc_size) {
 758                 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc,
 759                                   &sc_buf[enc_sc_offset]))
 760                         return -EINVAL;
 761         }
 762
 763         /* Prepare context for authentication engine */
 764         if (ad->auth_eng.sc_size)
 765                 sa_set_sc_auth(ad, auth_key, auth_key_sz,
 766                                &sc_buf[auth_sc_offset]);
 767
 768         /* Set the ownership of context to CP_ACE */
 769         sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
 770
 771         /* swizzle the security context */
 772         sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
 773
 774         sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
 775                       SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
 776
 777         sa_dump_sc(sc_buf, ctx->sc_phys);
 778
 779         return 0;
 780 }
 781
 782 /* Free the per direction context memory */
 783 static void sa_free_ctx_info(struct sa_ctx_info *ctx,
 784                              struct sa_crypto_data *data)
 785 {
 786         unsigned long bn;
 787
 788         bn = ctx->sc_id - data->sc_id_start;
 789         spin_lock(&data->scid_lock);
 790         __clear_bit(bn, data->ctx_bm);
 791         data->sc_id--;
 792         spin_unlock(&data->scid_lock);
 793
 794         if (ctx->sc) {
 795                 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
 796                 ctx->sc = NULL;
 797         }
 798 }
 799
 800 static int sa_init_ctx_info(struct sa_ctx_info *ctx,
 801                             struct sa_crypto_data *data)
 802 {
 803         unsigned long bn;
 804         int err;
 805
 806         spin_lock(&data->scid_lock);
 807         bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
 808         __set_bit(bn, data->ctx_bm);
 809         data->sc_id++;
 810         spin_unlock(&data->scid_lock);
 811
 812         ctx->sc_id = (u16)(data->sc_id_start + bn);
 813
 814         ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
 815         if (!ctx->sc) {
 816                 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
 817                 err = -ENOMEM;
 818                 goto scid_rollback;
 819         }
 820
 821         return 0;
 822
 823 scid_rollback:
 824         spin_lock(&data->scid_lock);
 825         __clear_bit(bn, data->ctx_bm);
 826         data->sc_id--;
 827         spin_unlock(&data->scid_lock);
 828
 829         return err;
 830 }
 831
 832 static void sa_cipher_cra_exit(struct crypto_skcipher *tfm)
 833 {
 834         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 835         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
 836
 837         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
 838                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
 839                 ctx->dec.sc_id, &ctx->dec.sc_phys);
 840
 841         sa_free_ctx_info(&ctx->enc, data);
 842         sa_free_ctx_info(&ctx->dec, data);
 843
 844         crypto_free_skcipher(ctx->fallback.skcipher);
 845 }
 846
 847 static int sa_cipher_cra_init(struct crypto_skcipher *tfm)
 848 {
 849         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 850         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
 851         const char *name = crypto_tfm_alg_name(&tfm->base);
 852         struct crypto_skcipher *child;
 853         int ret;
 854
 855         memzero_explicit(ctx, sizeof(*ctx));
 856         ctx->dev_data = data;
 857
 858         ret = sa_init_ctx_info(&ctx->enc, data);
 859         if (ret)
 860                 return ret;
 861         ret = sa_init_ctx_info(&ctx->dec, data);
 862         if (ret) {
 863                 sa_free_ctx_info(&ctx->enc, data);
 864                 return ret;
 865         }
 866
 867         child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
 868
 869         if (IS_ERR(child)) {
 870                 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name);
 871                 return PTR_ERR(child);
 872         }
 873
 874         ctx->fallback.skcipher = child;
 875         crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
 876                                          sizeof(struct skcipher_request));
 877
 878         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
 879                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
 880                 ctx->dec.sc_id, &ctx->dec.sc_phys);
 881         return 0;
 882 }
 883
 884 static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 885                             unsigned int keylen, struct algo_data *ad)
 886 {
 887         struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 888         struct crypto_skcipher *child = ctx->fallback.skcipher;
 889         int cmdl_len;
 890         struct sa_cmdl_cfg cfg;
 891         int ret;
 892
 893         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 894             keylen != AES_KEYSIZE_256)
 895                 return -EINVAL;
 896
 897         ad->enc_eng.eng_id = SA_ENG_ID_EM1;
 898         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
 899
 900         memzero_explicit(&cfg, sizeof(cfg));
 901         cfg.enc_eng_id = ad->enc_eng.eng_id;
 902         cfg.iv_size = crypto_skcipher_ivsize(tfm);
 903
 904         crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 905         crypto_skcipher_set_flags(child, tfm->base.crt_flags &
 906                                          CRYPTO_TFM_REQ_MASK);
 907         ret = crypto_skcipher_setkey(child, key, keylen);
 908         if (ret)
 909                 return ret;
 910
 911         /* Setup Encryption Security Context & Command label template */
 912         if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, key, keylen, NULL, 0,
 913                        ad, 1, &ctx->enc.epib[1]))
 914                 goto badkey;
 915
 916         cmdl_len = sa_format_cmdl_gen(&cfg,
 917                                       (u8 *)ctx->enc.cmdl,
 918                                       &ctx->enc.cmdl_upd_info);
 919         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
 920                 goto badkey;
 921
 922         ctx->enc.cmdl_size = cmdl_len;
 923
 924         /* Setup Decryption Security Context & Command label template */
 925         if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, key, keylen, NULL, 0,
 926                        ad, 0, &ctx->dec.epib[1]))
 927                 goto badkey;
 928
 929         cfg.enc_eng_id = ad->enc_eng.eng_id;
 930         cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
 931                                       &ctx->dec.cmdl_upd_info);
 932
 933         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
 934                 goto badkey;
 935
 936         ctx->dec.cmdl_size = cmdl_len;
 937         ctx->iv_idx = ad->iv_idx;
 938
 939         return 0;
 940
 941 badkey:
 942         dev_err(sa_k3_dev, "%s: badkey\n", __func__);
 943         return -EINVAL;
 944 }
 945
 946 static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
 947                              unsigned int keylen)
 948 {
 949         struct algo_data ad = { 0 };
 950         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
 951         int key_idx = (keylen >> 3) - 2;
 952
 953         if (key_idx >= 3)
 954                 return -EINVAL;
 955
 956         ad.mci_enc = mci_cbc_enc_array[key_idx];
 957         ad.mci_dec = mci_cbc_dec_array[key_idx];
 958         ad.inv_key = true;
 959         ad.ealg_id = SA_EALG_ID_AES_CBC;
 960         ad.iv_idx = 4;
 961         ad.iv_out_size = 16;
 962
 963         return sa_cipher_setkey(tfm, key, keylen, &ad);
 964 }
 965
 966 static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
 967                              unsigned int keylen)
 968 {
 969         struct algo_data ad = { 0 };
 970         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
 971         int key_idx = (keylen >> 3) - 2;
 972
 973         if (key_idx >= 3)
 974                 return -EINVAL;
 975
 976         ad.mci_enc = mci_ecb_enc_array[key_idx];
 977         ad.mci_dec = mci_ecb_dec_array[key_idx];
 978         ad.inv_key = true;
 979         ad.ealg_id = SA_EALG_ID_AES_ECB;
 980
 981         return sa_cipher_setkey(tfm, key, keylen, &ad);
 982 }
 983
 984 static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
 985                               unsigned int keylen)
 986 {
 987         struct algo_data ad = { 0 };
 988
 989         ad.mci_enc = mci_cbc_3des_enc_array;
 990         ad.mci_dec = mci_cbc_3des_dec_array;
 991         ad.ealg_id = SA_EALG_ID_3DES_CBC;
 992         ad.iv_idx = 6;
 993         ad.iv_out_size = 8;
 994
 995         return sa_cipher_setkey(tfm, key, keylen, &ad);
 996 }
 997
 998 static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
 999                               unsigned int keylen)
1000 {
1001         struct algo_data ad = { 0 };
1002
1003         ad.mci_enc = mci_ecb_3des_enc_array;
1004         ad.mci_dec = mci_ecb_3des_dec_array;
1005
1006         return sa_cipher_setkey(tfm, key, keylen, &ad);
1007 }
1008
1009 static void sa_sync_from_device(struct sa_rx_data *rxd)
1010 {
1011         struct sg_table *sgt;
1012
1013         if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
1014                 sgt = &rxd->mapped_sg[0].sgt;
1015         else
1016                 sgt = &rxd->mapped_sg[1].sgt;
1017
1018         dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
1019 }
1020
1021 static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
1022 {
1023         int i;
1024
1025         for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
1026                 struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
1027
1028                 if (mapped_sg->mapped) {
1029                         dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
1030                                           mapped_sg->dir, 0);
1031                         kfree(mapped_sg->split_sg);
1032                 }
1033         }
1034
1035         kfree(rxd);
1036 }
1037
1038 static void sa_aes_dma_in_callback(void *data)
1039 {
1040         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1041         struct skcipher_request *req;
1042         u32 *result;
1043         __be32 *mdptr;
1044         size_t ml, pl;
1045         int i;
1046
1047         sa_sync_from_device(rxd);
1048         req = container_of(rxd->req, struct skcipher_request, base);
1049
1050         if (req->iv) {
1051                 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
1052                                                                &ml);
1053                 result = (u32 *)req->iv;
1054
1055                 for (i = 0; i < (rxd->enc_iv_size / 4); i++)
1056                         result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
1057         }
1058
1059         sa_free_sa_rx_data(rxd);
1060
1061         skcipher_request_complete(req, 0);
1062 }
1063
1064 static void
1065 sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
1066 {
1067         u32 *out, *in;
1068         int i;
1069
1070         for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
1071                 *out++ = *in++;
1072
1073         mdptr[4] = (0xFFFF << 16);
1074         for (out = &mdptr[5], in = psdata, i = 0;
1075              i < pslen / sizeof(u32); i++)
1076                 *out++ = *in++;
1077 }
1078
1079 static int sa_run(struct sa_req *req)
1080 {
1081         struct sa_rx_data *rxd;
1082         gfp_t gfp_flags;
1083         u32 cmdl[SA_MAX_CMDL_WORDS];
1084         struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
1085         struct device *ddev;
1086         struct dma_chan *dma_rx;
1087         int sg_nents, src_nents, dst_nents;
1088         struct scatterlist *src, *dst;
1089         size_t pl, ml, split_size;
1090         struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
1091         int ret;
1092         struct dma_async_tx_descriptor *tx_out;
1093         u32 *mdptr;
1094         bool diff_dst;
1095         enum dma_data_direction dir_src;
1096         struct sa_mapped_sg *mapped_sg;
1097
1098         gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1099                 GFP_KERNEL : GFP_ATOMIC;
1100
1101         rxd = kzalloc(sizeof(*rxd), gfp_flags);
1102         if (!rxd)
1103                 return -ENOMEM;
1104
1105         if (req->src != req->dst) {
1106                 diff_dst = true;
1107                 dir_src = DMA_TO_DEVICE;
1108         } else {
1109                 diff_dst = false;
1110                 dir_src = DMA_BIDIRECTIONAL;
1111         }
1112
1113         /*
1114          * SA2UL has an interesting feature where the receive DMA channel
1115          * is selected based on the data passed to the engine. Within the
1116          * transition range, there is also a space where it is impossible
1117          * to determine where the data will end up, and this should be
1118          * avoided. This will be handled by the SW fallback mechanism by
1119          * the individual algorithm implementations.
1120          */
1121         if (req->size >= 256)
1122                 dma_rx = pdata->dma_rx2;
1123         else
1124                 dma_rx = pdata->dma_rx1;
1125
1126         ddev = dmaengine_get_dma_device(pdata->dma_tx);
1127         rxd->ddev = ddev;
1128
1129         memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
1130
1131         sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info);
1132
1133         if (req->type != CRYPTO_ALG_TYPE_AHASH) {
1134                 if (req->enc)
1135                         req->type |=
1136                                 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
1137                 else
1138                         req->type |=
1139                                 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
1140         }
1141
1142         cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type;
1143
1144         /*
1145          * Map the packets, first we check if the data fits into a single
1146          * sg entry and use that if possible. If it does not fit, we check
1147          * if we need to do sg_split to align the scatterlist data on the
1148          * actual data size being processed by the crypto engine.
1149          */
1150         src = req->src;
1151         sg_nents = sg_nents_for_len(src, req->size);
1152
1153         split_size = req->size;
1154
1155         mapped_sg = &rxd->mapped_sg[0];
1156         if (sg_nents == 1 && split_size <= req->src->length) {
1157                 src = &mapped_sg->static_sg;
1158                 src_nents = 1;
1159                 sg_init_table(src, 1);
1160                 sg_set_page(src, sg_page(req->src), split_size,
1161                             req->src->offset);
1162
1163                 mapped_sg->sgt.sgl = src;
1164                 mapped_sg->sgt.orig_nents = src_nents;
1165                 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1166                 if (ret) {
1167                         kfree(rxd);
1168                         return ret;
1169                 }
1170
1171                 mapped_sg->dir = dir_src;
1172                 mapped_sg->mapped = true;
1173         } else {
1174                 mapped_sg->sgt.sgl = req->src;
1175                 mapped_sg->sgt.orig_nents = sg_nents;
1176                 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1177                 if (ret) {
1178                         kfree(rxd);
1179                         return ret;
1180                 }
1181
1182                 mapped_sg->dir = dir_src;
1183                 mapped_sg->mapped = true;
1184
1185                 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
1186                                &split_size, &src, &src_nents, gfp_flags);
1187                 if (ret) {
1188                         src_nents = mapped_sg->sgt.nents;
1189                         src = mapped_sg->sgt.sgl;
1190                 } else {
1191                         mapped_sg->split_sg = src;
1192                 }
1193         }
1194
1195         dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
1196
1197         if (!diff_dst) {
1198                 dst_nents = src_nents;
1199                 dst = src;
1200         } else {
1201                 dst_nents = sg_nents_for_len(req->dst, req->size);
1202                 mapped_sg = &rxd->mapped_sg[1];
1203
1204                 if (dst_nents == 1 && split_size <= req->dst->length) {
1205                         dst = &mapped_sg->static_sg;
1206                         dst_nents = 1;
1207                         sg_init_table(dst, 1);
1208                         sg_set_page(dst, sg_page(req->dst), split_size,
1209                                     req->dst->offset);
1210
1211                         mapped_sg->sgt.sgl = dst;
1212                         mapped_sg->sgt.orig_nents = dst_nents;
1213                         ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
1214                                               DMA_FROM_DEVICE, 0);
1215                         if (ret)
1216                                 goto err_cleanup;
1217
1218                         mapped_sg->dir = DMA_FROM_DEVICE;
1219                         mapped_sg->mapped = true;
1220                 } else {
1221                         mapped_sg->sgt.sgl = req->dst;
1222                         mapped_sg->sgt.orig_nents = dst_nents;
1223                         ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
1224                                               DMA_FROM_DEVICE, 0);
1225                         if (ret)
1226                                 goto err_cleanup;
1227
1228                         mapped_sg->dir = DMA_FROM_DEVICE;
1229                         mapped_sg->mapped = true;
1230
1231                         ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
1232                                        0, 1, &split_size, &dst, &dst_nents,
1233                                        gfp_flags);
1234                         if (ret) {
1235                                 dst_nents = mapped_sg->sgt.nents;
1236                                 dst = mapped_sg->sgt.sgl;
1237                         } else {
1238                                 mapped_sg->split_sg = dst;
1239                         }
1240                 }
1241         }
1242
1243         rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
1244                                              DMA_DEV_TO_MEM,
1245                                              DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1246         if (!rxd->tx_in) {
1247                 dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
1248                 ret = -EINVAL;
1249                 goto err_cleanup;
1250         }
1251
1252         rxd->req = (void *)req->base;
1253         rxd->enc = req->enc;
1254         rxd->iv_idx = req->ctx->iv_idx;
1255         rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
1256         rxd->tx_in->callback = req->callback;
1257         rxd->tx_in->callback_param = rxd;
1258
1259         tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src,
1260                                          src_nents, DMA_MEM_TO_DEV,
1261                                          DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1262
1263         if (!tx_out) {
1264                 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
1265                 ret = -EINVAL;
1266                 goto err_cleanup;
1267         }
1268
1269         /*
1270          * Prepare metadata for DMA engine. This essentially describes the
1271          * crypto algorithm to be used, data sizes, different keys etc.
1272          */
1273         mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
1274
1275         sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
1276                                    sizeof(u32))), cmdl, sizeof(sa_ctx->epib),
1277                            sa_ctx->epib);
1278
1279         ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
1280         dmaengine_desc_set_metadata_len(tx_out, req->mdata_size);
1281
1282         dmaengine_submit(tx_out);
1283         dmaengine_submit(rxd->tx_in);
1284
1285         dma_async_issue_pending(dma_rx);
1286         dma_async_issue_pending(pdata->dma_tx);
1287
1288         return -EINPROGRESS;
1289
1290 err_cleanup:
1291         sa_free_sa_rx_data(rxd);
1292
1293         return ret;
1294 }
1295
1296 static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
1297 {
1298         struct sa_tfm_ctx *ctx =
1299             crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1300         struct crypto_alg *alg = req->base.tfm->__crt_alg;
1301         struct sa_req sa_req = { 0 };
1302
1303         if (!req->cryptlen)
1304                 return 0;
1305
1306         if (req->cryptlen % alg->cra_blocksize)
1307                 return -EINVAL;
1308
1309         /* Use SW fallback if the data size is not supported */
1310         if (req->cryptlen > SA_MAX_DATA_SZ ||
1311             (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN &&
1312              req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) {
1313                 struct skcipher_request *subreq = skcipher_request_ctx(req);
1314
1315                 skcipher_request_set_tfm(subreq, ctx->fallback.skcipher);
1316                 skcipher_request_set_callback(subreq, req->base.flags,
1317                                               req->base.complete,
1318                                               req->base.data);
1319                 skcipher_request_set_crypt(subreq, req->src, req->dst,
1320                                            req->cryptlen, req->iv);
1321                 if (enc)
1322                         return crypto_skcipher_encrypt(subreq);
1323                 else
1324                         return crypto_skcipher_decrypt(subreq);
1325         }
1326
1327         sa_req.size = req->cryptlen;
1328         sa_req.enc_size = req->cryptlen;
1329         sa_req.src = req->src;
1330         sa_req.dst = req->dst;
1331         sa_req.enc_iv = iv;
1332         sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
1333         sa_req.enc = enc;
1334         sa_req.callback = sa_aes_dma_in_callback;
1335         sa_req.mdata_size = 44;
1336         sa_req.base = &req->base;
1337         sa_req.ctx = ctx;
1338
1339         return sa_run(&sa_req);
1340 }
1341
1342 static int sa_encrypt(struct skcipher_request *req)
1343 {
1344         return sa_cipher_run(req, req->iv, 1);
1345 }
1346
1347 static int sa_decrypt(struct skcipher_request *req)
1348 {
1349         return sa_cipher_run(req, req->iv, 0);
1350 }
1351
1352 static void sa_sha_dma_in_callback(void *data)
1353 {
1354         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1355         struct ahash_request *req;
1356         struct crypto_ahash *tfm;
1357         unsigned int authsize;
1358         int i;
1359         size_t ml, pl;
1360         u32 *result;
1361         __be32 *mdptr;
1362
1363         sa_sync_from_device(rxd);
1364         req = container_of(rxd->req, struct ahash_request, base);
1365         tfm = crypto_ahash_reqtfm(req);
1366         authsize = crypto_ahash_digestsize(tfm);
1367
1368         mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1369         result = (u32 *)req->result;
1370
1371         for (i = 0; i < (authsize / 4); i++)
1372                 result[i] = be32_to_cpu(mdptr[i + 4]);
1373
1374         sa_free_sa_rx_data(rxd);
1375
1376         ahash_request_complete(req, 0);
1377 }
1378
1379 static int zero_message_process(struct ahash_request *req)
1380 {
1381         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1382         int sa_digest_size = crypto_ahash_digestsize(tfm);
1383
1384         switch (sa_digest_size) {
1385         case SHA1_DIGEST_SIZE:
1386                 memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
1387                 break;
1388         case SHA256_DIGEST_SIZE:
1389                 memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
1390                 break;
1391         case SHA512_DIGEST_SIZE:
1392                 memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
1393                 break;
1394         default:
1395                 return -EINVAL;
1396         }
1397
1398         return 0;
1399 }
1400
1401 static int sa_sha_run(struct ahash_request *req)
1402 {
1403         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1404         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1405         struct sa_req sa_req = { 0 };
1406         size_t auth_len;
1407
1408         auth_len = req->nbytes;
1409
1410         if (!auth_len)
1411                 return zero_message_process(req);
1412
1413         if (auth_len > SA_MAX_DATA_SZ ||
1414             (auth_len >= SA_UNSAFE_DATA_SZ_MIN &&
1415              auth_len <= SA_UNSAFE_DATA_SZ_MAX)) {
1416                 struct ahash_request *subreq = &rctx->fallback_req;
1417                 int ret = 0;
1418
1419                 ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1420                 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1421
1422                 crypto_ahash_init(subreq);
1423
1424                 subreq->nbytes = auth_len;
1425                 subreq->src = req->src;
1426                 subreq->result = req->result;
1427
1428                 ret |= crypto_ahash_update(subreq);
1429
1430                 subreq->nbytes = 0;
1431
1432                 ret |= crypto_ahash_final(subreq);
1433
1434                 return ret;
1435         }
1436
1437         sa_req.size = auth_len;
1438         sa_req.auth_size = auth_len;
1439         sa_req.src = req->src;
1440         sa_req.dst = req->src;
1441         sa_req.enc = true;
1442         sa_req.type = CRYPTO_ALG_TYPE_AHASH;
1443         sa_req.callback = sa_sha_dma_in_callback;
1444         sa_req.mdata_size = 28;
1445         sa_req.ctx = ctx;
1446         sa_req.base = &req->base;
1447
1448         return sa_run(&sa_req);
1449 }
1450
1451 static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct  algo_data *ad)
1452 {
1453         int bs = crypto_shash_blocksize(ctx->shash);
1454         int cmdl_len;
1455         struct sa_cmdl_cfg cfg;
1456
1457         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1458         ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1459         ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1460
1461         memset(ctx->authkey, 0, bs);
1462         memset(&cfg, 0, sizeof(cfg));
1463         cfg.aalg = ad->aalg_id;
1464         cfg.enc_eng_id = ad->enc_eng.eng_id;
1465         cfg.auth_eng_id = ad->auth_eng.eng_id;
1466         cfg.iv_size = 0;
1467         cfg.akey = NULL;
1468         cfg.akey_len = 0;
1469
1470         ctx->dev_data = dev_get_drvdata(sa_k3_dev);
1471         /* Setup Encryption Security Context & Command label template */
1472         if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, NULL, 0, NULL, 0,
1473                        ad, 0, &ctx->enc.epib[1]))
1474                 goto badkey;
1475
1476         cmdl_len = sa_format_cmdl_gen(&cfg,
1477                                       (u8 *)ctx->enc.cmdl,
1478                                       &ctx->enc.cmdl_upd_info);
1479         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1480                 goto badkey;
1481
1482         ctx->enc.cmdl_size = cmdl_len;
1483
1484         return 0;
1485
1486 badkey:
1487         dev_err(sa_k3_dev, "%s: badkey\n", __func__);
1488         return -EINVAL;
1489 }
1490
1491 static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
1492 {
1493         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1494         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1495         int ret;
1496
1497         memset(ctx, 0, sizeof(*ctx));
1498         ctx->dev_data = data;
1499         ret = sa_init_ctx_info(&ctx->enc, data);
1500         if (ret)
1501                 return ret;
1502
1503         if (alg_base) {
1504                 ctx->shash = crypto_alloc_shash(alg_base, 0,
1505                                                 CRYPTO_ALG_NEED_FALLBACK);
1506                 if (IS_ERR(ctx->shash)) {
1507                         dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n",
1508                                 alg_base);
1509                         return PTR_ERR(ctx->shash);
1510                 }
1511                 /* for fallback */
1512                 ctx->fallback.ahash =
1513                         crypto_alloc_ahash(alg_base, 0,
1514                                            CRYPTO_ALG_NEED_FALLBACK);
1515                 if (IS_ERR(ctx->fallback.ahash)) {
1516                         dev_err(ctx->dev_data->dev,
1517                                 "Could not load fallback driver\n");
1518                         return PTR_ERR(ctx->fallback.ahash);
1519                 }
1520         }
1521
1522         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1523                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1524                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1525
1526         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1527                                  sizeof(struct sa_sha_req_ctx) +
1528                                  crypto_ahash_reqsize(ctx->fallback.ahash));
1529
1530         return 0;
1531 }
1532
1533 static int sa_sha_digest(struct ahash_request *req)
1534 {
1535         return sa_sha_run(req);
1536 }
1537
1538 static int sa_sha_init(struct ahash_request *req)
1539 {
1540         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1541         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1542         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1543
1544         dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
1545                 crypto_ahash_digestsize(tfm), rctx);
1546
1547         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1548         rctx->fallback_req.base.flags =
1549                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1550
1551         return crypto_ahash_init(&rctx->fallback_req);
1552 }
1553
1554 static int sa_sha_update(struct ahash_request *req)
1555 {
1556         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1557         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1558         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1559
1560         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1561         rctx->fallback_req.base.flags =
1562                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1563         rctx->fallback_req.nbytes = req->nbytes;
1564         rctx->fallback_req.src = req->src;
1565
1566         return crypto_ahash_update(&rctx->fallback_req);
1567 }
1568
1569 static int sa_sha_final(struct ahash_request *req)
1570 {
1571         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1572         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1573         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1574
1575         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1576         rctx->fallback_req.base.flags =
1577                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1578         rctx->fallback_req.result = req->result;
1579
1580         return crypto_ahash_final(&rctx->fallback_req);
1581 }
1582
1583 static int sa_sha_finup(struct ahash_request *req)
1584 {
1585         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1586         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1587         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1588
1589         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1590         rctx->fallback_req.base.flags =
1591                 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1592
1593         rctx->fallback_req.nbytes = req->nbytes;
1594         rctx->fallback_req.src = req->src;
1595         rctx->fallback_req.result = req->result;
1596
1597         return crypto_ahash_finup(&rctx->fallback_req);
1598 }
1599
1600 static int sa_sha_import(struct ahash_request *req, const void *in)
1601 {
1602         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1603         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1604         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1605
1606         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1607         rctx->fallback_req.base.flags = req->base.flags &
1608                 CRYPTO_TFM_REQ_MAY_SLEEP;
1609
1610         return crypto_ahash_import(&rctx->fallback_req, in);
1611 }
1612
1613 static int sa_sha_export(struct ahash_request *req, void *out)
1614 {
1615         struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1616         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1617         struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1618         struct ahash_request *subreq = &rctx->fallback_req;
1619
1620         ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1621         subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1622
1623         return crypto_ahash_export(subreq, out);
1624 }
1625
1626 static int sa_sha1_cra_init(struct crypto_tfm *tfm)
1627 {
1628         struct algo_data ad = { 0 };
1629         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1630
1631         sa_sha_cra_init_alg(tfm, "sha1");
1632
1633         ad.aalg_id = SA_AALG_ID_SHA1;
1634         ad.hash_size = SHA1_DIGEST_SIZE;
1635         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1636
1637         sa_sha_setup(ctx, &ad);
1638
1639         return 0;
1640 }
1641
1642 static int sa_sha256_cra_init(struct crypto_tfm *tfm)
1643 {
1644         struct algo_data ad = { 0 };
1645         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1646
1647         sa_sha_cra_init_alg(tfm, "sha256");
1648
1649         ad.aalg_id = SA_AALG_ID_SHA2_256;
1650         ad.hash_size = SHA256_DIGEST_SIZE;
1651         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1652
1653         sa_sha_setup(ctx, &ad);
1654
1655         return 0;
1656 }
1657
1658 static int sa_sha512_cra_init(struct crypto_tfm *tfm)
1659 {
1660         struct algo_data ad = { 0 };
1661         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1662
1663         sa_sha_cra_init_alg(tfm, "sha512");
1664
1665         ad.aalg_id = SA_AALG_ID_SHA2_512;
1666         ad.hash_size = SHA512_DIGEST_SIZE;
1667         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512;
1668
1669         sa_sha_setup(ctx, &ad);
1670
1671         return 0;
1672 }
1673
1674 static void sa_sha_cra_exit(struct crypto_tfm *tfm)
1675 {
1676         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1677         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1678
1679         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1680                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1681                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1682
1683         if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH)
1684                 sa_free_ctx_info(&ctx->enc, data);
1685
1686         crypto_free_shash(ctx->shash);
1687         crypto_free_ahash(ctx->fallback.ahash);
1688 }
1689
1690 static void sa_aead_dma_in_callback(void *data)
1691 {
1692         struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1693         struct aead_request *req;
1694         struct crypto_aead *tfm;
1695         unsigned int start;
1696         unsigned int authsize;
1697         u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
1698         size_t pl, ml;
1699         int i;
1700         int err = 0;
1701         u16 auth_len;
1702         u32 *mdptr;
1703
1704         sa_sync_from_device(rxd);
1705         req = container_of(rxd->req, struct aead_request, base);
1706         tfm = crypto_aead_reqtfm(req);
1707         start = req->assoclen + req->cryptlen;
1708         authsize = crypto_aead_authsize(tfm);
1709
1710         mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1711         for (i = 0; i < (authsize / 4); i++)
1712                 mdptr[i + 4] = swab32(mdptr[i + 4]);
1713
1714         auth_len = req->assoclen + req->cryptlen;
1715
1716         if (rxd->enc) {
1717                 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1718                                          1);
1719         } else {
1720                 auth_len -= authsize;
1721                 start -= authsize;
1722                 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
1723                                          0);
1724
1725                 err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
1726         }
1727
1728         sa_free_sa_rx_data(rxd);
1729
1730         aead_request_complete(req, err);
1731 }
1732
1733 static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash,
1734                             const char *fallback)
1735 {
1736         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1737         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1738         int ret;
1739
1740         memzero_explicit(ctx, sizeof(*ctx));
1741         ctx->dev_data = data;
1742
1743         ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK);
1744         if (IS_ERR(ctx->shash)) {
1745                 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash);
1746                 return PTR_ERR(ctx->shash);
1747         }
1748
1749         ctx->fallback.aead = crypto_alloc_aead(fallback, 0,
1750                                                CRYPTO_ALG_NEED_FALLBACK);
1751
1752         if (IS_ERR(ctx->fallback.aead)) {
1753                 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n",
1754                         fallback);
1755                 return PTR_ERR(ctx->fallback.aead);
1756         }
1757
1758         crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
1759                                 crypto_aead_reqsize(ctx->fallback.aead));
1760
1761         ret = sa_init_ctx_info(&ctx->enc, data);
1762         if (ret)
1763                 return ret;
1764
1765         ret = sa_init_ctx_info(&ctx->dec, data);
1766         if (ret) {
1767                 sa_free_ctx_info(&ctx->enc, data);
1768                 return ret;
1769         }
1770
1771         dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1772                 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1773                 ctx->dec.sc_id, &ctx->dec.sc_phys);
1774
1775         return ret;
1776 }
1777
1778 static int sa_cra_init_aead_sha1(struct crypto_aead *tfm)
1779 {
1780         return sa_cra_init_aead(tfm, "sha1",
1781                                 "authenc(hmac(sha1-ce),cbc(aes-ce))");
1782 }
1783
1784 static int sa_cra_init_aead_sha256(struct crypto_aead *tfm)
1785 {
1786         return sa_cra_init_aead(tfm, "sha256",
1787                                 "authenc(hmac(sha256-ce),cbc(aes-ce))");
1788 }
1789
1790 static void sa_exit_tfm_aead(struct crypto_aead *tfm)
1791 {
1792         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1793         struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1794
1795         crypto_free_shash(ctx->shash);
1796         crypto_free_aead(ctx->fallback.aead);
1797
1798         sa_free_ctx_info(&ctx->enc, data);
1799         sa_free_ctx_info(&ctx->dec, data);
1800 }
1801
1802 /* AEAD algorithm configuration interface function */
1803 static int sa_aead_setkey(struct crypto_aead *authenc,
1804                           const u8 *key, unsigned int keylen,
1805                           struct algo_data *ad)
1806 {
1807         struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
1808         struct crypto_authenc_keys keys;
1809         int cmdl_len;
1810         struct sa_cmdl_cfg cfg;
1811         int key_idx;
1812
1813         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1814                 return -EINVAL;
1815
1816         /* Convert the key size (16/24/32) to the key size index (0/1/2) */
1817         key_idx = (keys.enckeylen >> 3) - 2;
1818         if (key_idx >= 3)
1819                 return -EINVAL;
1820
1821         ad->ctx = ctx;
1822         ad->enc_eng.eng_id = SA_ENG_ID_EM1;
1823         ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1824         ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1825         ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1826         ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx];
1827         ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx];
1828         ad->inv_key = true;
1829         ad->keyed_mac = true;
1830         ad->ealg_id = SA_EALG_ID_AES_CBC;
1831         ad->prep_iopad = sa_prepare_iopads;
1832
1833         memset(&cfg, 0, sizeof(cfg));
1834         cfg.enc = true;
1835         cfg.aalg = ad->aalg_id;
1836         cfg.enc_eng_id = ad->enc_eng.eng_id;
1837         cfg.auth_eng_id = ad->auth_eng.eng_id;
1838         cfg.iv_size = crypto_aead_ivsize(authenc);
1839         cfg.akey = keys.authkey;
1840         cfg.akey_len = keys.authkeylen;
1841
1842         /* Setup Encryption Security Context & Command label template */
1843         if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, keys.enckey,
1844                        keys.enckeylen, keys.authkey, keys.authkeylen,
1845                        ad, 1, &ctx->enc.epib[1]))
1846                 return -EINVAL;
1847
1848         cmdl_len = sa_format_cmdl_gen(&cfg,
1849                                       (u8 *)ctx->enc.cmdl,
1850                                       &ctx->enc.cmdl_upd_info);
1851         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1852                 return -EINVAL;
1853
1854         ctx->enc.cmdl_size = cmdl_len;
1855
1856         /* Setup Decryption Security Context & Command label template */
1857         if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, keys.enckey,
1858                        keys.enckeylen, keys.authkey, keys.authkeylen,
1859                        ad, 0, &ctx->dec.epib[1]))
1860                 return -EINVAL;
1861
1862         cfg.enc = false;
1863         cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
1864                                       &ctx->dec.cmdl_upd_info);
1865
1866         if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1867                 return -EINVAL;
1868
1869         ctx->dec.cmdl_size = cmdl_len;
1870
1871         crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK);
1872         crypto_aead_set_flags(ctx->fallback.aead,
1873                               crypto_aead_get_flags(authenc) &
1874                               CRYPTO_TFM_REQ_MASK);
1875         crypto_aead_setkey(ctx->fallback.aead, key, keylen);
1876
1877         return 0;
1878 }
1879
1880 static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1881 {
1882         struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
1883
1884         return crypto_aead_setauthsize(ctx->fallback.aead, authsize);
1885 }
1886
1887 static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
1888                                    const u8 *key, unsigned int keylen)
1889 {
1890         struct algo_data ad = { 0 };
1891
1892         ad.ealg_id = SA_EALG_ID_AES_CBC;
1893         ad.aalg_id = SA_AALG_ID_HMAC_SHA1;
1894         ad.hash_size = SHA1_DIGEST_SIZE;
1895         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1896
1897         return sa_aead_setkey(authenc, key, keylen, &ad);
1898 }
1899
1900 static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
1901                                      const u8 *key, unsigned int keylen)
1902 {
1903         struct algo_data ad = { 0 };
1904
1905         ad.ealg_id = SA_EALG_ID_AES_CBC;
1906         ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256;
1907         ad.hash_size = SHA256_DIGEST_SIZE;
1908         ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1909
1910         return sa_aead_setkey(authenc, key, keylen, &ad);
1911 }
1912
1913 static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
1914 {
1915         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1916         struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1917         struct sa_req sa_req = { 0 };
1918         size_t auth_size, enc_size;
1919
1920         enc_size = req->cryptlen;
1921         auth_size = req->assoclen + req->cryptlen;
1922
1923         if (!enc) {
1924                 enc_size -= crypto_aead_authsize(tfm);
1925                 auth_size -= crypto_aead_authsize(tfm);
1926         }
1927
1928         if (auth_size > SA_MAX_DATA_SZ ||
1929             (auth_size >= SA_UNSAFE_DATA_SZ_MIN &&
1930              auth_size <= SA_UNSAFE_DATA_SZ_MAX)) {
1931                 struct aead_request *subreq = aead_request_ctx(req);
1932                 int ret;
1933
1934                 aead_request_set_tfm(subreq, ctx->fallback.aead);
1935                 aead_request_set_callback(subreq, req->base.flags,
1936                                           req->base.complete, req->base.data);
1937                 aead_request_set_crypt(subreq, req->src, req->dst,
1938                                        req->cryptlen, req->iv);
1939                 aead_request_set_ad(subreq, req->assoclen);
1940
1941                 ret = enc ? crypto_aead_encrypt(subreq) :
1942                         crypto_aead_decrypt(subreq);
1943                 return ret;
1944         }
1945
1946         sa_req.enc_offset = req->assoclen;
1947         sa_req.enc_size = enc_size;
1948         sa_req.auth_size = auth_size;
1949         sa_req.size = auth_size;
1950         sa_req.enc_iv = iv;
1951         sa_req.type = CRYPTO_ALG_TYPE_AEAD;
1952         sa_req.enc = enc;
1953         sa_req.callback = sa_aead_dma_in_callback;
1954         sa_req.mdata_size = 52;
1955         sa_req.base = &req->base;
1956         sa_req.ctx = ctx;
1957         sa_req.src = req->src;
1958         sa_req.dst = req->dst;
1959
1960         return sa_run(&sa_req);
1961 }
1962
1963 /* AEAD algorithm encrypt interface function */
1964 static int sa_aead_encrypt(struct aead_request *req)
1965 {
1966         return sa_aead_run(req, req->iv, 1);
1967 }
1968
1969 /* AEAD algorithm decrypt interface function */
1970 static int sa_aead_decrypt(struct aead_request *req)
1971 {
1972         return sa_aead_run(req, req->iv, 0);
1973 }
1974
1975 static struct sa_alg_tmpl sa_algs[] = {
1976         [SA_ALG_CBC_AES] = {
1977                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1978                 .alg.skcipher = {
1979                         .base.cra_name          = "cbc(aes)",
1980                         .base.cra_driver_name   = "cbc-aes-sa2ul",
1981                         .base.cra_priority      = 30000,
1982                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
1983                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
1984                                                   CRYPTO_ALG_ASYNC |
1985                                                   CRYPTO_ALG_NEED_FALLBACK,
1986                         .base.cra_blocksize     = AES_BLOCK_SIZE,
1987                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
1988                         .base.cra_module        = THIS_MODULE,
1989                         .init                   = sa_cipher_cra_init,
1990                         .exit                   = sa_cipher_cra_exit,
1991                         .min_keysize            = AES_MIN_KEY_SIZE,
1992                         .max_keysize            = AES_MAX_KEY_SIZE,
1993                         .ivsize                 = AES_BLOCK_SIZE,
1994                         .setkey                 = sa_aes_cbc_setkey,
1995                         .encrypt                = sa_encrypt,
1996                         .decrypt                = sa_decrypt,
1997                 }
1998         },
1999         [SA_ALG_EBC_AES] = {
2000                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2001                 .alg.skcipher = {
2002                         .base.cra_name          = "ecb(aes)",
2003                         .base.cra_driver_name   = "ecb-aes-sa2ul",
2004                         .base.cra_priority      = 30000,
2005                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
2006                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2007                                                   CRYPTO_ALG_ASYNC |
2008                                                   CRYPTO_ALG_NEED_FALLBACK,
2009                         .base.cra_blocksize     = AES_BLOCK_SIZE,
2010                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
2011                         .base.cra_module        = THIS_MODULE,
2012                         .init                   = sa_cipher_cra_init,
2013                         .exit                   = sa_cipher_cra_exit,
2014                         .min_keysize            = AES_MIN_KEY_SIZE,
2015                         .max_keysize            = AES_MAX_KEY_SIZE,
2016                         .setkey                 = sa_aes_ecb_setkey,
2017                         .encrypt                = sa_encrypt,
2018                         .decrypt                = sa_decrypt,
2019                 }
2020         },
2021         [SA_ALG_CBC_DES3] = {
2022                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2023                 .alg.skcipher = {
2024                         .base.cra_name          = "cbc(des3_ede)",
2025                         .base.cra_driver_name   = "cbc-des3-sa2ul",
2026                         .base.cra_priority      = 30000,
2027                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
2028                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2029                                                   CRYPTO_ALG_ASYNC |
2030                                                   CRYPTO_ALG_NEED_FALLBACK,
2031                         .base.cra_blocksize     = DES_BLOCK_SIZE,
2032                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
2033                         .base.cra_module        = THIS_MODULE,
2034                         .init                   = sa_cipher_cra_init,
2035                         .exit                   = sa_cipher_cra_exit,
2036                         .min_keysize            = 3 * DES_KEY_SIZE,
2037                         .max_keysize            = 3 * DES_KEY_SIZE,
2038                         .ivsize                 = DES_BLOCK_SIZE,
2039                         .setkey                 = sa_3des_cbc_setkey,
2040                         .encrypt                = sa_encrypt,
2041                         .decrypt                = sa_decrypt,
2042                 }
2043         },
2044         [SA_ALG_ECB_DES3] = {
2045                 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2046                 .alg.skcipher = {
2047                         .base.cra_name          = "ecb(des3_ede)",
2048                         .base.cra_driver_name   = "ecb-des3-sa2ul",
2049                         .base.cra_priority      = 30000,
2050                         .base.cra_flags         = CRYPTO_ALG_TYPE_SKCIPHER |
2051                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2052                                                   CRYPTO_ALG_ASYNC |
2053                                                   CRYPTO_ALG_NEED_FALLBACK,
2054                         .base.cra_blocksize     = DES_BLOCK_SIZE,
2055                         .base.cra_ctxsize       = sizeof(struct sa_tfm_ctx),
2056                         .base.cra_module        = THIS_MODULE,
2057                         .init                   = sa_cipher_cra_init,
2058                         .exit                   = sa_cipher_cra_exit,
2059                         .min_keysize            = 3 * DES_KEY_SIZE,
2060                         .max_keysize            = 3 * DES_KEY_SIZE,
2061                         .setkey                 = sa_3des_ecb_setkey,
2062                         .encrypt                = sa_encrypt,
2063                         .decrypt                = sa_decrypt,
2064                 }
2065         },
2066         [SA_ALG_SHA1] = {
2067                 .type = CRYPTO_ALG_TYPE_AHASH,
2068                 .alg.ahash = {
2069                         .halg.base = {
2070                                 .cra_name       = "sha1",
2071                                 .cra_driver_name        = "sha1-sa2ul",
2072                                 .cra_priority   = 400,
2073                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2074                                                   CRYPTO_ALG_ASYNC |
2075                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2076                                                   CRYPTO_ALG_NEED_FALLBACK,
2077                                 .cra_blocksize  = SHA1_BLOCK_SIZE,
2078                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2079                                 .cra_module     = THIS_MODULE,
2080                                 .cra_init       = sa_sha1_cra_init,
2081                                 .cra_exit       = sa_sha_cra_exit,
2082                         },
2083                         .halg.digestsize        = SHA1_DIGEST_SIZE,
2084                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2085                                                   sizeof(struct sha1_state),
2086                         .init                   = sa_sha_init,
2087                         .update                 = sa_sha_update,
2088                         .final                  = sa_sha_final,
2089                         .finup                  = sa_sha_finup,
2090                         .digest                 = sa_sha_digest,
2091                         .export                 = sa_sha_export,
2092                         .import                 = sa_sha_import,
2093                 },
2094         },
2095         [SA_ALG_SHA256] = {
2096                 .type = CRYPTO_ALG_TYPE_AHASH,
2097                 .alg.ahash = {
2098                         .halg.base = {
2099                                 .cra_name       = "sha256",
2100                                 .cra_driver_name        = "sha256-sa2ul",
2101                                 .cra_priority   = 400,
2102                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2103                                                   CRYPTO_ALG_ASYNC |
2104                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2105                                                   CRYPTO_ALG_NEED_FALLBACK,
2106                                 .cra_blocksize  = SHA256_BLOCK_SIZE,
2107                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2108                                 .cra_module     = THIS_MODULE,
2109                                 .cra_init       = sa_sha256_cra_init,
2110                                 .cra_exit       = sa_sha_cra_exit,
2111                         },
2112                         .halg.digestsize        = SHA256_DIGEST_SIZE,
2113                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2114                                                   sizeof(struct sha256_state),
2115                         .init                   = sa_sha_init,
2116                         .update                 = sa_sha_update,
2117                         .final                  = sa_sha_final,
2118                         .finup                  = sa_sha_finup,
2119                         .digest                 = sa_sha_digest,
2120                         .export                 = sa_sha_export,
2121                         .import                 = sa_sha_import,
2122                 },
2123         },
2124         [SA_ALG_SHA512] = {
2125                 .type = CRYPTO_ALG_TYPE_AHASH,
2126                 .alg.ahash = {
2127                         .halg.base = {
2128                                 .cra_name       = "sha512",
2129                                 .cra_driver_name        = "sha512-sa2ul",
2130                                 .cra_priority   = 400,
2131                                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH |
2132                                                   CRYPTO_ALG_ASYNC |
2133                                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
2134                                                   CRYPTO_ALG_NEED_FALLBACK,
2135                                 .cra_blocksize  = SHA512_BLOCK_SIZE,
2136                                 .cra_ctxsize    = sizeof(struct sa_tfm_ctx),
2137                                 .cra_module     = THIS_MODULE,
2138                                 .cra_init       = sa_sha512_cra_init,
2139                                 .cra_exit       = sa_sha_cra_exit,
2140                         },
2141                         .halg.digestsize        = SHA512_DIGEST_SIZE,
2142                         .halg.statesize         = sizeof(struct sa_sha_req_ctx) +
2143                                                   sizeof(struct sha512_state),
2144                         .init                   = sa_sha_init,
2145                         .update                 = sa_sha_update,
2146                         .final                  = sa_sha_final,
2147                         .finup                  = sa_sha_finup,
2148                         .digest                 = sa_sha_digest,
2149                         .export                 = sa_sha_export,
2150                         .import                 = sa_sha_import,
2151                 },
2152         },
2153         [SA_ALG_AUTHENC_SHA1_AES] = {
2154                 .type   = CRYPTO_ALG_TYPE_AEAD,
2155                 .alg.aead = {
2156                         .base = {
2157                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2158                                 .cra_driver_name =
2159                                         "authenc(hmac(sha1),cbc(aes))-sa2ul",
2160                                 .cra_blocksize = AES_BLOCK_SIZE,
2161                                 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2162                                         CRYPTO_ALG_KERN_DRIVER_ONLY |
2163                                         CRYPTO_ALG_ASYNC |
2164                                         CRYPTO_ALG_NEED_FALLBACK,
2165                                 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2166                                 .cra_module = THIS_MODULE,
2167                                 .cra_priority = 3000,
2168                         },
2169                         .ivsize = AES_BLOCK_SIZE,
2170                         .maxauthsize = SHA1_DIGEST_SIZE,
2171
2172                         .init = sa_cra_init_aead_sha1,
2173                         .exit = sa_exit_tfm_aead,
2174                         .setkey = sa_aead_cbc_sha1_setkey,
2175                         .setauthsize = sa_aead_setauthsize,
2176                         .encrypt = sa_aead_encrypt,
2177                         .decrypt = sa_aead_decrypt,
2178                 },
2179         },
2180         [SA_ALG_AUTHENC_SHA256_AES] = {
2181                 .type   = CRYPTO_ALG_TYPE_AEAD,
2182                 .alg.aead = {
2183                         .base = {
2184                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2185                                 .cra_driver_name =
2186                                         "authenc(hmac(sha256),cbc(aes))-sa2ul",
2187                                 .cra_blocksize = AES_BLOCK_SIZE,
2188                                 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2189                                         CRYPTO_ALG_KERN_DRIVER_ONLY |
2190                                         CRYPTO_ALG_ASYNC |
2191                                         CRYPTO_ALG_NEED_FALLBACK,
2192                                 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2193                                 .cra_module = THIS_MODULE,
2194                                 .cra_alignmask = 0,
2195                                 .cra_priority = 3000,
2196                         },
2197                         .ivsize = AES_BLOCK_SIZE,
2198                         .maxauthsize = SHA256_DIGEST_SIZE,
2199
2200                         .init = sa_cra_init_aead_sha256,
2201                         .exit = sa_exit_tfm_aead,
2202                         .setkey = sa_aead_cbc_sha256_setkey,
2203                         .setauthsize = sa_aead_setauthsize,
2204                         .encrypt = sa_aead_encrypt,
2205                         .decrypt = sa_aead_decrypt,
2206                 },
2207         },
2208 };
2209
2210 /* Register the algorithms in crypto framework */
2211 static void sa_register_algos(struct sa_crypto_data *dev_data)
2212 {
2213         const struct sa_match_data *match_data = dev_data->match_data;
2214         struct device *dev = dev_data->dev;
2215         char *alg_name;
2216         u32 type;
2217         int i, err;
2218
2219         for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2220                 /* Skip unsupported algos */
2221                 if (!(match_data->supported_algos & BIT(i)))
2222                         continue;
2223
2224                 type = sa_algs[i].type;
2225                 if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
2226                         alg_name = sa_algs[i].alg.skcipher.base.cra_name;
2227                         err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
2228                 } else if (type == CRYPTO_ALG_TYPE_AHASH) {
2229                         alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
2230                         err = crypto_register_ahash(&sa_algs[i].alg.ahash);
2231                 } else if (type == CRYPTO_ALG_TYPE_AEAD) {
2232                         alg_name = sa_algs[i].alg.aead.base.cra_name;
2233                         err = crypto_register_aead(&sa_algs[i].alg.aead);
2234                 } else {
2235                         dev_err(dev,
2236                                 "un-supported crypto algorithm (%d)",
2237                                 sa_algs[i].type);
2238                         continue;
2239                 }
2240
2241                 if (err)
2242                         dev_err(dev, "Failed to register '%s'\n", alg_name);
2243                 else
2244                         sa_algs[i].registered = true;
2245         }
2246 }
2247
2248 /* Unregister the algorithms in crypto framework */
2249 static void sa_unregister_algos(const struct device *dev)
2250 {
2251         u32 type;
2252         int i;
2253
2254         for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2255                 type = sa_algs[i].type;
2256                 if (!sa_algs[i].registered)
2257                         continue;
2258                 if (type == CRYPTO_ALG_TYPE_SKCIPHER)
2259                         crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
2260                 else if (type == CRYPTO_ALG_TYPE_AHASH)
2261                         crypto_unregister_ahash(&sa_algs[i].alg.ahash);
2262                 else if (type == CRYPTO_ALG_TYPE_AEAD)
2263                         crypto_unregister_aead(&sa_algs[i].alg.aead);
2264
2265                 sa_algs[i].registered = false;
2266         }
2267 }
2268
2269 static int sa_init_mem(struct sa_crypto_data *dev_data)
2270 {
2271         struct device *dev = &dev_data->pdev->dev;
2272         /* Setup dma pool for security context buffers */
2273         dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
2274                                             SA_CTX_MAX_SZ, 64, 0);
2275         if (!dev_data->sc_pool) {
2276                 dev_err(dev, "Failed to create dma pool");
2277                 return -ENOMEM;
2278         }
2279
2280         return 0;
2281 }
2282
2283 static int sa_dma_init(struct sa_crypto_data *dd)
2284 {
2285         int ret;
2286         struct dma_slave_config cfg;
2287
2288         dd->dma_rx1 = NULL;
2289         dd->dma_tx = NULL;
2290         dd->dma_rx2 = NULL;
2291
2292         ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
2293         if (ret)
2294                 return ret;
2295
2296         dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
2297         if (IS_ERR(dd->dma_rx1))
2298                 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
2299                                      "Unable to request rx1 DMA channel\n");
2300
2301         dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
2302         if (IS_ERR(dd->dma_rx2)) {
2303                 dma_release_channel(dd->dma_rx1);
2304                 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
2305                                      "Unable to request rx2 DMA channel\n");
2306         }
2307
2308         dd->dma_tx = dma_request_chan(dd->dev, "tx");
2309         if (IS_ERR(dd->dma_tx)) {
2310                 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
2311                                     "Unable to request tx DMA channel\n");
2312                 goto err_dma_tx;
2313         }
2314
2315         memzero_explicit(&cfg, sizeof(cfg));
2316
2317         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2318         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2319         cfg.src_maxburst = 4;
2320         cfg.dst_maxburst = 4;
2321
2322         ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
2323         if (ret) {
2324                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2325                         ret);
2326                 return ret;
2327         }
2328
2329         ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
2330         if (ret) {
2331                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2332                         ret);
2333                 return ret;
2334         }
2335
2336         ret = dmaengine_slave_config(dd->dma_tx, &cfg);
2337         if (ret) {
2338                 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
2339                         ret);
2340                 return ret;
2341         }
2342
2343         return 0;
2344
2345 err_dma_tx:
2346         dma_release_channel(dd->dma_rx1);
2347         dma_release_channel(dd->dma_rx2);
2348
2349         return ret;
2350 }
2351
2352 static int sa_link_child(struct device *dev, void *data)
2353 {
2354         struct device *parent = data;
2355
2356         device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER);
2357
2358         return 0;
2359 }
2360
2361 static struct sa_match_data am654_match_data = {
2362         .priv = 1,
2363         .priv_id = 1,
2364         .supported_algos = GENMASK(SA_ALG_AUTHENC_SHA256_AES, 0),
2365 };
2366
2367 static struct sa_match_data am64_match_data = {
2368         .priv = 0,
2369         .priv_id = 0,
2370         .supported_algos = BIT(SA_ALG_CBC_AES) |
2371                            BIT(SA_ALG_EBC_AES) |
2372                            BIT(SA_ALG_SHA256) |
2373                            BIT(SA_ALG_SHA512) |
2374                            BIT(SA_ALG_AUTHENC_SHA256_AES),
2375         .skip_engine_control = true,
2376 };
2377
2378 static const struct of_device_id of_match[] = {
2379         { .compatible = "ti,j721e-sa2ul", .data = &am654_match_data, },
2380         { .compatible = "ti,am654-sa2ul", .data = &am654_match_data, },
2381         { .compatible = "ti,am64-sa2ul", .data = &am64_match_data, },
2382         {},
2383 };
2384 MODULE_DEVICE_TABLE(of, of_match);
2385
2386 static int sa_ul_probe(struct platform_device *pdev)
2387 {
2388         const struct of_device_id *match;
2389         struct device *dev = &pdev->dev;
2390         struct device_node *node = dev->of_node;
2391         struct resource *res;
2392         static void __iomem *saul_base;
2393         struct sa_crypto_data *dev_data;
2394         int ret;
2395
2396         dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
2397         if (!dev_data)
2398                 return -ENOMEM;
2399
2400         sa_k3_dev = dev;
2401         dev_data->dev = dev;
2402         dev_data->pdev = pdev;
2403         platform_set_drvdata(pdev, dev_data);
2404         dev_set_drvdata(sa_k3_dev, dev_data);
2405
2406         pm_runtime_enable(dev);
2407         ret = pm_runtime_resume_and_get(dev);
2408         if (ret < 0) {
2409                 dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__,
2410                         ret);
2411                 return ret;
2412         }
2413
2414         sa_init_mem(dev_data);
2415         ret = sa_dma_init(dev_data);
2416         if (ret)
2417                 goto disable_pm_runtime;
2418
2419         match = of_match_node(of_match, dev->of_node);
2420         if (!match) {
2421                 dev_err(dev, "No compatible match found\n");
2422                 return -ENODEV;
2423         }
2424         dev_data->match_data = match->data;
2425
2426         spin_lock_init(&dev_data->scid_lock);
2427         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2428         saul_base = devm_ioremap_resource(dev, res);
2429
2430         dev_data->base = saul_base;
2431
2432         if (!dev_data->match_data->skip_engine_control) {
2433                 u32 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
2434                           SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN |
2435                           SA_EEC_TRNG_EN;
2436
2437                 writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
2438         }
2439
2440         sa_register_algos(dev_data);
2441
2442         ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
2443         if (ret)
2444                 goto release_dma;
2445
2446         device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child);
2447
2448         return 0;
2449
2450 release_dma:
2451         sa_unregister_algos(&pdev->dev);
2452
2453         dma_release_channel(dev_data->dma_rx2);
2454         dma_release_channel(dev_data->dma_rx1);
2455         dma_release_channel(dev_data->dma_tx);
2456
2457         dma_pool_destroy(dev_data->sc_pool);
2458
2459 disable_pm_runtime:
2460         pm_runtime_put_sync(&pdev->dev);
2461         pm_runtime_disable(&pdev->dev);
2462
2463         return ret;
2464 }
2465
2466 static int sa_ul_remove(struct platform_device *pdev)
2467 {
2468         struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
2469
2470         sa_unregister_algos(&pdev->dev);
2471
2472         dma_release_channel(dev_data->dma_rx2);
2473         dma_release_channel(dev_data->dma_rx1);
2474         dma_release_channel(dev_data->dma_tx);
2475
2476         dma_pool_destroy(dev_data->sc_pool);
2477
2478         platform_set_drvdata(pdev, NULL);
2479
2480         pm_runtime_put_sync(&pdev->dev);
2481         pm_runtime_disable(&pdev->dev);
2482
2483         return 0;
2484 }
2485
2486 static struct platform_driver sa_ul_driver = {
2487         .probe = sa_ul_probe,
2488         .remove = sa_ul_remove,
2489         .driver = {
2490                    .name = "saul-crypto",
2491                    .of_match_table = of_match,
2492                    },
2493 };
2494 module_platform_driver(sa_ul_driver);
2495 MODULE_LICENSE("GPL v2");