1 // SPDX-License-Identifier: GPL-2.0-only
4 * Copyright (C) 2016 Cavium, Inc.
7 #include <crypto/aes.h>
8 #include <crypto/algapi.h>
9 #include <crypto/authenc.h>
10 #include <crypto/crypto_wq.h>
11 #include <crypto/des.h>
12 #include <crypto/xts.h>
13 #include <linux/crypto.h>
14 #include <linux/err.h>
15 #include <linux/list.h>
16 #include <linux/scatterlist.h>
19 #include "cptvf_algs.h"
21 struct cpt_device_handle {
22 void *cdev[MAX_DEVICES];
26 static struct cpt_device_handle dev_handle;
28 static void cvm_callback(u32 status, void *arg)
30 struct crypto_async_request *req = (struct crypto_async_request *)arg;
32 req->complete(req, !status);
35 static inline void update_input_iv(struct cpt_request_info *req_info,
36 u8 *iv, u32 enc_iv_len,
39 /* Setting the iv information */
40 req_info->in[*argcnt].vptr = (void *)iv;
41 req_info->in[*argcnt].size = enc_iv_len;
42 req_info->req.dlen += enc_iv_len;
47 static inline void update_output_iv(struct cpt_request_info *req_info,
48 u8 *iv, u32 enc_iv_len,
51 /* Setting the iv information */
52 req_info->out[*argcnt].vptr = (void *)iv;
53 req_info->out[*argcnt].size = enc_iv_len;
54 req_info->rlen += enc_iv_len;
59 static inline void update_input_data(struct cpt_request_info *req_info,
60 struct scatterlist *inp_sg,
61 u32 nbytes, u32 *argcnt)
63 req_info->req.dlen += nbytes;
66 u32 len = min(nbytes, inp_sg->length);
67 u8 *ptr = sg_virt(inp_sg);
69 req_info->in[*argcnt].vptr = (void *)ptr;
70 req_info->in[*argcnt].size = len;
78 static inline void update_output_data(struct cpt_request_info *req_info,
79 struct scatterlist *outp_sg,
80 u32 nbytes, u32 *argcnt)
82 req_info->rlen += nbytes;
85 u32 len = min(nbytes, outp_sg->length);
86 u8 *ptr = sg_virt(outp_sg);
88 req_info->out[*argcnt].vptr = (void *)ptr;
89 req_info->out[*argcnt].size = len;
96 static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
99 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
100 struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
101 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
102 struct fc_context *fctx = &rctx->fctx;
103 u64 *offset_control = &rctx->control_word;
104 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
105 struct cpt_request_info *req_info = &rctx->cpt_req;
106 u64 *ctrl_flags = NULL;
108 req_info->ctrl.s.grp = 0;
109 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
110 req_info->ctrl.s.se_req = SE_CORE_REQ;
112 req_info->req.opcode.s.major = MAJOR_OP_FC |
113 DMA_MODE_FLAG(DMA_GATHER_SCATTER);
115 req_info->req.opcode.s.minor = 2;
117 req_info->req.opcode.s.minor = 3;
119 req_info->req.param1 = req->nbytes; /* Encryption Data length */
120 req_info->req.param2 = 0; /*Auth data length */
122 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
123 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
124 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
126 if (ctx->cipher_type == AES_XTS)
127 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
129 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
130 ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
131 *ctrl_flags = cpu_to_be64(*ctrl_flags);
133 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
134 /* Storing Packet Data Information in offset
135 * Control Word First 8 bytes
137 req_info->in[*argcnt].vptr = (u8 *)offset_control;
138 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
139 req_info->req.dlen += CONTROL_WORD_LEN;
142 req_info->in[*argcnt].vptr = (u8 *)fctx;
143 req_info->in[*argcnt].size = sizeof(struct fc_context);
144 req_info->req.dlen += sizeof(struct fc_context);
151 static inline u32 create_input_list(struct ablkcipher_request *req, u32 enc,
154 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
155 struct cpt_request_info *req_info = &rctx->cpt_req;
158 create_ctx_hdr(req, enc, &argcnt);
159 update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
160 update_input_data(req_info, req->src, req->nbytes, &argcnt);
161 req_info->incnt = argcnt;
166 static inline void store_cb_info(struct ablkcipher_request *req,
167 struct cpt_request_info *req_info)
169 req_info->callback = (void *)cvm_callback;
170 req_info->callback_arg = (void *)&req->base;
173 static inline void create_output_list(struct ablkcipher_request *req,
176 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
177 struct cpt_request_info *req_info = &rctx->cpt_req;
180 /* OUTPUT Buffer Processing
181 * AES encryption/decryption output would be
182 * received in the following format
184 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
185 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
187 /* Reading IV information */
188 update_output_iv(req_info, req->info, enc_iv_len, &argcnt);
189 update_output_data(req_info, req->dst, req->nbytes, &argcnt);
190 req_info->outcnt = argcnt;
193 static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc)
195 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
196 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
197 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
198 struct fc_context *fctx = &rctx->fctx;
199 struct cpt_request_info *req_info = &rctx->cpt_req;
203 memset(req_info, 0, sizeof(struct cpt_request_info));
204 memset(fctx, 0, sizeof(struct fc_context));
205 create_input_list(req, enc, enc_iv_len);
206 create_output_list(req, enc_iv_len);
207 store_cb_info(req, req_info);
208 cdev = dev_handle.cdev[smp_processor_id()];
209 status = cptvf_do_request(cdev, req_info);
210 /* We perform an asynchronous send and once
211 * the request is completed the driver would
212 * intimate through registered call back functions
221 static int cvm_encrypt(struct ablkcipher_request *req)
223 return cvm_enc_dec(req, true);
226 static int cvm_decrypt(struct ablkcipher_request *req)
228 return cvm_enc_dec(req, false);
231 static int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
234 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
235 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
237 const u8 *key1 = key;
238 const u8 *key2 = key + (keylen / 2);
240 err = xts_check_key(tfm, key, keylen);
243 ctx->key_len = keylen;
244 memcpy(ctx->enc_key, key1, keylen / 2);
245 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
246 ctx->cipher_type = AES_XTS;
247 switch (ctx->key_len) {
249 ctx->key_type = AES_128_BIT;
252 ctx->key_type = AES_256_BIT;
261 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
263 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
264 ctx->key_len = keylen;
265 switch (ctx->key_len) {
267 ctx->key_type = AES_128_BIT;
270 ctx->key_type = AES_192_BIT;
273 ctx->key_type = AES_256_BIT;
279 if (ctx->cipher_type == DES3_CBC)
288 static int cvm_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
289 u32 keylen, u8 cipher_type)
291 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
292 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
294 ctx->cipher_type = cipher_type;
295 if (!cvm_validate_keylen(ctx, keylen)) {
296 memcpy(ctx->enc_key, key, keylen);
299 crypto_ablkcipher_set_flags(cipher,
300 CRYPTO_TFM_RES_BAD_KEY_LEN);
305 static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
308 return cvm_setkey(cipher, key, keylen, AES_CBC);
311 static int cvm_ecb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
314 return cvm_setkey(cipher, key, keylen, AES_ECB);
317 static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
320 return cvm_setkey(cipher, key, keylen, AES_CFB);
323 static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
326 u32 flags = crypto_ablkcipher_get_flags(cipher);
329 err = __des3_verify_key(&flags, key);
331 crypto_ablkcipher_set_flags(cipher, flags);
335 return cvm_setkey(cipher, key, keylen, DES3_CBC);
338 static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
341 u32 flags = crypto_ablkcipher_get_flags(cipher);
344 err = __des3_verify_key(&flags, key);
346 crypto_ablkcipher_set_flags(cipher, flags);
350 return cvm_setkey(cipher, key, keylen, DES3_ECB);
353 static int cvm_enc_dec_init(struct crypto_tfm *tfm)
355 tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx);
359 static struct crypto_alg algs[] = { {
360 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
361 .cra_blocksize = AES_BLOCK_SIZE,
362 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
364 .cra_priority = 4001,
365 .cra_name = "xts(aes)",
366 .cra_driver_name = "cavium-xts-aes",
367 .cra_type = &crypto_ablkcipher_type,
370 .ivsize = AES_BLOCK_SIZE,
371 .min_keysize = 2 * AES_MIN_KEY_SIZE,
372 .max_keysize = 2 * AES_MAX_KEY_SIZE,
373 .setkey = cvm_xts_setkey,
374 .encrypt = cvm_encrypt,
375 .decrypt = cvm_decrypt,
378 .cra_init = cvm_enc_dec_init,
379 .cra_module = THIS_MODULE,
381 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
382 .cra_blocksize = AES_BLOCK_SIZE,
383 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
385 .cra_priority = 4001,
386 .cra_name = "cbc(aes)",
387 .cra_driver_name = "cavium-cbc-aes",
388 .cra_type = &crypto_ablkcipher_type,
391 .ivsize = AES_BLOCK_SIZE,
392 .min_keysize = AES_MIN_KEY_SIZE,
393 .max_keysize = AES_MAX_KEY_SIZE,
394 .setkey = cvm_cbc_aes_setkey,
395 .encrypt = cvm_encrypt,
396 .decrypt = cvm_decrypt,
399 .cra_init = cvm_enc_dec_init,
400 .cra_module = THIS_MODULE,
402 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
403 .cra_blocksize = AES_BLOCK_SIZE,
404 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
406 .cra_priority = 4001,
407 .cra_name = "ecb(aes)",
408 .cra_driver_name = "cavium-ecb-aes",
409 .cra_type = &crypto_ablkcipher_type,
412 .ivsize = AES_BLOCK_SIZE,
413 .min_keysize = AES_MIN_KEY_SIZE,
414 .max_keysize = AES_MAX_KEY_SIZE,
415 .setkey = cvm_ecb_aes_setkey,
416 .encrypt = cvm_encrypt,
417 .decrypt = cvm_decrypt,
420 .cra_init = cvm_enc_dec_init,
421 .cra_module = THIS_MODULE,
423 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
424 .cra_blocksize = AES_BLOCK_SIZE,
425 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
427 .cra_priority = 4001,
428 .cra_name = "cfb(aes)",
429 .cra_driver_name = "cavium-cfb-aes",
430 .cra_type = &crypto_ablkcipher_type,
433 .ivsize = AES_BLOCK_SIZE,
434 .min_keysize = AES_MIN_KEY_SIZE,
435 .max_keysize = AES_MAX_KEY_SIZE,
436 .setkey = cvm_cfb_aes_setkey,
437 .encrypt = cvm_encrypt,
438 .decrypt = cvm_decrypt,
441 .cra_init = cvm_enc_dec_init,
442 .cra_module = THIS_MODULE,
444 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
445 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
446 .cra_ctxsize = sizeof(struct cvm_des3_ctx),
448 .cra_priority = 4001,
449 .cra_name = "cbc(des3_ede)",
450 .cra_driver_name = "cavium-cbc-des3_ede",
451 .cra_type = &crypto_ablkcipher_type,
454 .min_keysize = DES3_EDE_KEY_SIZE,
455 .max_keysize = DES3_EDE_KEY_SIZE,
456 .ivsize = DES_BLOCK_SIZE,
457 .setkey = cvm_cbc_des3_setkey,
458 .encrypt = cvm_encrypt,
459 .decrypt = cvm_decrypt,
462 .cra_init = cvm_enc_dec_init,
463 .cra_module = THIS_MODULE,
465 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
466 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
467 .cra_ctxsize = sizeof(struct cvm_des3_ctx),
469 .cra_priority = 4001,
470 .cra_name = "ecb(des3_ede)",
471 .cra_driver_name = "cavium-ecb-des3_ede",
472 .cra_type = &crypto_ablkcipher_type,
475 .min_keysize = DES3_EDE_KEY_SIZE,
476 .max_keysize = DES3_EDE_KEY_SIZE,
477 .ivsize = DES_BLOCK_SIZE,
478 .setkey = cvm_ecb_des3_setkey,
479 .encrypt = cvm_encrypt,
480 .decrypt = cvm_decrypt,
483 .cra_init = cvm_enc_dec_init,
484 .cra_module = THIS_MODULE,
487 static inline int cav_register_algs(void)
491 err = crypto_register_algs(algs, ARRAY_SIZE(algs));
498 static inline void cav_unregister_algs(void)
500 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
503 int cvm_crypto_init(struct cpt_vf *cptvf)
505 struct pci_dev *pdev = cptvf->pdev;
508 dev_count = dev_handle.dev_count;
509 dev_handle.cdev[dev_count] = cptvf;
510 dev_handle.dev_count++;
512 if (dev_count == 3) {
513 if (cav_register_algs()) {
514 dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
522 void cvm_crypto_exit(void)
526 dev_count = --dev_handle.dev_count;
528 cav_unregister_algs();