1 // SPDX-License-Identifier: GPL-2.0+
3 * Shared descriptors for aead, skcipher algorithms
5 * Copyright 2016-2018 NXP
9 #include "desc_constr.h"
10 #include "caamalg_desc.h"
13 * For aead functions, read payload and write payload,
14 * both of which are specified in req->src and req->dst
16 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
18 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
19 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
20 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
23 /* Set DK bit in class 1 operation if shared */
24 static inline void append_dec_op1(u32 *desc, u32 type)
26 u32 *jump_cmd, *uncond_jump_cmd;
28 /* DK bit is valid only for AES */
29 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
30 append_operation(desc, type | OP_ALG_AS_INITFINAL |
35 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
36 append_operation(desc, type | OP_ALG_AS_INITFINAL |
38 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
39 set_jump_tgt_here(desc, jump_cmd);
40 append_operation(desc, type | OP_ALG_AS_INITFINAL |
41 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
42 set_jump_tgt_here(desc, uncond_jump_cmd);
46 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
47 * (non-protocol) with no (null) encryption.
48 * @desc: pointer to buffer used for descriptor construction
49 * @adata: pointer to authentication transform definitions.
50 * A split key is required for SEC Era < 6; the size of the split key
51 * is specified in this case. Valid algorithm values - one of
52 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
53 * with OP_ALG_AAI_HMAC_PRECOMP.
54 * @icvsize: integrity check value (ICV) size (truncated or full)
57 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
58 unsigned int icvsize, int era)
60 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
62 init_sh_desc(desc, HDR_SHARE_SERIAL);
64 /* Skip if already shared */
65 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
68 if (adata->key_inline)
69 append_key_as_imm(desc, adata->key_virt,
70 adata->keylen_pad, adata->keylen,
71 CLASS_2 | KEY_DEST_MDHA_SPLIT |
74 append_key(desc, adata->key_dma, adata->keylen,
75 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
77 append_proto_dkp(desc, adata);
79 set_jump_tgt_here(desc, key_jump_cmd);
81 /* assoclen + cryptlen = seqinlen */
82 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
84 /* Prepare to read and write cryptlen + assoclen bytes */
85 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
86 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
89 * MOVE_LEN opcode is not available in all SEC HW revisions,
90 * thus need to do some magic, i.e. self-patch the descriptor
93 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
95 (0x6 << MOVE_LEN_SHIFT));
96 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
99 (0x8 << MOVE_LEN_SHIFT));
101 /* Class 2 operation */
102 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
105 /* Read and write cryptlen bytes */
106 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
108 set_move_tgt_here(desc, read_move_cmd);
109 set_move_tgt_here(desc, write_move_cmd);
110 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
111 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
115 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
116 LDST_SRCDST_BYTE_CONTEXT);
119 print_hex_dump(KERN_ERR,
120 "aead null enc shdesc@" __stringify(__LINE__)": ",
121 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
124 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
127 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
128 * (non-protocol) with no (null) decryption.
129 * @desc: pointer to buffer used for descriptor construction
130 * @adata: pointer to authentication transform definitions.
131 * A split key is required for SEC Era < 6; the size of the split key
132 * is specified in this case. Valid algorithm values - one of
133 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
134 * with OP_ALG_AAI_HMAC_PRECOMP.
135 * @icvsize: integrity check value (ICV) size (truncated or full)
138 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
139 unsigned int icvsize, int era)
141 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
143 init_sh_desc(desc, HDR_SHARE_SERIAL);
145 /* Skip if already shared */
146 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
149 if (adata->key_inline)
150 append_key_as_imm(desc, adata->key_virt,
151 adata->keylen_pad, adata->keylen,
152 CLASS_2 | KEY_DEST_MDHA_SPLIT |
155 append_key(desc, adata->key_dma, adata->keylen,
156 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
158 append_proto_dkp(desc, adata);
160 set_jump_tgt_here(desc, key_jump_cmd);
162 /* Class 2 operation */
163 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
164 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
166 /* assoclen + cryptlen = seqoutlen */
167 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
169 /* Prepare to read and write cryptlen + assoclen bytes */
170 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
171 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
174 * MOVE_LEN opcode is not available in all SEC HW revisions,
175 * thus need to do some magic, i.e. self-patch the descriptor
178 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
180 (0x6 << MOVE_LEN_SHIFT));
181 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
184 (0x8 << MOVE_LEN_SHIFT));
186 /* Read and write cryptlen bytes */
187 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
190 * Insert a NOP here, since we need at least 4 instructions between
191 * code patching the descriptor buffer and the location being patched.
193 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
194 set_jump_tgt_here(desc, jump_cmd);
196 set_move_tgt_here(desc, read_move_cmd);
197 set_move_tgt_here(desc, write_move_cmd);
198 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
199 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
201 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
204 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
205 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
208 print_hex_dump(KERN_ERR,
209 "aead null dec shdesc@" __stringify(__LINE__)": ",
210 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
213 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
215 static void init_sh_desc_key_aead(u32 * const desc,
216 struct alginfo * const cdata,
217 struct alginfo * const adata,
218 const bool is_rfc3686, u32 *nonce, int era)
221 unsigned int enckeylen = cdata->keylen;
223 /* Note: Context registers are saved. */
224 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
226 /* Skip if already shared */
227 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
232 * | key = {AUTH_KEY, ENC_KEY, NONCE}
233 * | enckeylen = encryption key size + nonce size
236 enckeylen -= CTR_RFC3686_NONCE_SIZE;
239 if (adata->key_inline)
240 append_key_as_imm(desc, adata->key_virt,
241 adata->keylen_pad, adata->keylen,
242 CLASS_2 | KEY_DEST_MDHA_SPLIT |
245 append_key(desc, adata->key_dma, adata->keylen,
246 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
248 append_proto_dkp(desc, adata);
251 if (cdata->key_inline)
252 append_key_as_imm(desc, cdata->key_virt, enckeylen,
253 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
255 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
258 /* Load Counter into CONTEXT1 reg */
260 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
262 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
265 MOVE_DEST_CLASS1CTX |
266 (16 << MOVE_OFFSET_SHIFT) |
267 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
270 set_jump_tgt_here(desc, key_jump_cmd);
274 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
276 * @desc: pointer to buffer used for descriptor construction
277 * @cdata: pointer to block cipher transform definitions
278 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
279 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
280 * @adata: pointer to authentication transform definitions.
281 * A split key is required for SEC Era < 6; the size of the split key
282 * is specified in this case. Valid algorithm values - one of
283 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
284 * with OP_ALG_AAI_HMAC_PRECOMP.
285 * @ivsize: initialization vector size
286 * @icvsize: integrity check value (ICV) size (truncated or full)
287 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
288 * @nonce: pointer to rfc3686 nonce
289 * @ctx1_iv_off: IV offset in CONTEXT1 register
290 * @is_qi: true when called from caam/qi
293 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
294 struct alginfo *adata, unsigned int ivsize,
295 unsigned int icvsize, const bool is_rfc3686,
296 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
299 /* Note: Context registers are saved. */
300 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
302 /* Class 2 operation */
303 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
309 /* REG3 = assoclen */
310 append_seq_load(desc, 4, LDST_CLASS_DECO |
311 LDST_SRCDST_WORD_DECO_MATH3 |
312 (4 << LDST_OFFSET_SHIFT));
314 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
315 JUMP_COND_CALM | JUMP_COND_NCP |
316 JUMP_COND_NOP | JUMP_COND_NIP |
318 set_jump_tgt_here(desc, wait_load_cmd);
320 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
321 LDST_SRCDST_BYTE_CONTEXT |
322 (ctx1_iv_off << LDST_OFFSET_SHIFT));
325 /* Read and write assoclen bytes */
326 if (is_qi || era < 3) {
327 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
328 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
330 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
331 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
334 /* Skip assoc data */
335 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
337 /* read assoc before reading payload */
338 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
341 /* Load Counter into CONTEXT1 reg */
343 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
344 LDST_SRCDST_BYTE_CONTEXT |
345 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
348 /* Class 1 operation */
349 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
352 /* Read and write cryptlen bytes */
353 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
354 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
355 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
358 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
359 LDST_SRCDST_BYTE_CONTEXT);
362 print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
363 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
366 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
369 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
371 * @desc: pointer to buffer used for descriptor construction
372 * @cdata: pointer to block cipher transform definitions
373 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
374 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
375 * @adata: pointer to authentication transform definitions.
376 * A split key is required for SEC Era < 6; the size of the split key
377 * is specified in this case. Valid algorithm values - one of
378 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
379 * with OP_ALG_AAI_HMAC_PRECOMP.
380 * @ivsize: initialization vector size
381 * @icvsize: integrity check value (ICV) size (truncated or full)
382 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
383 * @nonce: pointer to rfc3686 nonce
384 * @ctx1_iv_off: IV offset in CONTEXT1 register
385 * @is_qi: true when called from caam/qi
388 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
389 struct alginfo *adata, unsigned int ivsize,
390 unsigned int icvsize, const bool geniv,
391 const bool is_rfc3686, u32 *nonce,
392 const u32 ctx1_iv_off, const bool is_qi, int era)
394 /* Note: Context registers are saved. */
395 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
397 /* Class 2 operation */
398 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
399 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
404 /* REG3 = assoclen */
405 append_seq_load(desc, 4, LDST_CLASS_DECO |
406 LDST_SRCDST_WORD_DECO_MATH3 |
407 (4 << LDST_OFFSET_SHIFT));
409 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
410 JUMP_COND_CALM | JUMP_COND_NCP |
411 JUMP_COND_NOP | JUMP_COND_NIP |
413 set_jump_tgt_here(desc, wait_load_cmd);
416 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
417 LDST_SRCDST_BYTE_CONTEXT |
418 (ctx1_iv_off << LDST_OFFSET_SHIFT));
421 /* Read and write assoclen bytes */
422 if (is_qi || era < 3) {
423 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
425 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
428 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
431 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
433 append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
436 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
440 /* Skip assoc data */
441 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
443 /* read assoc before reading payload */
444 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
448 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
449 LDST_SRCDST_BYTE_CONTEXT |
450 (ctx1_iv_off << LDST_OFFSET_SHIFT));
451 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
452 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
455 /* Load Counter into CONTEXT1 reg */
457 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
458 LDST_SRCDST_BYTE_CONTEXT |
459 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
462 /* Choose operation */
464 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
467 append_dec_op1(desc, cdata->algtype);
469 /* Read and write cryptlen bytes */
470 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
471 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
472 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
475 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
476 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
479 print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
480 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
483 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
486 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
487 * (non-protocol) with HW-generated initialization
489 * @desc: pointer to buffer used for descriptor construction
490 * @cdata: pointer to block cipher transform definitions
491 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
492 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
493 * @adata: pointer to authentication transform definitions.
494 * A split key is required for SEC Era < 6; the size of the split key
495 * is specified in this case. Valid algorithm values - one of
496 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
497 * with OP_ALG_AAI_HMAC_PRECOMP.
498 * @ivsize: initialization vector size
499 * @icvsize: integrity check value (ICV) size (truncated or full)
500 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
501 * @nonce: pointer to rfc3686 nonce
502 * @ctx1_iv_off: IV offset in CONTEXT1 register
503 * @is_qi: true when called from caam/qi
506 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
507 struct alginfo *adata, unsigned int ivsize,
508 unsigned int icvsize, const bool is_rfc3686,
509 u32 *nonce, const u32 ctx1_iv_off,
510 const bool is_qi, int era)
514 /* Note: Context registers are saved. */
515 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
520 /* REG3 = assoclen */
521 append_seq_load(desc, 4, LDST_CLASS_DECO |
522 LDST_SRCDST_WORD_DECO_MATH3 |
523 (4 << LDST_OFFSET_SHIFT));
525 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
526 JUMP_COND_CALM | JUMP_COND_NCP |
527 JUMP_COND_NOP | JUMP_COND_NIP |
529 set_jump_tgt_here(desc, wait_load_cmd);
534 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
535 LDST_SRCDST_BYTE_CONTEXT |
536 (ctx1_iv_off << LDST_OFFSET_SHIFT));
542 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
543 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
544 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
545 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
546 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
547 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
548 append_move(desc, MOVE_WAITCOMP |
549 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
550 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
551 (ivsize << MOVE_LEN_SHIFT));
552 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
555 /* Copy IV to class 1 context */
556 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
557 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
558 (ivsize << MOVE_LEN_SHIFT));
560 /* Return to encryption */
561 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
564 /* Read and write assoclen bytes */
565 if (is_qi || era < 3) {
566 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
567 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
569 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
570 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
573 /* Skip assoc data */
574 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
576 /* read assoc before reading payload */
577 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
580 /* Copy iv from outfifo to class 2 fifo */
581 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
582 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
583 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
584 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
585 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
586 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
588 /* Load Counter into CONTEXT1 reg */
590 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
591 LDST_SRCDST_BYTE_CONTEXT |
592 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
595 /* Class 1 operation */
596 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
599 /* Will write ivsize + cryptlen */
600 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
602 /* Not need to reload iv */
603 append_seq_fifo_load(desc, ivsize,
606 /* Will read cryptlen */
607 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
608 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
609 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
610 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
613 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
614 LDST_SRCDST_BYTE_CONTEXT);
617 print_hex_dump(KERN_ERR,
618 "aead givenc shdesc@" __stringify(__LINE__)": ",
619 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
622 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
625 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
626 * @desc: pointer to buffer used for descriptor construction
627 * @cdata: pointer to block cipher transform definitions
628 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
629 * @ivsize: initialization vector size
630 * @icvsize: integrity check value (ICV) size (truncated or full)
631 * @is_qi: true when called from caam/qi
633 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
634 unsigned int ivsize, unsigned int icvsize,
637 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
638 *zero_assoc_jump_cmd2;
640 init_sh_desc(desc, HDR_SHARE_SERIAL);
642 /* skip key loading if they are loaded due to sharing */
643 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
645 if (cdata->key_inline)
646 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
647 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
649 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
651 set_jump_tgt_here(desc, key_jump_cmd);
653 /* class 1 operation */
654 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
660 /* REG3 = assoclen */
661 append_seq_load(desc, 4, LDST_CLASS_DECO |
662 LDST_SRCDST_WORD_DECO_MATH3 |
663 (4 << LDST_OFFSET_SHIFT));
665 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
666 JUMP_COND_CALM | JUMP_COND_NCP |
667 JUMP_COND_NOP | JUMP_COND_NIP |
669 set_jump_tgt_here(desc, wait_load_cmd);
671 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
674 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
678 /* if assoclen + cryptlen is ZERO, skip to ICV write */
679 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
683 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
684 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
686 /* if assoclen is ZERO, skip reading the assoc data */
687 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
688 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
691 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
693 /* skip assoc data */
694 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
696 /* cryptlen = seqinlen - assoclen */
697 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
699 /* if cryptlen is ZERO jump to zero-payload commands */
700 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
703 /* read assoc data */
704 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
705 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
706 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
708 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
710 /* write encrypted data */
711 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
713 /* read payload data */
714 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
715 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
717 /* jump to ICV writing */
719 append_jump(desc, JUMP_TEST_ALL | 4);
721 append_jump(desc, JUMP_TEST_ALL | 2);
723 /* zero-payload commands */
724 set_jump_tgt_here(desc, zero_payload_jump_cmd);
726 /* read assoc data */
727 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
728 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
730 /* jump to ICV writing */
731 append_jump(desc, JUMP_TEST_ALL | 2);
733 /* There is no input data */
734 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
737 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
738 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
742 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
743 LDST_SRCDST_BYTE_CONTEXT);
746 print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
747 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
750 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
753 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
754 * @desc: pointer to buffer used for descriptor construction
755 * @cdata: pointer to block cipher transform definitions
756 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
757 * @ivsize: initialization vector size
758 * @icvsize: integrity check value (ICV) size (truncated or full)
759 * @is_qi: true when called from caam/qi
761 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
762 unsigned int ivsize, unsigned int icvsize,
765 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
767 init_sh_desc(desc, HDR_SHARE_SERIAL);
769 /* skip key loading if they are loaded due to sharing */
770 key_jump_cmd = append_jump(desc, JUMP_JSL |
771 JUMP_TEST_ALL | JUMP_COND_SHRD);
772 if (cdata->key_inline)
773 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
774 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
776 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
778 set_jump_tgt_here(desc, key_jump_cmd);
780 /* class 1 operation */
781 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
782 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
787 /* REG3 = assoclen */
788 append_seq_load(desc, 4, LDST_CLASS_DECO |
789 LDST_SRCDST_WORD_DECO_MATH3 |
790 (4 << LDST_OFFSET_SHIFT));
792 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
793 JUMP_COND_CALM | JUMP_COND_NCP |
794 JUMP_COND_NOP | JUMP_COND_NIP |
796 set_jump_tgt_here(desc, wait_load_cmd);
798 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
799 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
802 /* if assoclen is ZERO, skip reading the assoc data */
803 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
804 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
807 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
809 /* skip assoc data */
810 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
812 /* read assoc data */
813 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
816 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
818 /* cryptlen = seqoutlen - assoclen */
819 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
821 /* jump to zero-payload command if cryptlen is zero */
822 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
825 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
827 /* store encrypted data */
828 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
830 /* read payload data */
831 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
834 /* zero-payload command */
835 set_jump_tgt_here(desc, zero_payload_jump_cmd);
838 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
839 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
842 print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
843 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
846 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
849 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
851 * @desc: pointer to buffer used for descriptor construction
852 * @cdata: pointer to block cipher transform definitions
853 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
854 * @ivsize: initialization vector size
855 * @icvsize: integrity check value (ICV) size (truncated or full)
856 * @is_qi: true when called from caam/qi
858 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
859 unsigned int ivsize, unsigned int icvsize,
864 init_sh_desc(desc, HDR_SHARE_SERIAL);
866 /* Skip key loading if it is loaded due to sharing */
867 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
869 if (cdata->key_inline)
870 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
871 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
873 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
875 set_jump_tgt_here(desc, key_jump_cmd);
877 /* Class 1 operation */
878 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
884 /* REG3 = assoclen */
885 append_seq_load(desc, 4, LDST_CLASS_DECO |
886 LDST_SRCDST_WORD_DECO_MATH3 |
887 (4 << LDST_OFFSET_SHIFT));
889 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
890 JUMP_COND_CALM | JUMP_COND_NCP |
891 JUMP_COND_NOP | JUMP_COND_NIP |
893 set_jump_tgt_here(desc, wait_load_cmd);
895 /* Read salt and IV */
896 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
897 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
899 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
900 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
903 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
904 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
906 /* Read assoc data */
907 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
908 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
911 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
913 /* Will read cryptlen bytes */
914 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
916 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
917 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
919 /* Skip assoc data */
920 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
922 /* cryptlen = seqoutlen - assoclen */
923 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
925 /* Write encrypted data */
926 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
928 /* Read payload data */
929 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
930 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
933 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
934 LDST_SRCDST_BYTE_CONTEXT);
937 print_hex_dump(KERN_ERR,
938 "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
939 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
942 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
945 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
947 * @desc: pointer to buffer used for descriptor construction
948 * @cdata: pointer to block cipher transform definitions
949 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
950 * @ivsize: initialization vector size
951 * @icvsize: integrity check value (ICV) size (truncated or full)
952 * @is_qi: true when called from caam/qi
954 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
955 unsigned int ivsize, unsigned int icvsize,
960 init_sh_desc(desc, HDR_SHARE_SERIAL);
962 /* Skip key loading if it is loaded due to sharing */
963 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
965 if (cdata->key_inline)
966 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
967 cdata->keylen, CLASS_1 |
970 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
972 set_jump_tgt_here(desc, key_jump_cmd);
974 /* Class 1 operation */
975 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
976 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
981 /* REG3 = assoclen */
982 append_seq_load(desc, 4, LDST_CLASS_DECO |
983 LDST_SRCDST_WORD_DECO_MATH3 |
984 (4 << LDST_OFFSET_SHIFT));
986 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
987 JUMP_COND_CALM | JUMP_COND_NCP |
988 JUMP_COND_NOP | JUMP_COND_NIP |
990 set_jump_tgt_here(desc, wait_load_cmd);
992 /* Read salt and IV */
993 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
994 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
996 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
997 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1000 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1001 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1003 /* Read assoc data */
1004 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1005 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1008 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1010 /* Will read cryptlen bytes */
1011 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1013 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1014 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1016 /* Skip assoc data */
1017 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1019 /* Will write cryptlen bytes */
1020 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1022 /* Store payload data */
1023 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1025 /* Read encrypted data */
1026 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1027 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1030 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1031 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1034 print_hex_dump(KERN_ERR,
1035 "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1036 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1039 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1042 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1044 * @desc: pointer to buffer used for descriptor construction
1045 * @cdata: pointer to block cipher transform definitions
1046 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1047 * @ivsize: initialization vector size
1048 * @icvsize: integrity check value (ICV) size (truncated or full)
1049 * @is_qi: true when called from caam/qi
1051 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1052 unsigned int ivsize, unsigned int icvsize,
1055 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1057 init_sh_desc(desc, HDR_SHARE_SERIAL);
1059 /* Skip key loading if it is loaded due to sharing */
1060 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1062 if (cdata->key_inline)
1063 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1064 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1066 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1067 KEY_DEST_CLASS_REG);
1068 set_jump_tgt_here(desc, key_jump_cmd);
1070 /* Class 1 operation */
1071 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1075 /* assoclen is not needed, skip it */
1076 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1078 /* Read salt and IV */
1079 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1080 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1082 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1083 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1086 /* assoclen + cryptlen = seqinlen */
1087 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1090 * MOVE_LEN opcode is not available in all SEC HW revisions,
1091 * thus need to do some magic, i.e. self-patch the descriptor
1094 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1095 (0x6 << MOVE_LEN_SHIFT));
1096 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1097 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1099 /* Will read assoclen + cryptlen bytes */
1100 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1102 /* Will write assoclen + cryptlen bytes */
1103 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1105 /* Read and write assoclen + cryptlen bytes */
1106 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1108 set_move_tgt_here(desc, read_move_cmd);
1109 set_move_tgt_here(desc, write_move_cmd);
1110 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1111 /* Move payload data to OFIFO */
1112 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1115 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1116 LDST_SRCDST_BYTE_CONTEXT);
1119 print_hex_dump(KERN_ERR,
1120 "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1121 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1124 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1127 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1129 * @desc: pointer to buffer used for descriptor construction
1130 * @cdata: pointer to block cipher transform definitions
1131 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1132 * @ivsize: initialization vector size
1133 * @icvsize: integrity check value (ICV) size (truncated or full)
1134 * @is_qi: true when called from caam/qi
1136 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1137 unsigned int ivsize, unsigned int icvsize,
1140 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1142 init_sh_desc(desc, HDR_SHARE_SERIAL);
1144 /* Skip key loading if it is loaded due to sharing */
1145 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1147 if (cdata->key_inline)
1148 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1149 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1151 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1152 KEY_DEST_CLASS_REG);
1153 set_jump_tgt_here(desc, key_jump_cmd);
1155 /* Class 1 operation */
1156 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1157 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1160 /* assoclen is not needed, skip it */
1161 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1163 /* Read salt and IV */
1164 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1165 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1167 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1168 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1171 /* assoclen + cryptlen = seqoutlen */
1172 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1175 * MOVE_LEN opcode is not available in all SEC HW revisions,
1176 * thus need to do some magic, i.e. self-patch the descriptor
1179 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1180 (0x6 << MOVE_LEN_SHIFT));
1181 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1182 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1184 /* Will read assoclen + cryptlen bytes */
1185 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1187 /* Will write assoclen + cryptlen bytes */
1188 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1190 /* Store payload data */
1191 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1193 /* In-snoop assoclen + cryptlen data */
1194 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1195 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1197 set_move_tgt_here(desc, read_move_cmd);
1198 set_move_tgt_here(desc, write_move_cmd);
1199 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1200 /* Move payload data to OFIFO */
1201 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1202 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1205 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1206 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1209 print_hex_dump(KERN_ERR,
1210 "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1211 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1214 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1217 * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1218 * IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1219 * descriptor (non-protocol).
1220 * @desc: pointer to buffer used for descriptor construction
1221 * @cdata: pointer to block cipher transform definitions
1222 * Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1224 * @adata: pointer to authentication transform definitions
1225 * Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1227 * @ivsize: initialization vector size
1228 * @icvsize: integrity check value (ICV) size (truncated or full)
1229 * @encap: true if encapsulation, false if decapsulation
1230 * @is_qi: true when called from caam/qi
1232 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1233 struct alginfo *adata, unsigned int ivsize,
1234 unsigned int icvsize, const bool encap,
1237 u32 *key_jump_cmd, *wait_cmd;
1239 const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1241 /* Note: Context registers are saved. */
1242 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1244 /* skip key loading if they are loaded due to sharing */
1245 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1248 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1249 CLASS_1 | KEY_DEST_CLASS_REG);
1251 /* For IPsec load the salt from keymat in the context register */
1253 append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1254 LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1255 4 << LDST_OFFSET_SHIFT);
1257 set_jump_tgt_here(desc, key_jump_cmd);
1259 /* Class 2 and 1 operations: Poly & ChaCha */
1261 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1263 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1266 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1267 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1268 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1274 u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1276 /* REG3 = assoclen */
1277 append_seq_load(desc, 4, LDST_CLASS_DECO |
1278 LDST_SRCDST_WORD_DECO_MATH3 |
1279 4 << LDST_OFFSET_SHIFT);
1281 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1282 JUMP_COND_CALM | JUMP_COND_NCP |
1283 JUMP_COND_NOP | JUMP_COND_NIP |
1285 set_jump_tgt_here(desc, wait_load_cmd);
1287 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1288 LDST_SRCDST_BYTE_CONTEXT |
1289 ctx1_iv_off << LDST_OFFSET_SHIFT);
1294 * Read associated data from the input and send them to class1 and
1295 * class2 alignment blocks. From class1 send data to output fifo and
1296 * then write it to memory since we don't need to encrypt AD.
1298 nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1299 NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1300 append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1301 LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1303 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1304 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1305 append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1306 FIFOLD_CLASS_CLASS1 | LDST_VLF);
1307 append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1308 MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1309 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1311 /* IPsec - copy IV at the output */
1313 append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1316 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1317 JUMP_COND_NOP | JUMP_TEST_ALL);
1318 set_jump_tgt_here(desc, wait_cmd);
1321 /* Read and write cryptlen bytes */
1322 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1323 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1325 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1328 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1329 LDST_SRCDST_BYTE_CONTEXT);
1331 /* Read and write cryptlen bytes */
1332 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1334 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1336 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1338 /* Load ICV for verification */
1339 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1340 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1343 print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1344 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1347 EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1349 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1350 static inline void skcipher_append_src_dst(u32 *desc)
1352 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1353 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1354 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1355 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1356 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1360 * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1361 * @desc: pointer to buffer used for descriptor construction
1362 * @cdata: pointer to block cipher transform definitions
1363 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1364 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1365 * - OP_ALG_ALGSEL_CHACHA20
1366 * @ivsize: initialization vector size
1367 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1368 * @ctx1_iv_off: IV offset in CONTEXT1 register
1370 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1371 unsigned int ivsize, const bool is_rfc3686,
1372 const u32 ctx1_iv_off)
1376 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1377 /* Skip if already shared */
1378 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1381 /* Load class1 key only */
1382 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1383 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1385 /* Load nonce into CONTEXT1 reg */
1387 const u8 *nonce = cdata->key_virt + cdata->keylen;
1389 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1390 LDST_CLASS_IND_CCB |
1391 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1392 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1393 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1394 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1397 set_jump_tgt_here(desc, key_jump_cmd);
1400 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1401 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1403 /* Load counter into CONTEXT1 reg */
1405 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1406 LDST_SRCDST_BYTE_CONTEXT |
1407 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1408 LDST_OFFSET_SHIFT));
1410 /* Load operation */
1411 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1414 /* Perform operation */
1415 skcipher_append_src_dst(desc);
1418 print_hex_dump(KERN_ERR,
1419 "skcipher enc shdesc@" __stringify(__LINE__)": ",
1420 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1423 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1426 * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1427 * @desc: pointer to buffer used for descriptor construction
1428 * @cdata: pointer to block cipher transform definitions
1429 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1430 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1431 * - OP_ALG_ALGSEL_CHACHA20
1432 * @ivsize: initialization vector size
1433 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1434 * @ctx1_iv_off: IV offset in CONTEXT1 register
1436 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1437 unsigned int ivsize, const bool is_rfc3686,
1438 const u32 ctx1_iv_off)
1442 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1443 /* Skip if already shared */
1444 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1447 /* Load class1 key only */
1448 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1449 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1451 /* Load nonce into CONTEXT1 reg */
1453 const u8 *nonce = cdata->key_virt + cdata->keylen;
1455 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1456 LDST_CLASS_IND_CCB |
1457 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1458 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1459 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1460 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1463 set_jump_tgt_here(desc, key_jump_cmd);
1466 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1467 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1469 /* Load counter into CONTEXT1 reg */
1471 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1472 LDST_SRCDST_BYTE_CONTEXT |
1473 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1474 LDST_OFFSET_SHIFT));
1476 /* Choose operation */
1478 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1481 append_dec_op1(desc, cdata->algtype);
1483 /* Perform operation */
1484 skcipher_append_src_dst(desc);
1487 print_hex_dump(KERN_ERR,
1488 "skcipher dec shdesc@" __stringify(__LINE__)": ",
1489 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1492 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1495 * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1496 * @desc: pointer to buffer used for descriptor construction
1497 * @cdata: pointer to block cipher transform definitions
1498 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1500 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1502 __be64 sector_size = cpu_to_be64(512);
1505 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1506 /* Skip if already shared */
1507 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1510 /* Load class1 keys only */
1511 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1512 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1514 /* Load sector size with index 40 bytes (0x28) */
1515 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB |
1516 LDST_SRCDST_BYTE_CONTEXT |
1517 (0x28 << LDST_OFFSET_SHIFT));
1519 set_jump_tgt_here(desc, key_jump_cmd);
1522 * create sequence for loading the sector index
1523 * Upper 8B of IV - will be used as sector index
1524 * Lower 8B of IV - will be discarded
1526 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1527 (0x20 << LDST_OFFSET_SHIFT));
1528 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1530 /* Load operation */
1531 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1534 /* Perform operation */
1535 skcipher_append_src_dst(desc);
1538 print_hex_dump(KERN_ERR,
1539 "xts skcipher enc shdesc@" __stringify(__LINE__) ": ",
1540 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1543 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1546 * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1547 * @desc: pointer to buffer used for descriptor construction
1548 * @cdata: pointer to block cipher transform definitions
1549 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1551 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1553 __be64 sector_size = cpu_to_be64(512);
1556 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1557 /* Skip if already shared */
1558 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1561 /* Load class1 key only */
1562 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1563 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1565 /* Load sector size with index 40 bytes (0x28) */
1566 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB |
1567 LDST_SRCDST_BYTE_CONTEXT |
1568 (0x28 << LDST_OFFSET_SHIFT));
1570 set_jump_tgt_here(desc, key_jump_cmd);
1573 * create sequence for loading the sector index
1574 * Upper 8B of IV - will be used as sector index
1575 * Lower 8B of IV - will be discarded
1577 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1578 (0x20 << LDST_OFFSET_SHIFT));
1579 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1581 /* Load operation */
1582 append_dec_op1(desc, cdata->algtype);
1584 /* Perform operation */
1585 skcipher_append_src_dst(desc);
1588 print_hex_dump(KERN_ERR,
1589 "xts skcipher dec shdesc@" __stringify(__LINE__) ": ",
1590 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1593 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1595 MODULE_LICENSE("GPL");
1596 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1597 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");