crypto: x86/aegis128 - eliminate some indirect calls

Instead of using a struct of function pointers to decide whether to call
the encryption or decryption assembly functions, use a conditional
branch on a bool.  Force-inline the functions to avoid actually
generating the branch.  This improves performance slightly since
indirect calls are slow.  Remove the now-unnecessary CFI stubs.

Note that just force-inlining the existing functions might cause the
compiler to optimize out the indirect branches, but that would not be a
reliable way to do it and the CFI stubs would still be required.

Reviewed-by: Ondrej Mosnacek <omosnace@redhat.com>
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/x86/crypto/aegis128-aesni-asm.S b/arch/x86/crypto/aegis128-aesni-asm.S
index 2de8591..1b57558 100644 (file)
--- a/arch/x86/crypto/aegis128-aesni-asm.S
+++ b/arch/x86/crypto/aegis128-aesni-asm.S
@@ -7,7 +7,6 @@
  */
 
 #include <linux/linkage.h>
-#include <linux/cfi_types.h>
 #include <asm/frame.h>
 
 #define STATE0 %xmm0
@@ -403,7 +402,7 @@ SYM_FUNC_END(crypto_aegis128_aesni_ad)
  * void crypto_aegis128_aesni_enc(void *state, unsigned int length,
  *                                const void *src, void *dst);
  */
-SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
+SYM_FUNC_START(crypto_aegis128_aesni_enc)
        FRAME_BEGIN
 
        cmp $0x10, LEN
@@ -500,7 +499,7 @@ SYM_FUNC_END(crypto_aegis128_aesni_enc)
  * void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length,
  *                                     const void *src, void *dst);
  */
-SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail)
+SYM_FUNC_START(crypto_aegis128_aesni_enc_tail)
        FRAME_BEGIN
 
        /* load the state: */
@@ -557,7 +556,7 @@ SYM_FUNC_END(crypto_aegis128_aesni_enc_tail)
  * void crypto_aegis128_aesni_dec(void *state, unsigned int length,
  *                                const void *src, void *dst);
  */
-SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
+SYM_FUNC_START(crypto_aegis128_aesni_dec)
        FRAME_BEGIN
 
        cmp $0x10, LEN
@@ -654,7 +653,7 @@ SYM_FUNC_END(crypto_aegis128_aesni_dec)
  * void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length,
  *                                     const void *src, void *dst);
  */
-SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail)
+SYM_FUNC_START(crypto_aegis128_aesni_dec_tail)
        FRAME_BEGIN
 
        /* load the state: */

diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c
index 9658647..deb39ce 100644 (file)
--- a/arch/x86/crypto/aegis128-aesni-glue.c
+++ b/arch/x86/crypto/aegis128-aesni-glue.c
@@ -56,16 +56,6 @@ struct aegis_ctx {
        struct aegis_block key;
 };
 
-struct aegis_crypt_ops {
-       int (*skcipher_walk_init)(struct skcipher_walk *walk,
-                                 struct aead_request *req, bool atomic);
-
-       void (*crypt_blocks)(void *state, unsigned int length, const void *src,
-                            void *dst);
-       void (*crypt_tail)(void *state, unsigned int length, const void *src,
-                          void *dst);
-};
-
 static void crypto_aegis128_aesni_process_ad(
                struct aegis_state *state, struct scatterlist *sg_src,
                unsigned int assoclen)
@@ -114,20 +104,37 @@ static void crypto_aegis128_aesni_process_ad(
        }
 }
 
-static void crypto_aegis128_aesni_process_crypt(
-               struct aegis_state *state, struct skcipher_walk *walk,
-               const struct aegis_crypt_ops *ops)
+static __always_inline void
+crypto_aegis128_aesni_process_crypt(struct aegis_state *state,
+                                   struct skcipher_walk *walk, bool enc)
 {
        while (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
-               ops->crypt_blocks(state,
-                                 round_down(walk->nbytes, AEGIS128_BLOCK_SIZE),
-                                 walk->src.virt.addr, walk->dst.virt.addr);
+               if (enc)
+                       crypto_aegis128_aesni_enc(
+                                       state,
+                                       round_down(walk->nbytes,
+                                                  AEGIS128_BLOCK_SIZE),
+                                       walk->src.virt.addr,
+                                       walk->dst.virt.addr);
+               else
+                       crypto_aegis128_aesni_dec(
+                                       state,
+                                       round_down(walk->nbytes,
+                                                  AEGIS128_BLOCK_SIZE),
+                                       walk->src.virt.addr,
+                                       walk->dst.virt.addr);
                skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE);
        }
 
        if (walk->nbytes) {
-               ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr,
-                               walk->dst.virt.addr);
+               if (enc)
+                       crypto_aegis128_aesni_enc_tail(state, walk->nbytes,
+                                                      walk->src.virt.addr,
+                                                      walk->dst.virt.addr);
+               else
+                       crypto_aegis128_aesni_dec_tail(state, walk->nbytes,
+                                                      walk->src.virt.addr,
+                                                      walk->dst.virt.addr);
                skcipher_walk_done(walk, 0);
        }
 }
@@ -162,23 +169,26 @@ static int crypto_aegis128_aesni_setauthsize(struct crypto_aead *tfm,
        return 0;
 }
 
-static void crypto_aegis128_aesni_crypt(struct aead_request *req,
-                                       struct aegis_block *tag_xor,
-                                       unsigned int cryptlen,
-                                       const struct aegis_crypt_ops *ops)
+static __always_inline void
+crypto_aegis128_aesni_crypt(struct aead_request *req,
+                           struct aegis_block *tag_xor,
+                           unsigned int cryptlen, bool enc)
 {
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
        struct skcipher_walk walk;
        struct aegis_state state;
 
-       ops->skcipher_walk_init(&walk, req, true);
+       if (enc)
+               skcipher_walk_aead_encrypt(&walk, req, true);
+       else
+               skcipher_walk_aead_decrypt(&walk, req, true);
 
        kernel_fpu_begin();
 
        crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv);
        crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
-       crypto_aegis128_aesni_process_crypt(&state, &walk, ops);
+       crypto_aegis128_aesni_process_crypt(&state, &walk, enc);
        crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
 
        kernel_fpu_end();
@@ -186,18 +196,12 @@ static void crypto_aegis128_aesni_crypt(struct aead_request *req,
 
 static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
 {
-       static const struct aegis_crypt_ops OPS = {
-               .skcipher_walk_init = skcipher_walk_aead_encrypt,
-               .crypt_blocks = crypto_aegis128_aesni_enc,
-               .crypt_tail = crypto_aegis128_aesni_enc_tail,
-       };
-
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        struct aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;
 
-       crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
+       crypto_aegis128_aesni_crypt(req, &tag, cryptlen, true);
 
        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
@@ -208,12 +212,6 @@ static int crypto_aegis128_aesni_decrypt(struct aead_request *req)
 {
        static const struct aegis_block zeros = {};
 
-       static const struct aegis_crypt_ops OPS = {
-               .skcipher_walk_init = skcipher_walk_aead_decrypt,
-               .crypt_blocks = crypto_aegis128_aesni_dec,
-               .crypt_tail = crypto_aegis128_aesni_dec_tail,
-       };
-
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        struct aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
@@ -222,7 +220,7 @@ static int crypto_aegis128_aesni_decrypt(struct aead_request *req)
        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);
 
-       crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
+       crypto_aegis128_aesni_crypt(req, &tag, cryptlen, false);
 
        return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
 }