arch/arm64/crypto/chacha-neon-glue.c

   1 /*
   2  * ARM NEON accelerated ChaCha and XChaCha stream ciphers,
   3  * including ChaCha20 (RFC7539)
   4  *
   5  * Copyright (C) 2016 - 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * Based on:
  12  * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
  13  *
  14  * Copyright (C) 2015 Martin Willi
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <crypto/algapi.h>
  23 #include <crypto/chacha.h>
  24 #include <crypto/internal/simd.h>
  25 #include <crypto/internal/skcipher.h>
  26 #include <linux/kernel.h>
  27 #include <linux/module.h>
  28
  29 #include <asm/hwcap.h>
  30 #include <asm/neon.h>
  31 #include <asm/simd.h>
  32
  33 asmlinkage void chacha_block_xor_neon(u32 *state, u8 *dst, const u8 *src,
  34                                       int nrounds);
  35 asmlinkage void chacha_4block_xor_neon(u32 *state, u8 *dst, const u8 *src,
  36                                        int nrounds, int bytes);
  37 asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
  38
  39 static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
  40                           int bytes, int nrounds)
  41 {
  42         while (bytes > 0) {
  43                 int l = min(bytes, CHACHA_BLOCK_SIZE * 5);
  44
  45                 if (l <= CHACHA_BLOCK_SIZE) {
  46                         u8 buf[CHACHA_BLOCK_SIZE];
  47
  48                         memcpy(buf, src, l);
  49                         chacha_block_xor_neon(state, buf, buf, nrounds);
  50                         memcpy(dst, buf, l);
  51                         state[12] += 1;
  52                         break;
  53                 }
  54                 chacha_4block_xor_neon(state, dst, src, nrounds, l);
  55                 bytes -= CHACHA_BLOCK_SIZE * 5;
  56                 src += CHACHA_BLOCK_SIZE * 5;
  57                 dst += CHACHA_BLOCK_SIZE * 5;
  58                 state[12] += 5;
  59         }
  60 }
  61
  62 static int chacha_neon_stream_xor(struct skcipher_request *req,
  63                                   struct chacha_ctx *ctx, u8 *iv)
  64 {
  65         struct skcipher_walk walk;
  66         u32 state[16];
  67         int err;
  68
  69         err = skcipher_walk_virt(&walk, req, false);
  70
  71         crypto_chacha_init(state, ctx, iv);
  72
  73         while (walk.nbytes > 0) {
  74                 unsigned int nbytes = walk.nbytes;
  75
  76                 if (nbytes < walk.total)
  77                         nbytes = rounddown(nbytes, walk.stride);
  78
  79                 kernel_neon_begin();
  80                 chacha_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
  81                               nbytes, ctx->nrounds);
  82                 kernel_neon_end();
  83                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
  84         }
  85
  86         return err;
  87 }
  88
  89 static int chacha_neon(struct skcipher_request *req)
  90 {
  91         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  92         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
  93
  94         if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
  95                 return crypto_chacha_crypt(req);
  96
  97         return chacha_neon_stream_xor(req, ctx, req->iv);
  98 }
  99
 100 static int xchacha_neon(struct skcipher_request *req)
 101 {
 102         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 103         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 104         struct chacha_ctx subctx;
 105         u32 state[16];
 106         u8 real_iv[16];
 107
 108         if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
 109                 return crypto_xchacha_crypt(req);
 110
 111         crypto_chacha_init(state, ctx, req->iv);
 112
 113         kernel_neon_begin();
 114         hchacha_block_neon(state, subctx.key, ctx->nrounds);
 115         kernel_neon_end();
 116         subctx.nrounds = ctx->nrounds;
 117
 118         memcpy(&real_iv[0], req->iv + 24, 8);
 119         memcpy(&real_iv[8], req->iv + 16, 8);
 120         return chacha_neon_stream_xor(req, &subctx, real_iv);
 121 }
 122
 123 static struct skcipher_alg algs[] = {
 124         {
 125                 .base.cra_name          = "chacha20",
 126                 .base.cra_driver_name   = "chacha20-neon",
 127                 .base.cra_priority      = 300,
 128                 .base.cra_blocksize     = 1,
 129                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 130                 .base.cra_module        = THIS_MODULE,
 131
 132                 .min_keysize            = CHACHA_KEY_SIZE,
 133                 .max_keysize            = CHACHA_KEY_SIZE,
 134                 .ivsize                 = CHACHA_IV_SIZE,
 135                 .chunksize              = CHACHA_BLOCK_SIZE,
 136                 .walksize               = 5 * CHACHA_BLOCK_SIZE,
 137                 .setkey                 = crypto_chacha20_setkey,
 138                 .encrypt                = chacha_neon,
 139                 .decrypt                = chacha_neon,
 140         }, {
 141                 .base.cra_name          = "xchacha20",
 142                 .base.cra_driver_name   = "xchacha20-neon",
 143                 .base.cra_priority      = 300,
 144                 .base.cra_blocksize     = 1,
 145                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 146                 .base.cra_module        = THIS_MODULE,
 147
 148                 .min_keysize            = CHACHA_KEY_SIZE,
 149                 .max_keysize            = CHACHA_KEY_SIZE,
 150                 .ivsize                 = XCHACHA_IV_SIZE,
 151                 .chunksize              = CHACHA_BLOCK_SIZE,
 152                 .walksize               = 5 * CHACHA_BLOCK_SIZE,
 153                 .setkey                 = crypto_chacha20_setkey,
 154                 .encrypt                = xchacha_neon,
 155                 .decrypt                = xchacha_neon,
 156         }, {
 157                 .base.cra_name          = "xchacha12",
 158                 .base.cra_driver_name   = "xchacha12-neon",
 159                 .base.cra_priority      = 300,
 160                 .base.cra_blocksize     = 1,
 161                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 162                 .base.cra_module        = THIS_MODULE,
 163
 164                 .min_keysize            = CHACHA_KEY_SIZE,
 165                 .max_keysize            = CHACHA_KEY_SIZE,
 166                 .ivsize                 = XCHACHA_IV_SIZE,
 167                 .chunksize              = CHACHA_BLOCK_SIZE,
 168                 .walksize               = 5 * CHACHA_BLOCK_SIZE,
 169                 .setkey                 = crypto_chacha12_setkey,
 170                 .encrypt                = xchacha_neon,
 171                 .decrypt                = xchacha_neon,
 172         }
 173 };
 174
 175 static int __init chacha_simd_mod_init(void)
 176 {
 177         if (!(elf_hwcap & HWCAP_ASIMD))
 178                 return -ENODEV;
 179
 180         return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
 181 }
 182
 183 static void __exit chacha_simd_mod_fini(void)
 184 {
 185         crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
 186 }
 187
 188 module_init(chacha_simd_mod_init);
 189 module_exit(chacha_simd_mod_fini);
 190
 191 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)");
 192 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 193 MODULE_LICENSE("GPL v2");
 194 MODULE_ALIAS_CRYPTO("chacha20");
 195 MODULE_ALIAS_CRYPTO("chacha20-neon");
 196 MODULE_ALIAS_CRYPTO("xchacha20");
 197 MODULE_ALIAS_CRYPTO("xchacha20-neon");
 198 MODULE_ALIAS_CRYPTO("xchacha12");
 199 MODULE_ALIAS_CRYPTO("xchacha12-neon");