arch/arm/crypto/crc32-ce-glue.c

   1 /*
   2  * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
   3  *
   4  * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <linux/cpufeature.h>
  12 #include <linux/crc32.h>
  13 #include <linux/init.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/string.h>
  17
  18 #include <crypto/internal/hash.h>
  19 #include <crypto/internal/simd.h>
  20
  21 #include <asm/hwcap.h>
  22 #include <asm/neon.h>
  23 #include <asm/simd.h>
  24 #include <asm/unaligned.h>
  25
  26 #define PMULL_MIN_LEN           64L     /* minimum size of buffer
  27                                          * for crc32_pmull_le_16 */
  28 #define SCALE_F                 16L     /* size of NEON register */
  29
  30 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
  31 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
  32
  33 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
  34 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
  35
  36 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len);
  37 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len);
  38
  39 static int crc32_cra_init(struct crypto_tfm *tfm)
  40 {
  41         u32 *key = crypto_tfm_ctx(tfm);
  42
  43         *key = 0;
  44         return 0;
  45 }
  46
  47 static int crc32c_cra_init(struct crypto_tfm *tfm)
  48 {
  49         u32 *key = crypto_tfm_ctx(tfm);
  50
  51         *key = ~0;
  52         return 0;
  53 }
  54
  55 static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
  56                         unsigned int keylen)
  57 {
  58         u32 *mctx = crypto_shash_ctx(hash);
  59
  60         if (keylen != sizeof(u32)) {
  61                 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
  62                 return -EINVAL;
  63         }
  64         *mctx = le32_to_cpup((__le32 *)key);
  65         return 0;
  66 }
  67
  68 static int crc32_init(struct shash_desc *desc)
  69 {
  70         u32 *mctx = crypto_shash_ctx(desc->tfm);
  71         u32 *crc = shash_desc_ctx(desc);
  72
  73         *crc = *mctx;
  74         return 0;
  75 }
  76
  77 static int crc32_update(struct shash_desc *desc, const u8 *data,
  78                         unsigned int length)
  79 {
  80         u32 *crc = shash_desc_ctx(desc);
  81
  82         *crc = crc32_armv8_le(*crc, data, length);
  83         return 0;
  84 }
  85
  86 static int crc32c_update(struct shash_desc *desc, const u8 *data,
  87                          unsigned int length)
  88 {
  89         u32 *crc = shash_desc_ctx(desc);
  90
  91         *crc = crc32c_armv8_le(*crc, data, length);
  92         return 0;
  93 }
  94
  95 static int crc32_final(struct shash_desc *desc, u8 *out)
  96 {
  97         u32 *crc = shash_desc_ctx(desc);
  98
  99         put_unaligned_le32(*crc, out);
 100         return 0;
 101 }
 102
 103 static int crc32c_final(struct shash_desc *desc, u8 *out)
 104 {
 105         u32 *crc = shash_desc_ctx(desc);
 106
 107         put_unaligned_le32(~*crc, out);
 108         return 0;
 109 }
 110
 111 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
 112                               unsigned int length)
 113 {
 114         u32 *crc = shash_desc_ctx(desc);
 115         unsigned int l;
 116
 117         if (crypto_simd_usable()) {
 118                 if ((u32)data % SCALE_F) {
 119                         l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
 120
 121                         *crc = fallback_crc32(*crc, data, l);
 122
 123                         data += l;
 124                         length -= l;
 125                 }
 126
 127                 if (length >= PMULL_MIN_LEN) {
 128                         l = round_down(length, SCALE_F);
 129
 130                         kernel_neon_begin();
 131                         *crc = crc32_pmull_le(data, l, *crc);
 132                         kernel_neon_end();
 133
 134                         data += l;
 135                         length -= l;
 136                 }
 137         }
 138
 139         if (length > 0)
 140                 *crc = fallback_crc32(*crc, data, length);
 141
 142         return 0;
 143 }
 144
 145 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
 146                                unsigned int length)
 147 {
 148         u32 *crc = shash_desc_ctx(desc);
 149         unsigned int l;
 150
 151         if (crypto_simd_usable()) {
 152                 if ((u32)data % SCALE_F) {
 153                         l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
 154
 155                         *crc = fallback_crc32c(*crc, data, l);
 156
 157                         data += l;
 158                         length -= l;
 159                 }
 160
 161                 if (length >= PMULL_MIN_LEN) {
 162                         l = round_down(length, SCALE_F);
 163
 164                         kernel_neon_begin();
 165                         *crc = crc32c_pmull_le(data, l, *crc);
 166                         kernel_neon_end();
 167
 168                         data += l;
 169                         length -= l;
 170                 }
 171         }
 172
 173         if (length > 0)
 174                 *crc = fallback_crc32c(*crc, data, length);
 175
 176         return 0;
 177 }
 178
 179 static struct shash_alg crc32_pmull_algs[] = { {
 180         .setkey                 = crc32_setkey,
 181         .init                   = crc32_init,
 182         .update                 = crc32_update,
 183         .final                  = crc32_final,
 184         .descsize               = sizeof(u32),
 185         .digestsize             = sizeof(u32),
 186
 187         .base.cra_ctxsize       = sizeof(u32),
 188         .base.cra_init          = crc32_cra_init,
 189         .base.cra_name          = "crc32",
 190         .base.cra_driver_name   = "crc32-arm-ce",
 191         .base.cra_priority      = 200,
 192         .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
 193         .base.cra_blocksize     = 1,
 194         .base.cra_module        = THIS_MODULE,
 195 }, {
 196         .setkey                 = crc32_setkey,
 197         .init                   = crc32_init,
 198         .update                 = crc32c_update,
 199         .final                  = crc32c_final,
 200         .descsize               = sizeof(u32),
 201         .digestsize             = sizeof(u32),
 202
 203         .base.cra_ctxsize       = sizeof(u32),
 204         .base.cra_init          = crc32c_cra_init,
 205         .base.cra_name          = "crc32c",
 206         .base.cra_driver_name   = "crc32c-arm-ce",
 207         .base.cra_priority      = 200,
 208         .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
 209         .base.cra_blocksize     = 1,
 210         .base.cra_module        = THIS_MODULE,
 211 } };
 212
 213 static int __init crc32_pmull_mod_init(void)
 214 {
 215         if (elf_hwcap2 & HWCAP2_PMULL) {
 216                 crc32_pmull_algs[0].update = crc32_pmull_update;
 217                 crc32_pmull_algs[1].update = crc32c_pmull_update;
 218
 219                 if (elf_hwcap2 & HWCAP2_CRC32) {
 220                         fallback_crc32 = crc32_armv8_le;
 221                         fallback_crc32c = crc32c_armv8_le;
 222                 } else {
 223                         fallback_crc32 = crc32_le;
 224                         fallback_crc32c = __crc32c_le;
 225                 }
 226         } else if (!(elf_hwcap2 & HWCAP2_CRC32)) {
 227                 return -ENODEV;
 228         }
 229
 230         return crypto_register_shashes(crc32_pmull_algs,
 231                                        ARRAY_SIZE(crc32_pmull_algs));
 232 }
 233
 234 static void __exit crc32_pmull_mod_exit(void)
 235 {
 236         crypto_unregister_shashes(crc32_pmull_algs,
 237                                   ARRAY_SIZE(crc32_pmull_algs));
 238 }
 239
 240 static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = {
 241         { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
 242 };
 243 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
 244
 245 module_init(crc32_pmull_mod_init);
 246 module_exit(crc32_pmull_mod_exit);
 247
 248 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 249 MODULE_LICENSE("GPL v2");
 250 MODULE_ALIAS_CRYPTO("crc32");
 251 MODULE_ALIAS_CRYPTO("crc32c");