4 * Glue code for the SHA256 Secure Hash Algorithm assembler
5 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
7 * This file is based on sha256_generic.c
9 * Copyright (C) 2013 Intel Corporation.
12 * Tim Chen <tim.c.chen@linux.intel.com>
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option)
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
32 #include <crypto/internal/hash.h>
33 #include <crypto/internal/simd.h>
34 #include <linux/init.h>
35 #include <linux/module.h>
37 #include <linux/cryptohash.h>
38 #include <linux/types.h>
39 #include <crypto/sha.h>
40 #include <crypto/sha256_base.h>
41 #include <linux/string.h>
44 asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
45 const u8 *data, int blocks);
47 static int _sha256_update(struct shash_desc *desc, const u8 *data,
48 unsigned int len, sha256_block_fn *sha256_xform)
50 struct sha256_state *sctx = shash_desc_ctx(desc);
52 if (!crypto_simd_usable() ||
53 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
54 return crypto_sha256_update(desc, data, len);
57 * Make sure struct sha256_state begins directly with the SHA256
58 * 256-bit internal state, as this is what the asm functions expect.
60 BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
63 sha256_base_do_update(desc, data, len, sha256_xform);
69 static int sha256_finup(struct shash_desc *desc, const u8 *data,
70 unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
72 if (!crypto_simd_usable())
73 return crypto_sha256_finup(desc, data, len, out);
77 sha256_base_do_update(desc, data, len, sha256_xform);
78 sha256_base_do_finalize(desc, sha256_xform);
81 return sha256_base_finish(desc, out);
84 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
87 return _sha256_update(desc, data, len, sha256_transform_ssse3);
90 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
91 unsigned int len, u8 *out)
93 return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
96 /* Add padding and return the message digest. */
97 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
99 return sha256_ssse3_finup(desc, NULL, 0, out);
102 static struct shash_alg sha256_ssse3_algs[] = { {
103 .digestsize = SHA256_DIGEST_SIZE,
104 .init = sha256_base_init,
105 .update = sha256_ssse3_update,
106 .final = sha256_ssse3_final,
107 .finup = sha256_ssse3_finup,
108 .descsize = sizeof(struct sha256_state),
110 .cra_name = "sha256",
111 .cra_driver_name = "sha256-ssse3",
113 .cra_blocksize = SHA256_BLOCK_SIZE,
114 .cra_module = THIS_MODULE,
117 .digestsize = SHA224_DIGEST_SIZE,
118 .init = sha224_base_init,
119 .update = sha256_ssse3_update,
120 .final = sha256_ssse3_final,
121 .finup = sha256_ssse3_finup,
122 .descsize = sizeof(struct sha256_state),
124 .cra_name = "sha224",
125 .cra_driver_name = "sha224-ssse3",
127 .cra_blocksize = SHA224_BLOCK_SIZE,
128 .cra_module = THIS_MODULE,
132 static int register_sha256_ssse3(void)
134 if (boot_cpu_has(X86_FEATURE_SSSE3))
135 return crypto_register_shashes(sha256_ssse3_algs,
136 ARRAY_SIZE(sha256_ssse3_algs));
140 static void unregister_sha256_ssse3(void)
142 if (boot_cpu_has(X86_FEATURE_SSSE3))
143 crypto_unregister_shashes(sha256_ssse3_algs,
144 ARRAY_SIZE(sha256_ssse3_algs));
147 asmlinkage void sha256_transform_avx(struct sha256_state *state,
148 const u8 *data, int blocks);
150 static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
153 return _sha256_update(desc, data, len, sha256_transform_avx);
156 static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
157 unsigned int len, u8 *out)
159 return sha256_finup(desc, data, len, out, sha256_transform_avx);
162 static int sha256_avx_final(struct shash_desc *desc, u8 *out)
164 return sha256_avx_finup(desc, NULL, 0, out);
167 static struct shash_alg sha256_avx_algs[] = { {
168 .digestsize = SHA256_DIGEST_SIZE,
169 .init = sha256_base_init,
170 .update = sha256_avx_update,
171 .final = sha256_avx_final,
172 .finup = sha256_avx_finup,
173 .descsize = sizeof(struct sha256_state),
175 .cra_name = "sha256",
176 .cra_driver_name = "sha256-avx",
178 .cra_blocksize = SHA256_BLOCK_SIZE,
179 .cra_module = THIS_MODULE,
182 .digestsize = SHA224_DIGEST_SIZE,
183 .init = sha224_base_init,
184 .update = sha256_avx_update,
185 .final = sha256_avx_final,
186 .finup = sha256_avx_finup,
187 .descsize = sizeof(struct sha256_state),
189 .cra_name = "sha224",
190 .cra_driver_name = "sha224-avx",
192 .cra_blocksize = SHA224_BLOCK_SIZE,
193 .cra_module = THIS_MODULE,
197 static bool avx_usable(void)
199 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
200 if (boot_cpu_has(X86_FEATURE_AVX))
201 pr_info("AVX detected but unusable.\n");
208 static int register_sha256_avx(void)
211 return crypto_register_shashes(sha256_avx_algs,
212 ARRAY_SIZE(sha256_avx_algs));
216 static void unregister_sha256_avx(void)
219 crypto_unregister_shashes(sha256_avx_algs,
220 ARRAY_SIZE(sha256_avx_algs));
223 #if defined(CONFIG_AS_AVX2)
224 asmlinkage void sha256_transform_rorx(struct sha256_state *state,
225 const u8 *data, int blocks);
227 static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
230 return _sha256_update(desc, data, len, sha256_transform_rorx);
233 static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
234 unsigned int len, u8 *out)
236 return sha256_finup(desc, data, len, out, sha256_transform_rorx);
239 static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
241 return sha256_avx2_finup(desc, NULL, 0, out);
244 static struct shash_alg sha256_avx2_algs[] = { {
245 .digestsize = SHA256_DIGEST_SIZE,
246 .init = sha256_base_init,
247 .update = sha256_avx2_update,
248 .final = sha256_avx2_final,
249 .finup = sha256_avx2_finup,
250 .descsize = sizeof(struct sha256_state),
252 .cra_name = "sha256",
253 .cra_driver_name = "sha256-avx2",
255 .cra_blocksize = SHA256_BLOCK_SIZE,
256 .cra_module = THIS_MODULE,
259 .digestsize = SHA224_DIGEST_SIZE,
260 .init = sha224_base_init,
261 .update = sha256_avx2_update,
262 .final = sha256_avx2_final,
263 .finup = sha256_avx2_finup,
264 .descsize = sizeof(struct sha256_state),
266 .cra_name = "sha224",
267 .cra_driver_name = "sha224-avx2",
269 .cra_blocksize = SHA224_BLOCK_SIZE,
270 .cra_module = THIS_MODULE,
274 static bool avx2_usable(void)
276 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
277 boot_cpu_has(X86_FEATURE_BMI2))
283 static int register_sha256_avx2(void)
286 return crypto_register_shashes(sha256_avx2_algs,
287 ARRAY_SIZE(sha256_avx2_algs));
291 static void unregister_sha256_avx2(void)
294 crypto_unregister_shashes(sha256_avx2_algs,
295 ARRAY_SIZE(sha256_avx2_algs));
299 static inline int register_sha256_avx2(void) { return 0; }
300 static inline void unregister_sha256_avx2(void) { }
303 #ifdef CONFIG_AS_SHA256_NI
304 asmlinkage void sha256_ni_transform(struct sha256_state *digest,
305 const u8 *data, int rounds);
307 static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
310 return _sha256_update(desc, data, len, sha256_ni_transform);
313 static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
314 unsigned int len, u8 *out)
316 return sha256_finup(desc, data, len, out, sha256_ni_transform);
319 static int sha256_ni_final(struct shash_desc *desc, u8 *out)
321 return sha256_ni_finup(desc, NULL, 0, out);
324 static struct shash_alg sha256_ni_algs[] = { {
325 .digestsize = SHA256_DIGEST_SIZE,
326 .init = sha256_base_init,
327 .update = sha256_ni_update,
328 .final = sha256_ni_final,
329 .finup = sha256_ni_finup,
330 .descsize = sizeof(struct sha256_state),
332 .cra_name = "sha256",
333 .cra_driver_name = "sha256-ni",
335 .cra_blocksize = SHA256_BLOCK_SIZE,
336 .cra_module = THIS_MODULE,
339 .digestsize = SHA224_DIGEST_SIZE,
340 .init = sha224_base_init,
341 .update = sha256_ni_update,
342 .final = sha256_ni_final,
343 .finup = sha256_ni_finup,
344 .descsize = sizeof(struct sha256_state),
346 .cra_name = "sha224",
347 .cra_driver_name = "sha224-ni",
349 .cra_blocksize = SHA224_BLOCK_SIZE,
350 .cra_module = THIS_MODULE,
354 static int register_sha256_ni(void)
356 if (boot_cpu_has(X86_FEATURE_SHA_NI))
357 return crypto_register_shashes(sha256_ni_algs,
358 ARRAY_SIZE(sha256_ni_algs));
362 static void unregister_sha256_ni(void)
364 if (boot_cpu_has(X86_FEATURE_SHA_NI))
365 crypto_unregister_shashes(sha256_ni_algs,
366 ARRAY_SIZE(sha256_ni_algs));
370 static inline int register_sha256_ni(void) { return 0; }
371 static inline void unregister_sha256_ni(void) { }
374 static int __init sha256_ssse3_mod_init(void)
376 if (register_sha256_ssse3())
379 if (register_sha256_avx()) {
380 unregister_sha256_ssse3();
384 if (register_sha256_avx2()) {
385 unregister_sha256_avx();
386 unregister_sha256_ssse3();
390 if (register_sha256_ni()) {
391 unregister_sha256_avx2();
392 unregister_sha256_avx();
393 unregister_sha256_ssse3();
402 static void __exit sha256_ssse3_mod_fini(void)
404 unregister_sha256_ni();
405 unregister_sha256_avx2();
406 unregister_sha256_avx();
407 unregister_sha256_ssse3();
410 module_init(sha256_ssse3_mod_init);
411 module_exit(sha256_ssse3_mod_fini);
413 MODULE_LICENSE("GPL");
414 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
416 MODULE_ALIAS_CRYPTO("sha256");
417 MODULE_ALIAS_CRYPTO("sha256-ssse3");
418 MODULE_ALIAS_CRYPTO("sha256-avx");
419 MODULE_ALIAS_CRYPTO("sha256-avx2");
420 MODULE_ALIAS_CRYPTO("sha224");
421 MODULE_ALIAS_CRYPTO("sha224-ssse3");
422 MODULE_ALIAS_CRYPTO("sha224-avx");
423 MODULE_ALIAS_CRYPTO("sha224-avx2");
424 #ifdef CONFIG_AS_SHA256_NI
425 MODULE_ALIAS_CRYPTO("sha256-ni");
426 MODULE_ALIAS_CRYPTO("sha224-ni");