crypto/testmgr.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Algorithm testing framework and tests.
   4  *
   5  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
   6  * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
   7  * Copyright (c) 2007 Nokia Siemens Networks
   8  * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
   9  * Copyright (c) 2019 Google LLC
  10  *
  11  * Updated RFC4106 AES-GCM testing.
  12  *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
  13  *             Adrian Hoban <adrian.hoban@intel.com>
  14  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  15  *             Tadeusz Struk (tadeusz.struk@intel.com)
  16  *    Copyright (c) 2010, Intel Corporation.
  17  */
  18
  19 #include <crypto/aead.h>
  20 #include <crypto/hash.h>
  21 #include <crypto/skcipher.h>
  22 #include <linux/err.h>
  23 #include <linux/fips.h>
  24 #include <linux/module.h>
  25 #include <linux/once.h>
  26 #include <linux/random.h>
  27 #include <linux/scatterlist.h>
  28 #include <linux/slab.h>
  29 #include <linux/string.h>
  30 #include <crypto/rng.h>
  31 #include <crypto/drbg.h>
  32 #include <crypto/akcipher.h>
  33 #include <crypto/kpp.h>
  34 #include <crypto/acompress.h>
  35 #include <crypto/internal/simd.h>
  36
  37 #include "internal.h"
  38
  39 static bool notests;
  40 module_param(notests, bool, 0644);
  41 MODULE_PARM_DESC(notests, "disable crypto self-tests");
  42
  43 static bool panic_on_fail;
  44 module_param(panic_on_fail, bool, 0444);
  45
  46 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
  47 static bool noextratests;
  48 module_param(noextratests, bool, 0644);
  49 MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests");
  50
  51 static unsigned int fuzz_iterations = 100;
  52 module_param(fuzz_iterations, uint, 0644);
  53 MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");
  54
  55 DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test);
  56 EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test);
  57 #endif
  58
  59 #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS
  60
  61 /* a perfect nop */
  62 int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
  63 {
  64         return 0;
  65 }
  66
  67 #else
  68
  69 #include "testmgr.h"
  70
  71 /*
  72  * Need slab memory for testing (size in number of pages).
  73  */
  74 #define XBUFSIZE        8
  75
  76 /*
  77 * Used by test_cipher()
  78 */
  79 #define ENCRYPT 1
  80 #define DECRYPT 0
  81
  82 struct aead_test_suite {
  83         const struct aead_testvec *vecs;
  84         unsigned int count;
  85
  86         /*
  87          * Set if trying to decrypt an inauthentic ciphertext with this
  88          * algorithm might result in EINVAL rather than EBADMSG, due to other
  89          * validation the algorithm does on the inputs such as length checks.
  90          */
  91         unsigned int einval_allowed : 1;
  92
  93         /*
  94          * Set if this algorithm requires that the IV be located at the end of
  95          * the AAD buffer, in addition to being given in the normal way.  The
  96          * behavior when the two IV copies differ is implementation-defined.
  97          */
  98         unsigned int aad_iv : 1;
  99 };
 100
 101 struct cipher_test_suite {
 102         const struct cipher_testvec *vecs;
 103         unsigned int count;
 104 };
 105
 106 struct comp_test_suite {
 107         struct {
 108                 const struct comp_testvec *vecs;
 109                 unsigned int count;
 110         } comp, decomp;
 111 };
 112
 113 struct hash_test_suite {
 114         const struct hash_testvec *vecs;
 115         unsigned int count;
 116 };
 117
 118 struct cprng_test_suite {
 119         const struct cprng_testvec *vecs;
 120         unsigned int count;
 121 };
 122
 123 struct drbg_test_suite {
 124         const struct drbg_testvec *vecs;
 125         unsigned int count;
 126 };
 127
 128 struct akcipher_test_suite {
 129         const struct akcipher_testvec *vecs;
 130         unsigned int count;
 131 };
 132
 133 struct kpp_test_suite {
 134         const struct kpp_testvec *vecs;
 135         unsigned int count;
 136 };
 137
 138 struct alg_test_desc {
 139         const char *alg;
 140         const char *generic_driver;
 141         int (*test)(const struct alg_test_desc *desc, const char *driver,
 142                     u32 type, u32 mask);
 143         int fips_allowed;       /* set if alg is allowed in fips mode */
 144
 145         union {
 146                 struct aead_test_suite aead;
 147                 struct cipher_test_suite cipher;
 148                 struct comp_test_suite comp;
 149                 struct hash_test_suite hash;
 150                 struct cprng_test_suite cprng;
 151                 struct drbg_test_suite drbg;
 152                 struct akcipher_test_suite akcipher;
 153                 struct kpp_test_suite kpp;
 154         } suite;
 155 };
 156
 157 static void hexdump(unsigned char *buf, unsigned int len)
 158 {
 159         print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
 160                         16, 1,
 161                         buf, len, false);
 162 }
 163
 164 static int __testmgr_alloc_buf(char *buf[XBUFSIZE], int order)
 165 {
 166         int i;
 167
 168         for (i = 0; i < XBUFSIZE; i++) {
 169                 buf[i] = (char *)__get_free_pages(GFP_KERNEL, order);
 170                 if (!buf[i])
 171                         goto err_free_buf;
 172         }
 173
 174         return 0;
 175
 176 err_free_buf:
 177         while (i-- > 0)
 178                 free_pages((unsigned long)buf[i], order);
 179
 180         return -ENOMEM;
 181 }
 182
 183 static int testmgr_alloc_buf(char *buf[XBUFSIZE])
 184 {
 185         return __testmgr_alloc_buf(buf, 0);
 186 }
 187
 188 static void __testmgr_free_buf(char *buf[XBUFSIZE], int order)
 189 {
 190         int i;
 191
 192         for (i = 0; i < XBUFSIZE; i++)
 193                 free_pages((unsigned long)buf[i], order);
 194 }
 195
 196 static void testmgr_free_buf(char *buf[XBUFSIZE])
 197 {
 198         __testmgr_free_buf(buf, 0);
 199 }
 200
 201 #define TESTMGR_POISON_BYTE     0xfe
 202 #define TESTMGR_POISON_LEN      16
 203
 204 static inline void testmgr_poison(void *addr, size_t len)
 205 {
 206         memset(addr, TESTMGR_POISON_BYTE, len);
 207 }
 208
 209 /* Is the memory region still fully poisoned? */
 210 static inline bool testmgr_is_poison(const void *addr, size_t len)
 211 {
 212         return memchr_inv(addr, TESTMGR_POISON_BYTE, len) == NULL;
 213 }
 214
 215 /* flush type for hash algorithms */
 216 enum flush_type {
 217         /* merge with update of previous buffer(s) */
 218         FLUSH_TYPE_NONE = 0,
 219
 220         /* update with previous buffer(s) before doing this one */
 221         FLUSH_TYPE_FLUSH,
 222
 223         /* likewise, but also export and re-import the intermediate state */
 224         FLUSH_TYPE_REIMPORT,
 225 };
 226
 227 /* finalization function for hash algorithms */
 228 enum finalization_type {
 229         FINALIZATION_TYPE_FINAL,        /* use final() */
 230         FINALIZATION_TYPE_FINUP,        /* use finup() */
 231         FINALIZATION_TYPE_DIGEST,       /* use digest() */
 232 };
 233
 234 #define TEST_SG_TOTAL   10000
 235
 236 /**
 237  * struct test_sg_division - description of a scatterlist entry
 238  *
 239  * This struct describes one entry of a scatterlist being constructed to check a
 240  * crypto test vector.
 241  *
 242  * @proportion_of_total: length of this chunk relative to the total length,
 243  *                       given as a proportion out of TEST_SG_TOTAL so that it
 244  *                       scales to fit any test vector
 245  * @offset: byte offset into a 2-page buffer at which this chunk will start
 246  * @offset_relative_to_alignmask: if true, add the algorithm's alignmask to the
 247  *                                @offset
 248  * @flush_type: for hashes, whether an update() should be done now vs.
 249  *              continuing to accumulate data
 250  * @nosimd: if doing the pending update(), do it with SIMD disabled?
 251  */
 252 struct test_sg_division {
 253         unsigned int proportion_of_total;
 254         unsigned int offset;
 255         bool offset_relative_to_alignmask;
 256         enum flush_type flush_type;
 257         bool nosimd;
 258 };
 259
 260 /**
 261  * struct testvec_config - configuration for testing a crypto test vector
 262  *
 263  * This struct describes the data layout and other parameters with which each
 264  * crypto test vector can be tested.
 265  *
 266  * @name: name of this config, logged for debugging purposes if a test fails
 267  * @inplace: operate on the data in-place, if applicable for the algorithm type?
 268  * @req_flags: extra request_flags, e.g. CRYPTO_TFM_REQ_MAY_SLEEP
 269  * @src_divs: description of how to arrange the source scatterlist
 270  * @dst_divs: description of how to arrange the dst scatterlist, if applicable
 271  *            for the algorithm type.  Defaults to @src_divs if unset.
 272  * @iv_offset: misalignment of the IV in the range [0..MAX_ALGAPI_ALIGNMASK+1],
 273  *             where 0 is aligned to a 2*(MAX_ALGAPI_ALIGNMASK+1) byte boundary
 274  * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 275  *                                   the @iv_offset
 276  * @key_offset: misalignment of the key, where 0 is default alignment
 277  * @key_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 278  *                                    the @key_offset
 279  * @finalization_type: what finalization function to use for hashes
 280  * @nosimd: execute with SIMD disabled?  Requires !CRYPTO_TFM_REQ_MAY_SLEEP.
 281  */
 282 struct testvec_config {
 283         const char *name;
 284         bool inplace;
 285         u32 req_flags;
 286         struct test_sg_division src_divs[XBUFSIZE];
 287         struct test_sg_division dst_divs[XBUFSIZE];
 288         unsigned int iv_offset;
 289         unsigned int key_offset;
 290         bool iv_offset_relative_to_alignmask;
 291         bool key_offset_relative_to_alignmask;
 292         enum finalization_type finalization_type;
 293         bool nosimd;
 294 };
 295
 296 #define TESTVEC_CONFIG_NAMELEN  192
 297
 298 /*
 299  * The following are the lists of testvec_configs to test for each algorithm
 300  * type when the basic crypto self-tests are enabled, i.e. when
 301  * CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is unset.  They aim to provide good test
 302  * coverage, while keeping the test time much shorter than the full fuzz tests
 303  * so that the basic tests can be enabled in a wider range of circumstances.
 304  */
 305
 306 /* Configs for skciphers and aeads */
 307 static const struct testvec_config default_cipher_testvec_configs[] = {
 308         {
 309                 .name = "in-place",
 310                 .inplace = true,
 311                 .src_divs = { { .proportion_of_total = 10000 } },
 312         }, {
 313                 .name = "out-of-place",
 314                 .src_divs = { { .proportion_of_total = 10000 } },
 315         }, {
 316                 .name = "unaligned buffer, offset=1",
 317                 .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 318                 .iv_offset = 1,
 319                 .key_offset = 1,
 320         }, {
 321                 .name = "buffer aligned only to alignmask",
 322                 .src_divs = {
 323                         {
 324                                 .proportion_of_total = 10000,
 325                                 .offset = 1,
 326                                 .offset_relative_to_alignmask = true,
 327                         },
 328                 },
 329                 .iv_offset = 1,
 330                 .iv_offset_relative_to_alignmask = true,
 331                 .key_offset = 1,
 332                 .key_offset_relative_to_alignmask = true,
 333         }, {
 334                 .name = "two even aligned splits",
 335                 .src_divs = {
 336                         { .proportion_of_total = 5000 },
 337                         { .proportion_of_total = 5000 },
 338                 },
 339         }, {
 340                 .name = "uneven misaligned splits, may sleep",
 341                 .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 342                 .src_divs = {
 343                         { .proportion_of_total = 1900, .offset = 33 },
 344                         { .proportion_of_total = 3300, .offset = 7  },
 345                         { .proportion_of_total = 4800, .offset = 18 },
 346                 },
 347                 .iv_offset = 3,
 348                 .key_offset = 3,
 349         }, {
 350                 .name = "misaligned splits crossing pages, inplace",
 351                 .inplace = true,
 352                 .src_divs = {
 353                         {
 354                                 .proportion_of_total = 7500,
 355                                 .offset = PAGE_SIZE - 32
 356                         }, {
 357                                 .proportion_of_total = 2500,
 358                                 .offset = PAGE_SIZE - 7
 359                         },
 360                 },
 361         }
 362 };
 363
 364 static const struct testvec_config default_hash_testvec_configs[] = {
 365         {
 366                 .name = "init+update+final aligned buffer",
 367                 .src_divs = { { .proportion_of_total = 10000 } },
 368                 .finalization_type = FINALIZATION_TYPE_FINAL,
 369         }, {
 370                 .name = "init+finup aligned buffer",
 371                 .src_divs = { { .proportion_of_total = 10000 } },
 372                 .finalization_type = FINALIZATION_TYPE_FINUP,
 373         }, {
 374                 .name = "digest aligned buffer",
 375                 .src_divs = { { .proportion_of_total = 10000 } },
 376                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 377         }, {
 378                 .name = "init+update+final misaligned buffer",
 379                 .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 380                 .finalization_type = FINALIZATION_TYPE_FINAL,
 381                 .key_offset = 1,
 382         }, {
 383                 .name = "digest buffer aligned only to alignmask",
 384                 .src_divs = {
 385                         {
 386                                 .proportion_of_total = 10000,
 387                                 .offset = 1,
 388                                 .offset_relative_to_alignmask = true,
 389                         },
 390                 },
 391                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 392                 .key_offset = 1,
 393                 .key_offset_relative_to_alignmask = true,
 394         }, {
 395                 .name = "init+update+update+final two even splits",
 396                 .src_divs = {
 397                         { .proportion_of_total = 5000 },
 398                         {
 399                                 .proportion_of_total = 5000,
 400                                 .flush_type = FLUSH_TYPE_FLUSH,
 401                         },
 402                 },
 403                 .finalization_type = FINALIZATION_TYPE_FINAL,
 404         }, {
 405                 .name = "digest uneven misaligned splits, may sleep",
 406                 .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 407                 .src_divs = {
 408                         { .proportion_of_total = 1900, .offset = 33 },
 409                         { .proportion_of_total = 3300, .offset = 7  },
 410                         { .proportion_of_total = 4800, .offset = 18 },
 411                 },
 412                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 413         }, {
 414                 .name = "digest misaligned splits crossing pages",
 415                 .src_divs = {
 416                         {
 417                                 .proportion_of_total = 7500,
 418                                 .offset = PAGE_SIZE - 32,
 419                         }, {
 420                                 .proportion_of_total = 2500,
 421                                 .offset = PAGE_SIZE - 7,
 422                         },
 423                 },
 424                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 425         }, {
 426                 .name = "import/export",
 427                 .src_divs = {
 428                         {
 429                                 .proportion_of_total = 6500,
 430                                 .flush_type = FLUSH_TYPE_REIMPORT,
 431                         }, {
 432                                 .proportion_of_total = 3500,
 433                                 .flush_type = FLUSH_TYPE_REIMPORT,
 434                         },
 435                 },
 436                 .finalization_type = FINALIZATION_TYPE_FINAL,
 437         }
 438 };
 439
 440 static unsigned int count_test_sg_divisions(const struct test_sg_division *divs)
 441 {
 442         unsigned int remaining = TEST_SG_TOTAL;
 443         unsigned int ndivs = 0;
 444
 445         do {
 446                 remaining -= divs[ndivs++].proportion_of_total;
 447         } while (remaining);
 448
 449         return ndivs;
 450 }
 451
 452 #define SGDIVS_HAVE_FLUSHES     BIT(0)
 453 #define SGDIVS_HAVE_NOSIMD      BIT(1)
 454
 455 static bool valid_sg_divisions(const struct test_sg_division *divs,
 456                                unsigned int count, int *flags_ret)
 457 {
 458         unsigned int total = 0;
 459         unsigned int i;
 460
 461         for (i = 0; i < count && total != TEST_SG_TOTAL; i++) {
 462                 if (divs[i].proportion_of_total <= 0 ||
 463                     divs[i].proportion_of_total > TEST_SG_TOTAL - total)
 464                         return false;
 465                 total += divs[i].proportion_of_total;
 466                 if (divs[i].flush_type != FLUSH_TYPE_NONE)
 467                         *flags_ret |= SGDIVS_HAVE_FLUSHES;
 468                 if (divs[i].nosimd)
 469                         *flags_ret |= SGDIVS_HAVE_NOSIMD;
 470         }
 471         return total == TEST_SG_TOTAL &&
 472                 memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL;
 473 }
 474
 475 /*
 476  * Check whether the given testvec_config is valid.  This isn't strictly needed
 477  * since every testvec_config should be valid, but check anyway so that people
 478  * don't unknowingly add broken configs that don't do what they wanted.
 479  */
 480 static bool valid_testvec_config(const struct testvec_config *cfg)
 481 {
 482         int flags = 0;
 483
 484         if (cfg->name == NULL)
 485                 return false;
 486
 487         if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs),
 488                                 &flags))
 489                 return false;
 490
 491         if (cfg->dst_divs[0].proportion_of_total) {
 492                 if (!valid_sg_divisions(cfg->dst_divs,
 493                                         ARRAY_SIZE(cfg->dst_divs), &flags))
 494                         return false;
 495         } else {
 496                 if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs)))
 497                         return false;
 498                 /* defaults to dst_divs=src_divs */
 499         }
 500
 501         if (cfg->iv_offset +
 502             (cfg->iv_offset_relative_to_alignmask ? MAX_ALGAPI_ALIGNMASK : 0) >
 503             MAX_ALGAPI_ALIGNMASK + 1)
 504                 return false;
 505
 506         if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) &&
 507             cfg->finalization_type == FINALIZATION_TYPE_DIGEST)
 508                 return false;
 509
 510         if ((cfg->nosimd || (flags & SGDIVS_HAVE_NOSIMD)) &&
 511             (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP))
 512                 return false;
 513
 514         return true;
 515 }
 516
 517 struct test_sglist {
 518         char *bufs[XBUFSIZE];
 519         struct scatterlist sgl[XBUFSIZE];
 520         struct scatterlist sgl_saved[XBUFSIZE];
 521         struct scatterlist *sgl_ptr;
 522         unsigned int nents;
 523 };
 524
 525 static int init_test_sglist(struct test_sglist *tsgl)
 526 {
 527         return __testmgr_alloc_buf(tsgl->bufs, 1 /* two pages per buffer */);
 528 }
 529
 530 static void destroy_test_sglist(struct test_sglist *tsgl)
 531 {
 532         return __testmgr_free_buf(tsgl->bufs, 1 /* two pages per buffer */);
 533 }
 534
 535 /**
 536  * build_test_sglist() - build a scatterlist for a crypto test
 537  *
 538  * @tsgl: the scatterlist to build.  @tsgl->bufs[] contains an array of 2-page
 539  *        buffers which the scatterlist @tsgl->sgl[] will be made to point into.
 540  * @divs: the layout specification on which the scatterlist will be based
 541  * @alignmask: the algorithm's alignmask
 542  * @total_len: the total length of the scatterlist to build in bytes
 543  * @data: if non-NULL, the buffers will be filled with this data until it ends.
 544  *        Otherwise the buffers will be poisoned.  In both cases, some bytes
 545  *        past the end of each buffer will be poisoned to help detect overruns.
 546  * @out_divs: if non-NULL, the test_sg_division to which each scatterlist entry
 547  *            corresponds will be returned here.  This will match @divs except
 548  *            that divisions resolving to a length of 0 are omitted as they are
 549  *            not included in the scatterlist.
 550  *
 551  * Return: 0 or a -errno value
 552  */
 553 static int build_test_sglist(struct test_sglist *tsgl,
 554                              const struct test_sg_division *divs,
 555                              const unsigned int alignmask,
 556                              const unsigned int total_len,
 557                              struct iov_iter *data,
 558                              const struct test_sg_division *out_divs[XBUFSIZE])
 559 {
 560         struct {
 561                 const struct test_sg_division *div;
 562                 size_t length;
 563         } partitions[XBUFSIZE];
 564         const unsigned int ndivs = count_test_sg_divisions(divs);
 565         unsigned int len_remaining = total_len;
 566         unsigned int i;
 567
 568         BUILD_BUG_ON(ARRAY_SIZE(partitions) != ARRAY_SIZE(tsgl->sgl));
 569         if (WARN_ON(ndivs > ARRAY_SIZE(partitions)))
 570                 return -EINVAL;
 571
 572         /* Calculate the (div, length) pairs */
 573         tsgl->nents = 0;
 574         for (i = 0; i < ndivs; i++) {
 575                 unsigned int len_this_sg =
 576                         min(len_remaining,
 577                             (total_len * divs[i].proportion_of_total +
 578                              TEST_SG_TOTAL / 2) / TEST_SG_TOTAL);
 579
 580                 if (len_this_sg != 0) {
 581                         partitions[tsgl->nents].div = &divs[i];
 582                         partitions[tsgl->nents].length = len_this_sg;
 583                         tsgl->nents++;
 584                         len_remaining -= len_this_sg;
 585                 }
 586         }
 587         if (tsgl->nents == 0) {
 588                 partitions[tsgl->nents].div = &divs[0];
 589                 partitions[tsgl->nents].length = 0;
 590                 tsgl->nents++;
 591         }
 592         partitions[tsgl->nents - 1].length += len_remaining;
 593
 594         /* Set up the sgl entries and fill the data or poison */
 595         sg_init_table(tsgl->sgl, tsgl->nents);
 596         for (i = 0; i < tsgl->nents; i++) {
 597                 unsigned int offset = partitions[i].div->offset;
 598                 void *addr;
 599
 600                 if (partitions[i].div->offset_relative_to_alignmask)
 601                         offset += alignmask;
 602
 603                 while (offset + partitions[i].length + TESTMGR_POISON_LEN >
 604                        2 * PAGE_SIZE) {
 605                         if (WARN_ON(offset <= 0))
 606                                 return -EINVAL;
 607                         offset /= 2;
 608                 }
 609
 610                 addr = &tsgl->bufs[i][offset];
 611                 sg_set_buf(&tsgl->sgl[i], addr, partitions[i].length);
 612
 613                 if (out_divs)
 614                         out_divs[i] = partitions[i].div;
 615
 616                 if (data) {
 617                         size_t copy_len, copied;
 618
 619                         copy_len = min(partitions[i].length, data->count);
 620                         copied = copy_from_iter(addr, copy_len, data);
 621                         if (WARN_ON(copied != copy_len))
 622                                 return -EINVAL;
 623                         testmgr_poison(addr + copy_len, partitions[i].length +
 624                                        TESTMGR_POISON_LEN - copy_len);
 625                 } else {
 626                         testmgr_poison(addr, partitions[i].length +
 627                                        TESTMGR_POISON_LEN);
 628                 }
 629         }
 630
 631         sg_mark_end(&tsgl->sgl[tsgl->nents - 1]);
 632         tsgl->sgl_ptr = tsgl->sgl;
 633         memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0]));
 634         return 0;
 635 }
 636
 637 /*
 638  * Verify that a scatterlist crypto operation produced the correct output.
 639  *
 640  * @tsgl: scatterlist containing the actual output
 641  * @expected_output: buffer containing the expected output
 642  * @len_to_check: length of @expected_output in bytes
 643  * @unchecked_prefix_len: number of ignored bytes in @tsgl prior to real result
 644  * @check_poison: verify that the poison bytes after each chunk are intact?
 645  *
 646  * Return: 0 if correct, -EINVAL if incorrect, -EOVERFLOW if buffer overrun.
 647  */
 648 static int verify_correct_output(const struct test_sglist *tsgl,
 649                                  const char *expected_output,
 650                                  unsigned int len_to_check,
 651                                  unsigned int unchecked_prefix_len,
 652                                  bool check_poison)
 653 {
 654         unsigned int i;
 655
 656         for (i = 0; i < tsgl->nents; i++) {
 657                 struct scatterlist *sg = &tsgl->sgl_ptr[i];
 658                 unsigned int len = sg->length;
 659                 unsigned int offset = sg->offset;
 660                 const char *actual_output;
 661
 662                 if (unchecked_prefix_len) {
 663                         if (unchecked_prefix_len >= len) {
 664                                 unchecked_prefix_len -= len;
 665                                 continue;
 666                         }
 667                         offset += unchecked_prefix_len;
 668                         len -= unchecked_prefix_len;
 669                         unchecked_prefix_len = 0;
 670                 }
 671                 len = min(len, len_to_check);
 672                 actual_output = page_address(sg_page(sg)) + offset;
 673                 if (memcmp(expected_output, actual_output, len) != 0)
 674                         return -EINVAL;
 675                 if (check_poison &&
 676                     !testmgr_is_poison(actual_output + len, TESTMGR_POISON_LEN))
 677                         return -EOVERFLOW;
 678                 len_to_check -= len;
 679                 expected_output += len;
 680         }
 681         if (WARN_ON(len_to_check != 0))
 682                 return -EINVAL;
 683         return 0;
 684 }
 685
 686 static bool is_test_sglist_corrupted(const struct test_sglist *tsgl)
 687 {
 688         unsigned int i;
 689
 690         for (i = 0; i < tsgl->nents; i++) {
 691                 if (tsgl->sgl[i].page_link != tsgl->sgl_saved[i].page_link)
 692                         return true;
 693                 if (tsgl->sgl[i].offset != tsgl->sgl_saved[i].offset)
 694                         return true;
 695                 if (tsgl->sgl[i].length != tsgl->sgl_saved[i].length)
 696                         return true;
 697         }
 698         return false;
 699 }
 700
 701 struct cipher_test_sglists {
 702         struct test_sglist src;
 703         struct test_sglist dst;
 704 };
 705
 706 static struct cipher_test_sglists *alloc_cipher_test_sglists(void)
 707 {
 708         struct cipher_test_sglists *tsgls;
 709
 710         tsgls = kmalloc(sizeof(*tsgls), GFP_KERNEL);
 711         if (!tsgls)
 712                 return NULL;
 713
 714         if (init_test_sglist(&tsgls->src) != 0)
 715                 goto fail_kfree;
 716         if (init_test_sglist(&tsgls->dst) != 0)
 717                 goto fail_destroy_src;
 718
 719         return tsgls;
 720
 721 fail_destroy_src:
 722         destroy_test_sglist(&tsgls->src);
 723 fail_kfree:
 724         kfree(tsgls);
 725         return NULL;
 726 }
 727
 728 static void free_cipher_test_sglists(struct cipher_test_sglists *tsgls)
 729 {
 730         if (tsgls) {
 731                 destroy_test_sglist(&tsgls->src);
 732                 destroy_test_sglist(&tsgls->dst);
 733                 kfree(tsgls);
 734         }
 735 }
 736
 737 /* Build the src and dst scatterlists for an skcipher or AEAD test */
 738 static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls,
 739                                      const struct testvec_config *cfg,
 740                                      unsigned int alignmask,
 741                                      unsigned int src_total_len,
 742                                      unsigned int dst_total_len,
 743                                      const struct kvec *inputs,
 744                                      unsigned int nr_inputs)
 745 {
 746         struct iov_iter input;
 747         int err;
 748
 749         iov_iter_kvec(&input, WRITE, inputs, nr_inputs, src_total_len);
 750         err = build_test_sglist(&tsgls->src, cfg->src_divs, alignmask,
 751                                 cfg->inplace ?
 752                                         max(dst_total_len, src_total_len) :
 753                                         src_total_len,
 754                                 &input, NULL);
 755         if (err)
 756                 return err;
 757
 758         if (cfg->inplace) {
 759                 tsgls->dst.sgl_ptr = tsgls->src.sgl;
 760                 tsgls->dst.nents = tsgls->src.nents;
 761                 return 0;
 762         }
 763         return build_test_sglist(&tsgls->dst,
 764                                  cfg->dst_divs[0].proportion_of_total ?
 765                                         cfg->dst_divs : cfg->src_divs,
 766                                  alignmask, dst_total_len, NULL, NULL);
 767 }
 768
 769 /*
 770  * Support for testing passing a misaligned key to setkey():
 771  *
 772  * If cfg->key_offset is set, copy the key into a new buffer at that offset,
 773  * optionally adding alignmask.  Else, just use the key directly.
 774  */
 775 static int prepare_keybuf(const u8 *key, unsigned int ksize,
 776                           const struct testvec_config *cfg,
 777                           unsigned int alignmask,
 778                           const u8 **keybuf_ret, const u8 **keyptr_ret)
 779 {
 780         unsigned int key_offset = cfg->key_offset;
 781         u8 *keybuf = NULL, *keyptr = (u8 *)key;
 782
 783         if (key_offset != 0) {
 784                 if (cfg->key_offset_relative_to_alignmask)
 785                         key_offset += alignmask;
 786                 keybuf = kmalloc(key_offset + ksize, GFP_KERNEL);
 787                 if (!keybuf)
 788                         return -ENOMEM;
 789                 keyptr = keybuf + key_offset;
 790                 memcpy(keyptr, key, ksize);
 791         }
 792         *keybuf_ret = keybuf;
 793         *keyptr_ret = keyptr;
 794         return 0;
 795 }
 796
 797 /* Like setkey_f(tfm, key, ksize), but sometimes misalign the key */
 798 #define do_setkey(setkey_f, tfm, key, ksize, cfg, alignmask)            \
 799 ({                                                                      \
 800         const u8 *keybuf, *keyptr;                                      \
 801         int err;                                                        \
 802                                                                         \
 803         err = prepare_keybuf((key), (ksize), (cfg), (alignmask),        \
 804                              &keybuf, &keyptr);                         \
 805         if (err == 0) {                                                 \
 806                 err = setkey_f((tfm), keyptr, (ksize));                 \
 807                 kfree(keybuf);                                          \
 808         }                                                               \
 809         err;                                                            \
 810 })
 811
 812 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
 813
 814 /* Generate a random length in range [0, max_len], but prefer smaller values */
 815 static unsigned int generate_random_length(unsigned int max_len)
 816 {
 817         unsigned int len = prandom_u32() % (max_len + 1);
 818
 819         switch (prandom_u32() % 4) {
 820         case 0:
 821                 return len % 64;
 822         case 1:
 823                 return len % 256;
 824         case 2:
 825                 return len % 1024;
 826         default:
 827                 return len;
 828         }
 829 }
 830
 831 /* Flip a random bit in the given nonempty data buffer */
 832 static void flip_random_bit(u8 *buf, size_t size)
 833 {
 834         size_t bitpos;
 835
 836         bitpos = prandom_u32() % (size * 8);
 837         buf[bitpos / 8] ^= 1 << (bitpos % 8);
 838 }
 839
 840 /* Flip a random byte in the given nonempty data buffer */
 841 static void flip_random_byte(u8 *buf, size_t size)
 842 {
 843         buf[prandom_u32() % size] ^= 0xff;
 844 }
 845
 846 /* Sometimes make some random changes to the given nonempty data buffer */
 847 static void mutate_buffer(u8 *buf, size_t size)
 848 {
 849         size_t num_flips;
 850         size_t i;
 851
 852         /* Sometimes flip some bits */
 853         if (prandom_u32() % 4 == 0) {
 854                 num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size * 8);
 855                 for (i = 0; i < num_flips; i++)
 856                         flip_random_bit(buf, size);
 857         }
 858
 859         /* Sometimes flip some bytes */
 860         if (prandom_u32() % 4 == 0) {
 861                 num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size);
 862                 for (i = 0; i < num_flips; i++)
 863                         flip_random_byte(buf, size);
 864         }
 865 }
 866
 867 /* Randomly generate 'count' bytes, but sometimes make them "interesting" */
 868 static void generate_random_bytes(u8 *buf, size_t count)
 869 {
 870         u8 b;
 871         u8 increment;
 872         size_t i;
 873
 874         if (count == 0)
 875                 return;
 876
 877         switch (prandom_u32() % 8) { /* Choose a generation strategy */
 878         case 0:
 879         case 1:
 880                 /* All the same byte, plus optional mutations */
 881                 switch (prandom_u32() % 4) {
 882                 case 0:
 883                         b = 0x00;
 884                         break;
 885                 case 1:
 886                         b = 0xff;
 887                         break;
 888                 default:
 889                         b = (u8)prandom_u32();
 890                         break;
 891                 }
 892                 memset(buf, b, count);
 893                 mutate_buffer(buf, count);
 894                 break;
 895         case 2:
 896                 /* Ascending or descending bytes, plus optional mutations */
 897                 increment = (u8)prandom_u32();
 898                 b = (u8)prandom_u32();
 899                 for (i = 0; i < count; i++, b += increment)
 900                         buf[i] = b;
 901                 mutate_buffer(buf, count);
 902                 break;
 903         default:
 904                 /* Fully random bytes */
 905                 for (i = 0; i < count; i++)
 906                         buf[i] = (u8)prandom_u32();
 907         }
 908 }
 909
 910 static char *generate_random_sgl_divisions(struct test_sg_division *divs,
 911                                            size_t max_divs, char *p, char *end,
 912                                            bool gen_flushes, u32 req_flags)
 913 {
 914         struct test_sg_division *div = divs;
 915         unsigned int remaining = TEST_SG_TOTAL;
 916
 917         do {
 918                 unsigned int this_len;
 919                 const char *flushtype_str;
 920
 921                 if (div == &divs[max_divs - 1] || prandom_u32() % 2 == 0)
 922                         this_len = remaining;
 923                 else
 924                         this_len = 1 + (prandom_u32() % remaining);
 925                 div->proportion_of_total = this_len;
 926
 927                 if (prandom_u32() % 4 == 0)
 928                         div->offset = (PAGE_SIZE - 128) + (prandom_u32() % 128);
 929                 else if (prandom_u32() % 2 == 0)
 930                         div->offset = prandom_u32() % 32;
 931                 else
 932                         div->offset = prandom_u32() % PAGE_SIZE;
 933                 if (prandom_u32() % 8 == 0)
 934                         div->offset_relative_to_alignmask = true;
 935
 936                 div->flush_type = FLUSH_TYPE_NONE;
 937                 if (gen_flushes) {
 938                         switch (prandom_u32() % 4) {
 939                         case 0:
 940                                 div->flush_type = FLUSH_TYPE_REIMPORT;
 941                                 break;
 942                         case 1:
 943                                 div->flush_type = FLUSH_TYPE_FLUSH;
 944                                 break;
 945                         }
 946                 }
 947
 948                 if (div->flush_type != FLUSH_TYPE_NONE &&
 949                     !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
 950                     prandom_u32() % 2 == 0)
 951                         div->nosimd = true;
 952
 953                 switch (div->flush_type) {
 954                 case FLUSH_TYPE_FLUSH:
 955                         if (div->nosimd)
 956                                 flushtype_str = "<flush,nosimd>";
 957                         else
 958                                 flushtype_str = "<flush>";
 959                         break;
 960                 case FLUSH_TYPE_REIMPORT:
 961                         if (div->nosimd)
 962                                 flushtype_str = "<reimport,nosimd>";
 963                         else
 964                                 flushtype_str = "<reimport>";
 965                         break;
 966                 default:
 967                         flushtype_str = "";
 968                         break;
 969                 }
 970
 971                 BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */
 972                 p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str,
 973                                this_len / 100, this_len % 100,
 974                                div->offset_relative_to_alignmask ?
 975                                         "alignmask" : "",
 976                                div->offset, this_len == remaining ? "" : ", ");
 977                 remaining -= this_len;
 978                 div++;
 979         } while (remaining);
 980
 981         return p;
 982 }
 983
 984 /* Generate a random testvec_config for fuzz testing */
 985 static void generate_random_testvec_config(struct testvec_config *cfg,
 986                                            char *name, size_t max_namelen)
 987 {
 988         char *p = name;
 989         char * const end = name + max_namelen;
 990
 991         memset(cfg, 0, sizeof(*cfg));
 992
 993         cfg->name = name;
 994
 995         p += scnprintf(p, end - p, "random:");
 996
 997         if (prandom_u32() % 2 == 0) {
 998                 cfg->inplace = true;
 999                 p += scnprintf(p, end - p, " inplace");
1000         }
1001
1002         if (prandom_u32() % 2 == 0) {
1003                 cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP;
1004                 p += scnprintf(p, end - p, " may_sleep");
1005         }
1006
1007         switch (prandom_u32() % 4) {
1008         case 0:
1009                 cfg->finalization_type = FINALIZATION_TYPE_FINAL;
1010                 p += scnprintf(p, end - p, " use_final");
1011                 break;
1012         case 1:
1013                 cfg->finalization_type = FINALIZATION_TYPE_FINUP;
1014                 p += scnprintf(p, end - p, " use_finup");
1015                 break;
1016         default:
1017                 cfg->finalization_type = FINALIZATION_TYPE_DIGEST;
1018                 p += scnprintf(p, end - p, " use_digest");
1019                 break;
1020         }
1021
1022         if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
1023             prandom_u32() % 2 == 0) {
1024                 cfg->nosimd = true;
1025                 p += scnprintf(p, end - p, " nosimd");
1026         }
1027
1028         p += scnprintf(p, end - p, " src_divs=[");
1029         p = generate_random_sgl_divisions(cfg->src_divs,
1030                                           ARRAY_SIZE(cfg->src_divs), p, end,
1031                                           (cfg->finalization_type !=
1032                                            FINALIZATION_TYPE_DIGEST),
1033                                           cfg->req_flags);
1034         p += scnprintf(p, end - p, "]");
1035
1036         if (!cfg->inplace && prandom_u32() % 2 == 0) {
1037                 p += scnprintf(p, end - p, " dst_divs=[");
1038                 p = generate_random_sgl_divisions(cfg->dst_divs,
1039                                                   ARRAY_SIZE(cfg->dst_divs),
1040                                                   p, end, false,
1041                                                   cfg->req_flags);
1042                 p += scnprintf(p, end - p, "]");
1043         }
1044
1045         if (prandom_u32() % 2 == 0) {
1046                 cfg->iv_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
1047                 p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset);
1048         }
1049
1050         if (prandom_u32() % 2 == 0) {
1051                 cfg->key_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
1052                 p += scnprintf(p, end - p, " key_offset=%u", cfg->key_offset);
1053         }
1054
1055         WARN_ON_ONCE(!valid_testvec_config(cfg));
1056 }
1057
1058 static void crypto_disable_simd_for_test(void)
1059 {
1060         preempt_disable();
1061         __this_cpu_write(crypto_simd_disabled_for_test, true);
1062 }
1063
1064 static void crypto_reenable_simd_for_test(void)
1065 {
1066         __this_cpu_write(crypto_simd_disabled_for_test, false);
1067         preempt_enable();
1068 }
1069
1070 /*
1071  * Given an algorithm name, build the name of the generic implementation of that
1072  * algorithm, assuming the usual naming convention.  Specifically, this appends
1073  * "-generic" to every part of the name that is not a template name.  Examples:
1074  *
1075  *      aes => aes-generic
1076  *      cbc(aes) => cbc(aes-generic)
1077  *      cts(cbc(aes)) => cts(cbc(aes-generic))
1078  *      rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic)
1079  *
1080  * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long
1081  */
1082 static int build_generic_driver_name(const char *algname,
1083                                      char driver_name[CRYPTO_MAX_ALG_NAME])
1084 {
1085         const char *in = algname;
1086         char *out = driver_name;
1087         size_t len = strlen(algname);
1088
1089         if (len >= CRYPTO_MAX_ALG_NAME)
1090                 goto too_long;
1091         do {
1092                 const char *in_saved = in;
1093
1094                 while (*in && *in != '(' && *in != ')' && *in != ',')
1095                         *out++ = *in++;
1096                 if (*in != '(' && in > in_saved) {
1097                         len += 8;
1098                         if (len >= CRYPTO_MAX_ALG_NAME)
1099                                 goto too_long;
1100                         memcpy(out, "-generic", 8);
1101                         out += 8;
1102                 }
1103         } while ((*out++ = *in++) != '\0');
1104         return 0;
1105
1106 too_long:
1107         pr_err("alg: generic driver name for \"%s\" would be too long\n",
1108                algname);
1109         return -ENAMETOOLONG;
1110 }
1111 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1112 static void crypto_disable_simd_for_test(void)
1113 {
1114 }
1115
1116 static void crypto_reenable_simd_for_test(void)
1117 {
1118 }
1119 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1120
1121 static int build_hash_sglist(struct test_sglist *tsgl,
1122                              const struct hash_testvec *vec,
1123                              const struct testvec_config *cfg,
1124                              unsigned int alignmask,
1125                              const struct test_sg_division *divs[XBUFSIZE])
1126 {
1127         struct kvec kv;
1128         struct iov_iter input;
1129
1130         kv.iov_base = (void *)vec->plaintext;
1131         kv.iov_len = vec->psize;
1132         iov_iter_kvec(&input, WRITE, &kv, 1, vec->psize);
1133         return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
1134                                  &input, divs);
1135 }
1136
1137 static int check_hash_result(const char *type,
1138                              const u8 *result, unsigned int digestsize,
1139                              const struct hash_testvec *vec,
1140                              const char *vec_name,
1141                              const char *driver,
1142                              const struct testvec_config *cfg)
1143 {
1144         if (memcmp(result, vec->digest, digestsize) != 0) {
1145                 pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
1146                        type, driver, vec_name, cfg->name);
1147                 return -EINVAL;
1148         }
1149         if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
1150                 pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
1151                        type, driver, vec_name, cfg->name);
1152                 return -EOVERFLOW;
1153         }
1154         return 0;
1155 }
1156
1157 static inline int check_shash_op(const char *op, int err,
1158                                  const char *driver, const char *vec_name,
1159                                  const struct testvec_config *cfg)
1160 {
1161         if (err)
1162                 pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1163                        driver, op, err, vec_name, cfg->name);
1164         return err;
1165 }
1166
1167 static inline const void *sg_data(struct scatterlist *sg)
1168 {
1169         return page_address(sg_page(sg)) + sg->offset;
1170 }
1171
1172 /* Test one hash test vector in one configuration, using the shash API */
1173 static int test_shash_vec_cfg(const char *driver,
1174                               const struct hash_testvec *vec,
1175                               const char *vec_name,
1176                               const struct testvec_config *cfg,
1177                               struct shash_desc *desc,
1178                               struct test_sglist *tsgl,
1179                               u8 *hashstate)
1180 {
1181         struct crypto_shash *tfm = desc->tfm;
1182         const unsigned int alignmask = crypto_shash_alignmask(tfm);
1183         const unsigned int digestsize = crypto_shash_digestsize(tfm);
1184         const unsigned int statesize = crypto_shash_statesize(tfm);
1185         const struct test_sg_division *divs[XBUFSIZE];
1186         unsigned int i;
1187         u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1188         int err;
1189
1190         /* Set the key, if specified */
1191         if (vec->ksize) {
1192                 err = do_setkey(crypto_shash_setkey, tfm, vec->key, vec->ksize,
1193                                 cfg, alignmask);
1194                 if (err) {
1195                         if (err == vec->setkey_error)
1196                                 return 0;
1197                         pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1198                                driver, vec_name, vec->setkey_error, err,
1199                                crypto_shash_get_flags(tfm));
1200                         return err;
1201                 }
1202                 if (vec->setkey_error) {
1203                         pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1204                                driver, vec_name, vec->setkey_error);
1205                         return -EINVAL;
1206                 }
1207         }
1208
1209         /* Build the scatterlist for the source data */
1210         err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
1211         if (err) {
1212                 pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1213                        driver, vec_name, cfg->name);
1214                 return err;
1215         }
1216
1217         /* Do the actual hashing */
1218
1219         testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
1220         testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1221
1222         if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1223             vec->digest_error) {
1224                 /* Just using digest() */
1225                 if (tsgl->nents != 1)
1226                         return 0;
1227                 if (cfg->nosimd)
1228                         crypto_disable_simd_for_test();
1229                 err = crypto_shash_digest(desc, sg_data(&tsgl->sgl[0]),
1230                                           tsgl->sgl[0].length, result);
1231                 if (cfg->nosimd)
1232                         crypto_reenable_simd_for_test();
1233                 if (err) {
1234                         if (err == vec->digest_error)
1235                                 return 0;
1236                         pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1237                                driver, vec_name, vec->digest_error, err,
1238                                cfg->name);
1239                         return err;
1240                 }
1241                 if (vec->digest_error) {
1242                         pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1243                                driver, vec_name, vec->digest_error, cfg->name);
1244                         return -EINVAL;
1245                 }
1246                 goto result_ready;
1247         }
1248
1249         /* Using init(), zero or more update(), then final() or finup() */
1250
1251         if (cfg->nosimd)
1252                 crypto_disable_simd_for_test();
1253         err = crypto_shash_init(desc);
1254         if (cfg->nosimd)
1255                 crypto_reenable_simd_for_test();
1256         err = check_shash_op("init", err, driver, vec_name, cfg);
1257         if (err)
1258                 return err;
1259
1260         for (i = 0; i < tsgl->nents; i++) {
1261                 if (i + 1 == tsgl->nents &&
1262                     cfg->finalization_type == FINALIZATION_TYPE_FINUP) {
1263                         if (divs[i]->nosimd)
1264                                 crypto_disable_simd_for_test();
1265                         err = crypto_shash_finup(desc, sg_data(&tsgl->sgl[i]),
1266                                                  tsgl->sgl[i].length, result);
1267                         if (divs[i]->nosimd)
1268                                 crypto_reenable_simd_for_test();
1269                         err = check_shash_op("finup", err, driver, vec_name,
1270                                              cfg);
1271                         if (err)
1272                                 return err;
1273                         goto result_ready;
1274                 }
1275                 if (divs[i]->nosimd)
1276                         crypto_disable_simd_for_test();
1277                 err = crypto_shash_update(desc, sg_data(&tsgl->sgl[i]),
1278                                           tsgl->sgl[i].length);
1279                 if (divs[i]->nosimd)
1280                         crypto_reenable_simd_for_test();
1281                 err = check_shash_op("update", err, driver, vec_name, cfg);
1282                 if (err)
1283                         return err;
1284                 if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1285                         /* Test ->export() and ->import() */
1286                         testmgr_poison(hashstate + statesize,
1287                                        TESTMGR_POISON_LEN);
1288                         err = crypto_shash_export(desc, hashstate);
1289                         err = check_shash_op("export", err, driver, vec_name,
1290                                              cfg);
1291                         if (err)
1292                                 return err;
1293                         if (!testmgr_is_poison(hashstate + statesize,
1294                                                TESTMGR_POISON_LEN)) {
1295                                 pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1296                                        driver, vec_name, cfg->name);
1297                                 return -EOVERFLOW;
1298                         }
1299                         testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
1300                         err = crypto_shash_import(desc, hashstate);
1301                         err = check_shash_op("import", err, driver, vec_name,
1302                                              cfg);
1303                         if (err)
1304                                 return err;
1305                 }
1306         }
1307
1308         if (cfg->nosimd)
1309                 crypto_disable_simd_for_test();
1310         err = crypto_shash_final(desc, result);
1311         if (cfg->nosimd)
1312                 crypto_reenable_simd_for_test();
1313         err = check_shash_op("final", err, driver, vec_name, cfg);
1314         if (err)
1315                 return err;
1316 result_ready:
1317         return check_hash_result("shash", result, digestsize, vec, vec_name,
1318                                  driver, cfg);
1319 }
1320
1321 static int do_ahash_op(int (*op)(struct ahash_request *req),
1322                        struct ahash_request *req,
1323                        struct crypto_wait *wait, bool nosimd)
1324 {
1325         int err;
1326
1327         if (nosimd)
1328                 crypto_disable_simd_for_test();
1329
1330         err = op(req);
1331
1332         if (nosimd)
1333                 crypto_reenable_simd_for_test();
1334
1335         return crypto_wait_req(err, wait);
1336 }
1337
1338 static int check_nonfinal_ahash_op(const char *op, int err,
1339                                    u8 *result, unsigned int digestsize,
1340                                    const char *driver, const char *vec_name,
1341                                    const struct testvec_config *cfg)
1342 {
1343         if (err) {
1344                 pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1345                        driver, op, err, vec_name, cfg->name);
1346                 return err;
1347         }
1348         if (!testmgr_is_poison(result, digestsize)) {
1349                 pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
1350                        driver, op, vec_name, cfg->name);
1351                 return -EINVAL;
1352         }
1353         return 0;
1354 }
1355
1356 /* Test one hash test vector in one configuration, using the ahash API */
1357 static int test_ahash_vec_cfg(const char *driver,
1358                               const struct hash_testvec *vec,
1359                               const char *vec_name,
1360                               const struct testvec_config *cfg,
1361                               struct ahash_request *req,
1362                               struct test_sglist *tsgl,
1363                               u8 *hashstate)
1364 {
1365         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1366         const unsigned int alignmask = crypto_ahash_alignmask(tfm);
1367         const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1368         const unsigned int statesize = crypto_ahash_statesize(tfm);
1369         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1370         const struct test_sg_division *divs[XBUFSIZE];
1371         DECLARE_CRYPTO_WAIT(wait);
1372         unsigned int i;
1373         struct scatterlist *pending_sgl;
1374         unsigned int pending_len;
1375         u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1376         int err;
1377
1378         /* Set the key, if specified */
1379         if (vec->ksize) {
1380                 err = do_setkey(crypto_ahash_setkey, tfm, vec->key, vec->ksize,
1381                                 cfg, alignmask);
1382                 if (err) {
1383                         if (err == vec->setkey_error)
1384                                 return 0;
1385                         pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1386                                driver, vec_name, vec->setkey_error, err,
1387                                crypto_ahash_get_flags(tfm));
1388                         return err;
1389                 }
1390                 if (vec->setkey_error) {
1391                         pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1392                                driver, vec_name, vec->setkey_error);
1393                         return -EINVAL;
1394                 }
1395         }
1396
1397         /* Build the scatterlist for the source data */
1398         err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
1399         if (err) {
1400                 pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1401                        driver, vec_name, cfg->name);
1402                 return err;
1403         }
1404
1405         /* Do the actual hashing */
1406
1407         testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1408         testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1409
1410         if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1411             vec->digest_error) {
1412                 /* Just using digest() */
1413                 ahash_request_set_callback(req, req_flags, crypto_req_done,
1414                                            &wait);
1415                 ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize);
1416                 err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd);
1417                 if (err) {
1418                         if (err == vec->digest_error)
1419                                 return 0;
1420                         pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1421                                driver, vec_name, vec->digest_error, err,
1422                                cfg->name);
1423                         return err;
1424                 }
1425                 if (vec->digest_error) {
1426                         pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1427                                driver, vec_name, vec->digest_error, cfg->name);
1428                         return -EINVAL;
1429                 }
1430                 goto result_ready;
1431         }
1432
1433         /* Using init(), zero or more update(), then final() or finup() */
1434
1435         ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1436         ahash_request_set_crypt(req, NULL, result, 0);
1437         err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);
1438         err = check_nonfinal_ahash_op("init", err, result, digestsize,
1439                                       driver, vec_name, cfg);
1440         if (err)
1441                 return err;
1442
1443         pending_sgl = NULL;
1444         pending_len = 0;
1445         for (i = 0; i < tsgl->nents; i++) {
1446                 if (divs[i]->flush_type != FLUSH_TYPE_NONE &&
1447                     pending_sgl != NULL) {
1448                         /* update() with the pending data */
1449                         ahash_request_set_callback(req, req_flags,
1450                                                    crypto_req_done, &wait);
1451                         ahash_request_set_crypt(req, pending_sgl, result,
1452                                                 pending_len);
1453                         err = do_ahash_op(crypto_ahash_update, req, &wait,
1454                                           divs[i]->nosimd);
1455                         err = check_nonfinal_ahash_op("update", err,
1456                                                       result, digestsize,
1457                                                       driver, vec_name, cfg);
1458                         if (err)
1459                                 return err;
1460                         pending_sgl = NULL;
1461                         pending_len = 0;
1462                 }
1463                 if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1464                         /* Test ->export() and ->import() */
1465                         testmgr_poison(hashstate + statesize,
1466                                        TESTMGR_POISON_LEN);
1467                         err = crypto_ahash_export(req, hashstate);
1468                         err = check_nonfinal_ahash_op("export", err,
1469                                                       result, digestsize,
1470                                                       driver, vec_name, cfg);
1471                         if (err)
1472                                 return err;
1473                         if (!testmgr_is_poison(hashstate + statesize,
1474                                                TESTMGR_POISON_LEN)) {
1475                                 pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1476                                        driver, vec_name, cfg->name);
1477                                 return -EOVERFLOW;
1478                         }
1479
1480                         testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1481                         err = crypto_ahash_import(req, hashstate);
1482                         err = check_nonfinal_ahash_op("import", err,
1483                                                       result, digestsize,
1484                                                       driver, vec_name, cfg);
1485                         if (err)
1486                                 return err;
1487                 }
1488                 if (pending_sgl == NULL)
1489                         pending_sgl = &tsgl->sgl[i];
1490                 pending_len += tsgl->sgl[i].length;
1491         }
1492
1493         ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1494         ahash_request_set_crypt(req, pending_sgl, result, pending_len);
1495         if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
1496                 /* finish with update() and final() */
1497                 err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);
1498                 err = check_nonfinal_ahash_op("update", err, result, digestsize,
1499                                               driver, vec_name, cfg);
1500                 if (err)
1501                         return err;
1502                 err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);
1503                 if (err) {
1504                         pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
1505                                driver, err, vec_name, cfg->name);
1506                         return err;
1507                 }
1508         } else {
1509                 /* finish with finup() */
1510                 err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);
1511                 if (err) {
1512                         pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
1513                                driver, err, vec_name, cfg->name);
1514                         return err;
1515                 }
1516         }
1517
1518 result_ready:
1519         return check_hash_result("ahash", result, digestsize, vec, vec_name,
1520                                  driver, cfg);
1521 }
1522
1523 static int test_hash_vec_cfg(const char *driver,
1524                              const struct hash_testvec *vec,
1525                              const char *vec_name,
1526                              const struct testvec_config *cfg,
1527                              struct ahash_request *req,
1528                              struct shash_desc *desc,
1529                              struct test_sglist *tsgl,
1530                              u8 *hashstate)
1531 {
1532         int err;
1533
1534         /*
1535          * For algorithms implemented as "shash", most bugs will be detected by
1536          * both the shash and ahash tests.  Test the shash API first so that the
1537          * failures involve less indirection, so are easier to debug.
1538          */
1539
1540         if (desc) {
1541                 err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl,
1542                                          hashstate);
1543                 if (err)
1544                         return err;
1545         }
1546
1547         return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl,
1548                                   hashstate);
1549 }
1550
1551 static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
1552                          unsigned int vec_num, struct ahash_request *req,
1553                          struct shash_desc *desc, struct test_sglist *tsgl,
1554                          u8 *hashstate)
1555 {
1556         char vec_name[16];
1557         unsigned int i;
1558         int err;
1559
1560         sprintf(vec_name, "%u", vec_num);
1561
1562         for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
1563                 err = test_hash_vec_cfg(driver, vec, vec_name,
1564                                         &default_hash_testvec_configs[i],
1565                                         req, desc, tsgl, hashstate);
1566                 if (err)
1567                         return err;
1568         }
1569
1570 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1571         if (!noextratests) {
1572                 struct testvec_config cfg;
1573                 char cfgname[TESTVEC_CONFIG_NAMELEN];
1574
1575                 for (i = 0; i < fuzz_iterations; i++) {
1576                         generate_random_testvec_config(&cfg, cfgname,
1577                                                        sizeof(cfgname));
1578                         err = test_hash_vec_cfg(driver, vec, vec_name, &cfg,
1579                                                 req, desc, tsgl, hashstate);
1580                         if (err)
1581                                 return err;
1582                         cond_resched();
1583                 }
1584         }
1585 #endif
1586         return 0;
1587 }
1588
1589 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1590 /*
1591  * Generate a hash test vector from the given implementation.
1592  * Assumes the buffers in 'vec' were already allocated.
1593  */
1594 static void generate_random_hash_testvec(struct shash_desc *desc,
1595                                          struct hash_testvec *vec,
1596                                          unsigned int maxkeysize,
1597                                          unsigned int maxdatasize,
1598                                          char *name, size_t max_namelen)
1599 {
1600         /* Data */
1601         vec->psize = generate_random_length(maxdatasize);
1602         generate_random_bytes((u8 *)vec->plaintext, vec->psize);
1603
1604         /*
1605          * Key: length in range [1, maxkeysize], but usually choose maxkeysize.
1606          * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0.
1607          */
1608         vec->setkey_error = 0;
1609         vec->ksize = 0;
1610         if (maxkeysize) {
1611                 vec->ksize = maxkeysize;
1612                 if (prandom_u32() % 4 == 0)
1613                         vec->ksize = 1 + (prandom_u32() % maxkeysize);
1614                 generate_random_bytes((u8 *)vec->key, vec->ksize);
1615
1616                 vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key,
1617                                                         vec->ksize);
1618                 /* If the key couldn't be set, no need to continue to digest. */
1619                 if (vec->setkey_error)
1620                         goto done;
1621         }
1622
1623         /* Digest */
1624         vec->digest_error = crypto_shash_digest(desc, vec->plaintext,
1625                                                 vec->psize, (u8 *)vec->digest);
1626 done:
1627         snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"",
1628                  vec->psize, vec->ksize);
1629 }
1630
1631 /*
1632  * Test the hash algorithm represented by @req against the corresponding generic
1633  * implementation, if one is available.
1634  */
1635 static int test_hash_vs_generic_impl(const char *driver,
1636                                      const char *generic_driver,
1637                                      unsigned int maxkeysize,
1638                                      struct ahash_request *req,
1639                                      struct shash_desc *desc,
1640                                      struct test_sglist *tsgl,
1641                                      u8 *hashstate)
1642 {
1643         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1644         const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1645         const unsigned int blocksize = crypto_ahash_blocksize(tfm);
1646         const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
1647         const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
1648         char _generic_driver[CRYPTO_MAX_ALG_NAME];
1649         struct crypto_shash *generic_tfm = NULL;
1650         struct shash_desc *generic_desc = NULL;
1651         unsigned int i;
1652         struct hash_testvec vec = { 0 };
1653         char vec_name[64];
1654         struct testvec_config *cfg;
1655         char cfgname[TESTVEC_CONFIG_NAMELEN];
1656         int err;
1657
1658         if (noextratests)
1659                 return 0;
1660
1661         if (!generic_driver) { /* Use default naming convention? */
1662                 err = build_generic_driver_name(algname, _generic_driver);
1663                 if (err)
1664                         return err;
1665                 generic_driver = _generic_driver;
1666         }
1667
1668         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
1669                 return 0;
1670
1671         generic_tfm = crypto_alloc_shash(generic_driver, 0, 0);
1672         if (IS_ERR(generic_tfm)) {
1673                 err = PTR_ERR(generic_tfm);
1674                 if (err == -ENOENT) {
1675                         pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n",
1676                                 driver, generic_driver);
1677                         return 0;
1678                 }
1679                 pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n",
1680                        generic_driver, algname, err);
1681                 return err;
1682         }
1683
1684         cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1685         if (!cfg) {
1686                 err = -ENOMEM;
1687                 goto out;
1688         }
1689
1690         generic_desc = kzalloc(sizeof(*desc) +
1691                                crypto_shash_descsize(generic_tfm), GFP_KERNEL);
1692         if (!generic_desc) {
1693                 err = -ENOMEM;
1694                 goto out;
1695         }
1696         generic_desc->tfm = generic_tfm;
1697
1698         /* Check the algorithm properties for consistency. */
1699
1700         if (digestsize != crypto_shash_digestsize(generic_tfm)) {
1701                 pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n",
1702                        driver, digestsize,
1703                        crypto_shash_digestsize(generic_tfm));
1704                 err = -EINVAL;
1705                 goto out;
1706         }
1707
1708         if (blocksize != crypto_shash_blocksize(generic_tfm)) {
1709                 pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n",
1710                        driver, blocksize, crypto_shash_blocksize(generic_tfm));
1711                 err = -EINVAL;
1712                 goto out;
1713         }
1714
1715         /*
1716          * Now generate test vectors using the generic implementation, and test
1717          * the other implementation against them.
1718          */
1719
1720         vec.key = kmalloc(maxkeysize, GFP_KERNEL);
1721         vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL);
1722         vec.digest = kmalloc(digestsize, GFP_KERNEL);
1723         if (!vec.key || !vec.plaintext || !vec.digest) {
1724                 err = -ENOMEM;
1725                 goto out;
1726         }
1727
1728         for (i = 0; i < fuzz_iterations * 8; i++) {
1729                 generate_random_hash_testvec(generic_desc, &vec,
1730                                              maxkeysize, maxdatasize,
1731                                              vec_name, sizeof(vec_name));
1732                 generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
1733
1734                 err = test_hash_vec_cfg(driver, &vec, vec_name, cfg,
1735                                         req, desc, tsgl, hashstate);
1736                 if (err)
1737                         goto out;
1738                 cond_resched();
1739         }
1740         err = 0;
1741 out:
1742         kfree(cfg);
1743         kfree(vec.key);
1744         kfree(vec.plaintext);
1745         kfree(vec.digest);
1746         crypto_free_shash(generic_tfm);
1747         kzfree(generic_desc);
1748         return err;
1749 }
1750 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1751 static int test_hash_vs_generic_impl(const char *driver,
1752                                      const char *generic_driver,
1753                                      unsigned int maxkeysize,
1754                                      struct ahash_request *req,
1755                                      struct shash_desc *desc,
1756                                      struct test_sglist *tsgl,
1757                                      u8 *hashstate)
1758 {
1759         return 0;
1760 }
1761 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1762
1763 static int alloc_shash(const char *driver, u32 type, u32 mask,
1764                        struct crypto_shash **tfm_ret,
1765                        struct shash_desc **desc_ret)
1766 {
1767         struct crypto_shash *tfm;
1768         struct shash_desc *desc;
1769
1770         tfm = crypto_alloc_shash(driver, type, mask);
1771         if (IS_ERR(tfm)) {
1772                 if (PTR_ERR(tfm) == -ENOENT) {
1773                         /*
1774                          * This algorithm is only available through the ahash
1775                          * API, not the shash API, so skip the shash tests.
1776                          */
1777                         return 0;
1778                 }
1779                 pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",
1780                        driver, PTR_ERR(tfm));
1781                 return PTR_ERR(tfm);
1782         }
1783
1784         desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);
1785         if (!desc) {
1786                 crypto_free_shash(tfm);
1787                 return -ENOMEM;
1788         }
1789         desc->tfm = tfm;
1790
1791         *tfm_ret = tfm;
1792         *desc_ret = desc;
1793         return 0;
1794 }
1795
1796 static int __alg_test_hash(const struct hash_testvec *vecs,
1797                            unsigned int num_vecs, const char *driver,
1798                            u32 type, u32 mask,
1799                            const char *generic_driver, unsigned int maxkeysize)
1800 {
1801         struct crypto_ahash *atfm = NULL;
1802         struct ahash_request *req = NULL;
1803         struct crypto_shash *stfm = NULL;
1804         struct shash_desc *desc = NULL;
1805         struct test_sglist *tsgl = NULL;
1806         u8 *hashstate = NULL;
1807         unsigned int statesize;
1808         unsigned int i;
1809         int err;
1810
1811         /*
1812          * Always test the ahash API.  This works regardless of whether the
1813          * algorithm is implemented as ahash or shash.
1814          */
1815
1816         atfm = crypto_alloc_ahash(driver, type, mask);
1817         if (IS_ERR(atfm)) {
1818                 pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
1819                        driver, PTR_ERR(atfm));
1820                 return PTR_ERR(atfm);
1821         }
1822
1823         req = ahash_request_alloc(atfm, GFP_KERNEL);
1824         if (!req) {
1825                 pr_err("alg: hash: failed to allocate request for %s\n",
1826                        driver);
1827                 err = -ENOMEM;
1828                 goto out;
1829         }
1830
1831         /*
1832          * If available also test the shash API, to cover corner cases that may
1833          * be missed by testing the ahash API only.
1834          */
1835         err = alloc_shash(driver, type, mask, &stfm, &desc);
1836         if (err)
1837                 goto out;
1838
1839         tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
1840         if (!tsgl || init_test_sglist(tsgl) != 0) {
1841                 pr_err("alg: hash: failed to allocate test buffers for %s\n",
1842                        driver);
1843                 kfree(tsgl);
1844                 tsgl = NULL;
1845                 err = -ENOMEM;
1846                 goto out;
1847         }
1848
1849         statesize = crypto_ahash_statesize(atfm);
1850         if (stfm)
1851                 statesize = max(statesize, crypto_shash_statesize(stfm));
1852         hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);
1853         if (!hashstate) {
1854                 pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
1855                        driver);
1856                 err = -ENOMEM;
1857                 goto out;
1858         }
1859
1860         for (i = 0; i < num_vecs; i++) {
1861                 err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl,
1862                                     hashstate);
1863                 if (err)
1864                         goto out;
1865                 cond_resched();
1866         }
1867         err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req,
1868                                         desc, tsgl, hashstate);
1869 out:
1870         kfree(hashstate);
1871         if (tsgl) {
1872                 destroy_test_sglist(tsgl);
1873                 kfree(tsgl);
1874         }
1875         kfree(desc);
1876         crypto_free_shash(stfm);
1877         ahash_request_free(req);
1878         crypto_free_ahash(atfm);
1879         return err;
1880 }
1881
1882 static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
1883                          u32 type, u32 mask)
1884 {
1885         const struct hash_testvec *template = desc->suite.hash.vecs;
1886         unsigned int tcount = desc->suite.hash.count;
1887         unsigned int nr_unkeyed, nr_keyed;
1888         unsigned int maxkeysize = 0;
1889         int err;
1890
1891         /*
1892          * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
1893          * first, before setting a key on the tfm.  To make this easier, we
1894          * require that the unkeyed test vectors (if any) are listed first.
1895          */
1896
1897         for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
1898                 if (template[nr_unkeyed].ksize)
1899                         break;
1900         }
1901         for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
1902                 if (!template[nr_unkeyed + nr_keyed].ksize) {
1903                         pr_err("alg: hash: test vectors for %s out of order, "
1904                                "unkeyed ones must come first\n", desc->alg);
1905                         return -EINVAL;
1906                 }
1907                 maxkeysize = max_t(unsigned int, maxkeysize,
1908                                    template[nr_unkeyed + nr_keyed].ksize);
1909         }
1910
1911         err = 0;
1912         if (nr_unkeyed) {
1913                 err = __alg_test_hash(template, nr_unkeyed, driver, type, mask,
1914                                       desc->generic_driver, maxkeysize);
1915                 template += nr_unkeyed;
1916         }
1917
1918         if (!err && nr_keyed)
1919                 err = __alg_test_hash(template, nr_keyed, driver, type, mask,
1920                                       desc->generic_driver, maxkeysize);
1921
1922         return err;
1923 }
1924
1925 static int test_aead_vec_cfg(const char *driver, int enc,
1926                              const struct aead_testvec *vec,
1927                              const char *vec_name,
1928                              const struct testvec_config *cfg,
1929                              struct aead_request *req,
1930                              struct cipher_test_sglists *tsgls)
1931 {
1932         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1933         const unsigned int alignmask = crypto_aead_alignmask(tfm);
1934         const unsigned int ivsize = crypto_aead_ivsize(tfm);
1935         const unsigned int authsize = vec->clen - vec->plen;
1936         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1937         const char *op = enc ? "encryption" : "decryption";
1938         DECLARE_CRYPTO_WAIT(wait);
1939         u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
1940         u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
1941                  cfg->iv_offset +
1942                  (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
1943         struct kvec input[2];
1944         int err;
1945
1946         /* Set the key */
1947         if (vec->wk)
1948                 crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
1949         else
1950                 crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
1951
1952         err = do_setkey(crypto_aead_setkey, tfm, vec->key, vec->klen,
1953                         cfg, alignmask);
1954         if (err && err != vec->setkey_error) {
1955                 pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1956                        driver, vec_name, vec->setkey_error, err,
1957                        crypto_aead_get_flags(tfm));
1958                 return err;
1959         }
1960         if (!err && vec->setkey_error) {
1961                 pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1962                        driver, vec_name, vec->setkey_error);
1963                 return -EINVAL;
1964         }
1965
1966         /* Set the authentication tag size */
1967         err = crypto_aead_setauthsize(tfm, authsize);
1968         if (err && err != vec->setauthsize_error) {
1969                 pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n",
1970                        driver, vec_name, vec->setauthsize_error, err);
1971                 return err;
1972         }
1973         if (!err && vec->setauthsize_error) {
1974                 pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n",
1975                        driver, vec_name, vec->setauthsize_error);
1976                 return -EINVAL;
1977         }
1978
1979         if (vec->setkey_error || vec->setauthsize_error)
1980                 return 0;
1981
1982         /* The IV must be copied to a buffer, as the algorithm may modify it */
1983         if (WARN_ON(ivsize > MAX_IVLEN))
1984                 return -EINVAL;
1985         if (vec->iv)
1986                 memcpy(iv, vec->iv, ivsize);
1987         else
1988                 memset(iv, 0, ivsize);
1989
1990         /* Build the src/dst scatterlists */
1991         input[0].iov_base = (void *)vec->assoc;
1992         input[0].iov_len = vec->alen;
1993         input[1].iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
1994         input[1].iov_len = enc ? vec->plen : vec->clen;
1995         err = build_cipher_test_sglists(tsgls, cfg, alignmask,
1996                                         vec->alen + (enc ? vec->plen :
1997                                                      vec->clen),
1998                                         vec->alen + (enc ? vec->clen :
1999                                                      vec->plen),
2000                                         input, 2);
2001         if (err) {
2002                 pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2003                        driver, op, vec_name, cfg->name);
2004                 return err;
2005         }
2006
2007         /* Do the actual encryption or decryption */
2008         testmgr_poison(req->__ctx, crypto_aead_reqsize(tfm));
2009         aead_request_set_callback(req, req_flags, crypto_req_done, &wait);
2010         aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2011                                enc ? vec->plen : vec->clen, iv);
2012         aead_request_set_ad(req, vec->alen);
2013         if (cfg->nosimd)
2014                 crypto_disable_simd_for_test();
2015         err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
2016         if (cfg->nosimd)
2017                 crypto_reenable_simd_for_test();
2018         err = crypto_wait_req(err, &wait);
2019
2020         /* Check that the algorithm didn't overwrite things it shouldn't have */
2021         if (req->cryptlen != (enc ? vec->plen : vec->clen) ||
2022             req->assoclen != vec->alen ||
2023             req->iv != iv ||
2024             req->src != tsgls->src.sgl_ptr ||
2025             req->dst != tsgls->dst.sgl_ptr ||
2026             crypto_aead_reqtfm(req) != tfm ||
2027             req->base.complete != crypto_req_done ||
2028             req->base.flags != req_flags ||
2029             req->base.data != &wait) {
2030                 pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2031                        driver, op, vec_name, cfg->name);
2032                 if (req->cryptlen != (enc ? vec->plen : vec->clen))
2033                         pr_err("alg: aead: changed 'req->cryptlen'\n");
2034                 if (req->assoclen != vec->alen)
2035                         pr_err("alg: aead: changed 'req->assoclen'\n");
2036                 if (req->iv != iv)
2037                         pr_err("alg: aead: changed 'req->iv'\n");
2038                 if (req->src != tsgls->src.sgl_ptr)
2039                         pr_err("alg: aead: changed 'req->src'\n");
2040                 if (req->dst != tsgls->dst.sgl_ptr)
2041                         pr_err("alg: aead: changed 'req->dst'\n");
2042                 if (crypto_aead_reqtfm(req) != tfm)
2043                         pr_err("alg: aead: changed 'req->base.tfm'\n");
2044                 if (req->base.complete != crypto_req_done)
2045                         pr_err("alg: aead: changed 'req->base.complete'\n");
2046                 if (req->base.flags != req_flags)
2047                         pr_err("alg: aead: changed 'req->base.flags'\n");
2048                 if (req->base.data != &wait)
2049                         pr_err("alg: aead: changed 'req->base.data'\n");
2050                 return -EINVAL;
2051         }
2052         if (is_test_sglist_corrupted(&tsgls->src)) {
2053                 pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2054                        driver, op, vec_name, cfg->name);
2055                 return -EINVAL;
2056         }
2057         if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2058             is_test_sglist_corrupted(&tsgls->dst)) {
2059                 pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2060                        driver, op, vec_name, cfg->name);
2061                 return -EINVAL;
2062         }
2063
2064         /* Check for unexpected success or failure, or wrong error code */
2065         if ((err == 0 && vec->novrfy) ||
2066             (err != vec->crypt_error && !(err == -EBADMSG && vec->novrfy))) {
2067                 char expected_error[32];
2068
2069                 if (vec->novrfy &&
2070                     vec->crypt_error != 0 && vec->crypt_error != -EBADMSG)
2071                         sprintf(expected_error, "-EBADMSG or %d",
2072                                 vec->crypt_error);
2073                 else if (vec->novrfy)
2074                         sprintf(expected_error, "-EBADMSG");
2075                 else
2076                         sprintf(expected_error, "%d", vec->crypt_error);
2077                 if (err) {
2078                         pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%s, actual_error=%d, cfg=\"%s\"\n",
2079                                driver, op, vec_name, expected_error, err,
2080                                cfg->name);
2081                         return err;
2082                 }
2083                 pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%s, cfg=\"%s\"\n",
2084                        driver, op, vec_name, expected_error, cfg->name);
2085                 return -EINVAL;
2086         }
2087         if (err) /* Expectedly failed. */
2088                 return 0;
2089
2090         /* Check for the correct output (ciphertext or plaintext) */
2091         err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2092                                     enc ? vec->clen : vec->plen,
2093                                     vec->alen, enc || !cfg->inplace);
2094         if (err == -EOVERFLOW) {
2095                 pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2096                        driver, op, vec_name, cfg->name);
2097                 return err;
2098         }
2099         if (err) {
2100                 pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2101                        driver, op, vec_name, cfg->name);
2102                 return err;
2103         }
2104
2105         return 0;
2106 }
2107
2108 static int test_aead_vec(const char *driver, int enc,
2109                          const struct aead_testvec *vec, unsigned int vec_num,
2110                          struct aead_request *req,
2111                          struct cipher_test_sglists *tsgls)
2112 {
2113         char vec_name[16];
2114         unsigned int i;
2115         int err;
2116
2117         if (enc && vec->novrfy)
2118                 return 0;
2119
2120         sprintf(vec_name, "%u", vec_num);
2121
2122         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2123                 err = test_aead_vec_cfg(driver, enc, vec, vec_name,
2124                                         &default_cipher_testvec_configs[i],
2125                                         req, tsgls);
2126                 if (err)
2127                         return err;
2128         }
2129
2130 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2131         if (!noextratests) {
2132                 struct testvec_config cfg;
2133                 char cfgname[TESTVEC_CONFIG_NAMELEN];
2134
2135                 for (i = 0; i < fuzz_iterations; i++) {
2136                         generate_random_testvec_config(&cfg, cfgname,
2137                                                        sizeof(cfgname));
2138                         err = test_aead_vec_cfg(driver, enc, vec, vec_name,
2139                                                 &cfg, req, tsgls);
2140                         if (err)
2141                                 return err;
2142                         cond_resched();
2143                 }
2144         }
2145 #endif
2146         return 0;
2147 }
2148
2149 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2150
2151 struct aead_extra_tests_ctx {
2152         struct aead_request *req;
2153         struct crypto_aead *tfm;
2154         const char *driver;
2155         const struct alg_test_desc *test_desc;
2156         struct cipher_test_sglists *tsgls;
2157         unsigned int maxdatasize;
2158         unsigned int maxkeysize;
2159
2160         struct aead_testvec vec;
2161         char vec_name[64];
2162         char cfgname[TESTVEC_CONFIG_NAMELEN];
2163         struct testvec_config cfg;
2164 };
2165
2166 /*
2167  * Make at least one random change to a (ciphertext, AAD) pair.  "Ciphertext"
2168  * here means the full ciphertext including the authentication tag.  The
2169  * authentication tag (and hence also the ciphertext) is assumed to be nonempty.
2170  */
2171 static void mutate_aead_message(struct aead_testvec *vec, bool aad_iv,
2172                                 unsigned int ivsize)
2173 {
2174         const unsigned int aad_tail_size = aad_iv ? ivsize : 0;
2175         const unsigned int authsize = vec->clen - vec->plen;
2176
2177         if (prandom_u32() % 2 == 0 && vec->alen > aad_tail_size) {
2178                  /* Mutate the AAD */
2179                 flip_random_bit((u8 *)vec->assoc, vec->alen - aad_tail_size);
2180                 if (prandom_u32() % 2 == 0)
2181                         return;
2182         }
2183         if (prandom_u32() % 2 == 0) {
2184                 /* Mutate auth tag (assuming it's at the end of ciphertext) */
2185                 flip_random_bit((u8 *)vec->ctext + vec->plen, authsize);
2186         } else {
2187                 /* Mutate any part of the ciphertext */
2188                 flip_random_bit((u8 *)vec->ctext, vec->clen);
2189         }
2190 }
2191
2192 /*
2193  * Minimum authentication tag size in bytes at which we assume that we can
2194  * reliably generate inauthentic messages, i.e. not generate an authentic
2195  * message by chance.
2196  */
2197 #define MIN_COLLISION_FREE_AUTHSIZE 8
2198
2199 static void generate_aead_message(struct aead_request *req,
2200                                   const struct aead_test_suite *suite,
2201                                   struct aead_testvec *vec,
2202                                   bool prefer_inauthentic)
2203 {
2204         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2205         const unsigned int ivsize = crypto_aead_ivsize(tfm);
2206         const unsigned int authsize = vec->clen - vec->plen;
2207         const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) &&
2208                                  (prefer_inauthentic || prandom_u32() % 4 == 0);
2209
2210         /* Generate the AAD. */
2211         generate_random_bytes((u8 *)vec->assoc, vec->alen);
2212         if (suite->aad_iv && vec->alen >= ivsize)
2213                 /* Avoid implementation-defined behavior. */
2214                 memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize);
2215
2216         if (inauthentic && prandom_u32() % 2 == 0) {
2217                 /* Generate a random ciphertext. */
2218                 generate_random_bytes((u8 *)vec->ctext, vec->clen);
2219         } else {
2220                 int i = 0;
2221                 struct scatterlist src[2], dst;
2222                 u8 iv[MAX_IVLEN];
2223                 DECLARE_CRYPTO_WAIT(wait);
2224
2225                 /* Generate a random plaintext and encrypt it. */
2226                 sg_init_table(src, 2);
2227                 if (vec->alen)
2228                         sg_set_buf(&src[i++], vec->assoc, vec->alen);
2229                 if (vec->plen) {
2230                         generate_random_bytes((u8 *)vec->ptext, vec->plen);
2231                         sg_set_buf(&src[i++], vec->ptext, vec->plen);
2232                 }
2233                 sg_init_one(&dst, vec->ctext, vec->alen + vec->clen);
2234                 memcpy(iv, vec->iv, ivsize);
2235                 aead_request_set_callback(req, 0, crypto_req_done, &wait);
2236                 aead_request_set_crypt(req, src, &dst, vec->plen, iv);
2237                 aead_request_set_ad(req, vec->alen);
2238                 vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req),
2239                                                    &wait);
2240                 /* If encryption failed, we're done. */
2241                 if (vec->crypt_error != 0)
2242                         return;
2243                 memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen);
2244                 if (!inauthentic)
2245                         return;
2246                 /*
2247                  * Mutate the authentic (ciphertext, AAD) pair to get an
2248                  * inauthentic one.
2249                  */
2250                 mutate_aead_message(vec, suite->aad_iv, ivsize);
2251         }
2252         vec->novrfy = 1;
2253         if (suite->einval_allowed)
2254                 vec->crypt_error = -EINVAL;
2255 }
2256
2257 /*
2258  * Generate an AEAD test vector 'vec' using the implementation specified by
2259  * 'req'.  The buffers in 'vec' must already be allocated.
2260  *
2261  * If 'prefer_inauthentic' is true, then this function will generate inauthentic
2262  * test vectors (i.e. vectors with 'vec->novrfy=1') more often.
2263  */
2264 static void generate_random_aead_testvec(struct aead_request *req,
2265                                          struct aead_testvec *vec,
2266                                          const struct aead_test_suite *suite,
2267                                          unsigned int maxkeysize,
2268                                          unsigned int maxdatasize,
2269                                          char *name, size_t max_namelen,
2270                                          bool prefer_inauthentic)
2271 {
2272         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2273         const unsigned int ivsize = crypto_aead_ivsize(tfm);
2274         const unsigned int maxauthsize = crypto_aead_maxauthsize(tfm);
2275         unsigned int authsize;
2276         unsigned int total_len;
2277
2278         /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
2279         vec->klen = maxkeysize;
2280         if (prandom_u32() % 4 == 0)
2281                 vec->klen = prandom_u32() % (maxkeysize + 1);
2282         generate_random_bytes((u8 *)vec->key, vec->klen);
2283         vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen);
2284
2285         /* IV */
2286         generate_random_bytes((u8 *)vec->iv, ivsize);
2287
2288         /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */
2289         authsize = maxauthsize;
2290         if (prandom_u32() % 4 == 0)
2291                 authsize = prandom_u32() % (maxauthsize + 1);
2292         if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE)
2293                 authsize = MIN_COLLISION_FREE_AUTHSIZE;
2294         if (WARN_ON(authsize > maxdatasize))
2295                 authsize = maxdatasize;
2296         maxdatasize -= authsize;
2297         vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize);
2298
2299         /* AAD, plaintext, and ciphertext lengths */
2300         total_len = generate_random_length(maxdatasize);
2301         if (prandom_u32() % 4 == 0)
2302                 vec->alen = 0;
2303         else
2304                 vec->alen = generate_random_length(total_len);
2305         vec->plen = total_len - vec->alen;
2306         vec->clen = vec->plen + authsize;
2307
2308         /*
2309          * Generate the AAD, plaintext, and ciphertext.  Not applicable if the
2310          * key or the authentication tag size couldn't be set.
2311          */
2312         vec->novrfy = 0;
2313         vec->crypt_error = 0;
2314         if (vec->setkey_error == 0 && vec->setauthsize_error == 0)
2315                 generate_aead_message(req, suite, vec, prefer_inauthentic);
2316         snprintf(name, max_namelen,
2317                  "\"random: alen=%u plen=%u authsize=%u klen=%u novrfy=%d\"",
2318                  vec->alen, vec->plen, authsize, vec->klen, vec->novrfy);
2319 }
2320
2321 static void try_to_generate_inauthentic_testvec(
2322                                         struct aead_extra_tests_ctx *ctx)
2323 {
2324         int i;
2325
2326         for (i = 0; i < 10; i++) {
2327                 generate_random_aead_testvec(ctx->req, &ctx->vec,
2328                                              &ctx->test_desc->suite.aead,
2329                                              ctx->maxkeysize, ctx->maxdatasize,
2330                                              ctx->vec_name,
2331                                              sizeof(ctx->vec_name), true);
2332                 if (ctx->vec.novrfy)
2333                         return;
2334         }
2335 }
2336
2337 /*
2338  * Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the
2339  * result of an encryption with the key) and verify that decryption fails.
2340  */
2341 static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx)
2342 {
2343         unsigned int i;
2344         int err;
2345
2346         for (i = 0; i < fuzz_iterations * 8; i++) {
2347                 /*
2348                  * Since this part of the tests isn't comparing the
2349                  * implementation to another, there's no point in testing any
2350                  * test vectors other than inauthentic ones (vec.novrfy=1) here.
2351                  *
2352                  * If we're having trouble generating such a test vector, e.g.
2353                  * if the algorithm keeps rejecting the generated keys, don't
2354                  * retry forever; just continue on.
2355                  */
2356                 try_to_generate_inauthentic_testvec(ctx);
2357                 if (ctx->vec.novrfy) {
2358                         generate_random_testvec_config(&ctx->cfg, ctx->cfgname,
2359                                                        sizeof(ctx->cfgname));
2360                         err = test_aead_vec_cfg(ctx->driver, DECRYPT, &ctx->vec,
2361                                                 ctx->vec_name, &ctx->cfg,
2362                                                 ctx->req, ctx->tsgls);
2363                         if (err)
2364                                 return err;
2365                 }
2366                 cond_resched();
2367         }
2368         return 0;
2369 }
2370
2371 /*
2372  * Test the AEAD algorithm against the corresponding generic implementation, if
2373  * one is available.
2374  */
2375 static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx)
2376 {
2377         struct crypto_aead *tfm = ctx->tfm;
2378         const char *algname = crypto_aead_alg(tfm)->base.cra_name;
2379         const char *driver = ctx->driver;
2380         const char *generic_driver = ctx->test_desc->generic_driver;
2381         char _generic_driver[CRYPTO_MAX_ALG_NAME];
2382         struct crypto_aead *generic_tfm = NULL;
2383         struct aead_request *generic_req = NULL;
2384         unsigned int i;
2385         int err;
2386
2387         if (!generic_driver) { /* Use default naming convention? */
2388                 err = build_generic_driver_name(algname, _generic_driver);
2389                 if (err)
2390                         return err;
2391                 generic_driver = _generic_driver;
2392         }
2393
2394         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
2395                 return 0;
2396
2397         generic_tfm = crypto_alloc_aead(generic_driver, 0, 0);
2398         if (IS_ERR(generic_tfm)) {
2399                 err = PTR_ERR(generic_tfm);
2400                 if (err == -ENOENT) {
2401                         pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n",
2402                                 driver, generic_driver);
2403                         return 0;
2404                 }
2405                 pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n",
2406                        generic_driver, algname, err);
2407                 return err;
2408         }
2409
2410         generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);
2411         if (!generic_req) {
2412                 err = -ENOMEM;
2413                 goto out;
2414         }
2415
2416         /* Check the algorithm properties for consistency. */
2417
2418         if (crypto_aead_maxauthsize(tfm) !=
2419             crypto_aead_maxauthsize(generic_tfm)) {
2420                 pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n",
2421                        driver, crypto_aead_maxauthsize(tfm),
2422                        crypto_aead_maxauthsize(generic_tfm));
2423                 err = -EINVAL;
2424                 goto out;
2425         }
2426
2427         if (crypto_aead_ivsize(tfm) != crypto_aead_ivsize(generic_tfm)) {
2428                 pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n",
2429                        driver, crypto_aead_ivsize(tfm),
2430                        crypto_aead_ivsize(generic_tfm));
2431                 err = -EINVAL;
2432                 goto out;
2433         }
2434
2435         if (crypto_aead_blocksize(tfm) != crypto_aead_blocksize(generic_tfm)) {
2436                 pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n",
2437                        driver, crypto_aead_blocksize(tfm),
2438                        crypto_aead_blocksize(generic_tfm));
2439                 err = -EINVAL;
2440                 goto out;
2441         }
2442
2443         /*
2444          * Now generate test vectors using the generic implementation, and test
2445          * the other implementation against them.
2446          */
2447         for (i = 0; i < fuzz_iterations * 8; i++) {
2448                 generate_random_aead_testvec(generic_req, &ctx->vec,
2449                                              &ctx->test_desc->suite.aead,
2450                                              ctx->maxkeysize, ctx->maxdatasize,
2451                                              ctx->vec_name,
2452                                              sizeof(ctx->vec_name), false);
2453                 generate_random_testvec_config(&ctx->cfg, ctx->cfgname,
2454                                                sizeof(ctx->cfgname));
2455                 if (!ctx->vec.novrfy) {
2456                         err = test_aead_vec_cfg(driver, ENCRYPT, &ctx->vec,
2457                                                 ctx->vec_name, &ctx->cfg,
2458                                                 ctx->req, ctx->tsgls);
2459                         if (err)
2460                                 goto out;
2461                 }
2462                 if (ctx->vec.crypt_error == 0 || ctx->vec.novrfy) {
2463                         err = test_aead_vec_cfg(driver, DECRYPT, &ctx->vec,
2464                                                 ctx->vec_name, &ctx->cfg,
2465                                                 ctx->req, ctx->tsgls);
2466                         if (err)
2467                                 goto out;
2468                 }
2469                 cond_resched();
2470         }
2471         err = 0;
2472 out:
2473         crypto_free_aead(generic_tfm);
2474         aead_request_free(generic_req);
2475         return err;
2476 }
2477
2478 static int test_aead_extra(const char *driver,
2479                            const struct alg_test_desc *test_desc,
2480                            struct aead_request *req,
2481                            struct cipher_test_sglists *tsgls)
2482 {
2483         struct aead_extra_tests_ctx *ctx;
2484         unsigned int i;
2485         int err;
2486
2487         if (noextratests)
2488                 return 0;
2489
2490         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2491         if (!ctx)
2492                 return -ENOMEM;
2493         ctx->req = req;
2494         ctx->tfm = crypto_aead_reqtfm(req);
2495         ctx->driver = driver;
2496         ctx->test_desc = test_desc;
2497         ctx->tsgls = tsgls;
2498         ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
2499         ctx->maxkeysize = 0;
2500         for (i = 0; i < test_desc->suite.aead.count; i++)
2501                 ctx->maxkeysize = max_t(unsigned int, ctx->maxkeysize,
2502                                         test_desc->suite.aead.vecs[i].klen);
2503
2504         ctx->vec.key = kmalloc(ctx->maxkeysize, GFP_KERNEL);
2505         ctx->vec.iv = kmalloc(crypto_aead_ivsize(ctx->tfm), GFP_KERNEL);
2506         ctx->vec.assoc = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2507         ctx->vec.ptext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2508         ctx->vec.ctext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2509         if (!ctx->vec.key || !ctx->vec.iv || !ctx->vec.assoc ||
2510             !ctx->vec.ptext || !ctx->vec.ctext) {
2511                 err = -ENOMEM;
2512                 goto out;
2513         }
2514
2515         err = test_aead_vs_generic_impl(ctx);
2516         if (err)
2517                 goto out;
2518
2519         err = test_aead_inauthentic_inputs(ctx);
2520 out:
2521         kfree(ctx->vec.key);
2522         kfree(ctx->vec.iv);
2523         kfree(ctx->vec.assoc);
2524         kfree(ctx->vec.ptext);
2525         kfree(ctx->vec.ctext);
2526         kfree(ctx);
2527         return err;
2528 }
2529 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
2530 static int test_aead_extra(const char *driver,
2531                            const struct alg_test_desc *test_desc,
2532                            struct aead_request *req,
2533                            struct cipher_test_sglists *tsgls)
2534 {
2535         return 0;
2536 }
2537 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
2538
2539 static int test_aead(const char *driver, int enc,
2540                      const struct aead_test_suite *suite,
2541                      struct aead_request *req,
2542                      struct cipher_test_sglists *tsgls)
2543 {
2544         unsigned int i;
2545         int err;
2546
2547         for (i = 0; i < suite->count; i++) {
2548                 err = test_aead_vec(driver, enc, &suite->vecs[i], i, req,
2549                                     tsgls);
2550                 if (err)
2551                         return err;
2552                 cond_resched();
2553         }
2554         return 0;
2555 }
2556
2557 static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,
2558                          u32 type, u32 mask)
2559 {
2560         const struct aead_test_suite *suite = &desc->suite.aead;
2561         struct crypto_aead *tfm;
2562         struct aead_request *req = NULL;
2563         struct cipher_test_sglists *tsgls = NULL;
2564         int err;
2565
2566         if (suite->count <= 0) {
2567                 pr_err("alg: aead: empty test suite for %s\n", driver);
2568                 return -EINVAL;
2569         }
2570
2571         tfm = crypto_alloc_aead(driver, type, mask);
2572         if (IS_ERR(tfm)) {
2573                 pr_err("alg: aead: failed to allocate transform for %s: %ld\n",
2574                        driver, PTR_ERR(tfm));
2575                 return PTR_ERR(tfm);
2576         }
2577
2578         req = aead_request_alloc(tfm, GFP_KERNEL);
2579         if (!req) {
2580                 pr_err("alg: aead: failed to allocate request for %s\n",
2581                        driver);
2582                 err = -ENOMEM;
2583                 goto out;
2584         }
2585
2586         tsgls = alloc_cipher_test_sglists();
2587         if (!tsgls) {
2588                 pr_err("alg: aead: failed to allocate test buffers for %s\n",
2589                        driver);
2590                 err = -ENOMEM;
2591                 goto out;
2592         }
2593
2594         err = test_aead(driver, ENCRYPT, suite, req, tsgls);
2595         if (err)
2596                 goto out;
2597
2598         err = test_aead(driver, DECRYPT, suite, req, tsgls);
2599         if (err)
2600                 goto out;
2601
2602         err = test_aead_extra(driver, desc, req, tsgls);
2603 out:
2604         free_cipher_test_sglists(tsgls);
2605         aead_request_free(req);
2606         crypto_free_aead(tfm);
2607         return err;
2608 }
2609
2610 static int test_cipher(struct crypto_cipher *tfm, int enc,
2611                        const struct cipher_testvec *template,
2612                        unsigned int tcount)
2613 {
2614         const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm));
2615         unsigned int i, j, k;
2616         char *q;
2617         const char *e;
2618         const char *input, *result;
2619         void *data;
2620         char *xbuf[XBUFSIZE];
2621         int ret = -ENOMEM;
2622
2623         if (testmgr_alloc_buf(xbuf))
2624                 goto out_nobuf;
2625
2626         if (enc == ENCRYPT)
2627                 e = "encryption";
2628         else
2629                 e = "decryption";
2630
2631         j = 0;
2632         for (i = 0; i < tcount; i++) {
2633
2634                 if (fips_enabled && template[i].fips_skip)
2635                         continue;
2636
2637                 input  = enc ? template[i].ptext : template[i].ctext;
2638                 result = enc ? template[i].ctext : template[i].ptext;
2639                 j++;
2640
2641                 ret = -EINVAL;
2642                 if (WARN_ON(template[i].len > PAGE_SIZE))
2643                         goto out;
2644
2645                 data = xbuf[0];
2646                 memcpy(data, input, template[i].len);
2647
2648                 crypto_cipher_clear_flags(tfm, ~0);
2649                 if (template[i].wk)
2650                         crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2651
2652                 ret = crypto_cipher_setkey(tfm, template[i].key,
2653                                            template[i].klen);
2654                 if (ret) {
2655                         if (ret == template[i].setkey_error)
2656                                 continue;
2657                         pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n",
2658                                algo, j, template[i].setkey_error, ret,
2659                                crypto_cipher_get_flags(tfm));
2660                         goto out;
2661                 }
2662                 if (template[i].setkey_error) {
2663                         pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n",
2664                                algo, j, template[i].setkey_error);
2665                         ret = -EINVAL;
2666                         goto out;
2667                 }
2668
2669                 for (k = 0; k < template[i].len;
2670                      k += crypto_cipher_blocksize(tfm)) {
2671                         if (enc)
2672                                 crypto_cipher_encrypt_one(tfm, data + k,
2673                                                           data + k);
2674                         else
2675                                 crypto_cipher_decrypt_one(tfm, data + k,
2676                                                           data + k);
2677                 }
2678
2679                 q = data;
2680                 if (memcmp(q, result, template[i].len)) {
2681                         printk(KERN_ERR "alg: cipher: Test %d failed "
2682                                "on %s for %s\n", j, e, algo);
2683                         hexdump(q, template[i].len);
2684                         ret = -EINVAL;
2685                         goto out;
2686                 }
2687         }
2688
2689         ret = 0;
2690
2691 out:
2692         testmgr_free_buf(xbuf);
2693 out_nobuf:
2694         return ret;
2695 }
2696
2697 static int test_skcipher_vec_cfg(const char *driver, int enc,
2698                                  const struct cipher_testvec *vec,
2699                                  const char *vec_name,
2700                                  const struct testvec_config *cfg,
2701                                  struct skcipher_request *req,
2702                                  struct cipher_test_sglists *tsgls)
2703 {
2704         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2705         const unsigned int alignmask = crypto_skcipher_alignmask(tfm);
2706         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2707         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2708         const char *op = enc ? "encryption" : "decryption";
2709         DECLARE_CRYPTO_WAIT(wait);
2710         u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2711         u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2712                  cfg->iv_offset +
2713                  (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2714         struct kvec input;
2715         int err;
2716
2717         /* Set the key */
2718         if (vec->wk)
2719                 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2720         else
2721                 crypto_skcipher_clear_flags(tfm,
2722                                             CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2723         err = do_setkey(crypto_skcipher_setkey, tfm, vec->key, vec->klen,
2724                         cfg, alignmask);
2725         if (err) {
2726                 if (err == vec->setkey_error)
2727                         return 0;
2728                 pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2729                        driver, vec_name, vec->setkey_error, err,
2730                        crypto_skcipher_get_flags(tfm));
2731                 return err;
2732         }
2733         if (vec->setkey_error) {
2734                 pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2735                        driver, vec_name, vec->setkey_error);
2736                 return -EINVAL;
2737         }
2738
2739         /* The IV must be copied to a buffer, as the algorithm may modify it */
2740         if (ivsize) {
2741                 if (WARN_ON(ivsize > MAX_IVLEN))
2742                         return -EINVAL;
2743                 if (vec->generates_iv && !enc)
2744                         memcpy(iv, vec->iv_out, ivsize);
2745                 else if (vec->iv)
2746                         memcpy(iv, vec->iv, ivsize);
2747                 else
2748                         memset(iv, 0, ivsize);
2749         } else {
2750                 if (vec->generates_iv) {
2751                         pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n",
2752                                driver, vec_name);
2753                         return -EINVAL;
2754                 }
2755                 iv = NULL;
2756         }
2757
2758         /* Build the src/dst scatterlists */
2759         input.iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2760         input.iov_len = vec->len;
2761         err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2762                                         vec->len, vec->len, &input, 1);
2763         if (err) {
2764                 pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2765                        driver, op, vec_name, cfg->name);
2766                 return err;
2767         }
2768
2769         /* Do the actual encryption or decryption */
2770         testmgr_poison(req->__ctx, crypto_skcipher_reqsize(tfm));
2771         skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait);
2772         skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2773                                    vec->len, iv);
2774         if (cfg->nosimd)
2775                 crypto_disable_simd_for_test();
2776         err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
2777         if (cfg->nosimd)
2778                 crypto_reenable_simd_for_test();
2779         err = crypto_wait_req(err, &wait);
2780
2781         /* Check that the algorithm didn't overwrite things it shouldn't have */
2782         if (req->cryptlen != vec->len ||
2783             req->iv != iv ||
2784             req->src != tsgls->src.sgl_ptr ||
2785             req->dst != tsgls->dst.sgl_ptr ||
2786             crypto_skcipher_reqtfm(req) != tfm ||
2787             req->base.complete != crypto_req_done ||
2788             req->base.flags != req_flags ||
2789             req->base.data != &wait) {
2790                 pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2791                        driver, op, vec_name, cfg->name);
2792                 if (req->cryptlen != vec->len)
2793                         pr_err("alg: skcipher: changed 'req->cryptlen'\n");
2794                 if (req->iv != iv)
2795                         pr_err("alg: skcipher: changed 'req->iv'\n");
2796                 if (req->src != tsgls->src.sgl_ptr)
2797                         pr_err("alg: skcipher: changed 'req->src'\n");
2798                 if (req->dst != tsgls->dst.sgl_ptr)
2799                         pr_err("alg: skcipher: changed 'req->dst'\n");
2800                 if (crypto_skcipher_reqtfm(req) != tfm)
2801                         pr_err("alg: skcipher: changed 'req->base.tfm'\n");
2802                 if (req->base.complete != crypto_req_done)
2803                         pr_err("alg: skcipher: changed 'req->base.complete'\n");
2804                 if (req->base.flags != req_flags)
2805                         pr_err("alg: skcipher: changed 'req->base.flags'\n");
2806                 if (req->base.data != &wait)
2807                         pr_err("alg: skcipher: changed 'req->base.data'\n");
2808                 return -EINVAL;
2809         }
2810         if (is_test_sglist_corrupted(&tsgls->src)) {
2811                 pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2812                        driver, op, vec_name, cfg->name);
2813                 return -EINVAL;
2814         }
2815         if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2816             is_test_sglist_corrupted(&tsgls->dst)) {
2817                 pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2818                        driver, op, vec_name, cfg->name);
2819                 return -EINVAL;
2820         }
2821
2822         /* Check for success or failure */
2823         if (err) {
2824                 if (err == vec->crypt_error)
2825                         return 0;
2826                 pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
2827                        driver, op, vec_name, vec->crypt_error, err, cfg->name);
2828                 return err;
2829         }
2830         if (vec->crypt_error) {
2831                 pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
2832                        driver, op, vec_name, vec->crypt_error, cfg->name);
2833                 return -EINVAL;
2834         }
2835
2836         /* Check for the correct output (ciphertext or plaintext) */
2837         err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2838                                     vec->len, 0, true);
2839         if (err == -EOVERFLOW) {
2840                 pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2841                        driver, op, vec_name, cfg->name);
2842                 return err;
2843         }
2844         if (err) {
2845                 pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2846                        driver, op, vec_name, cfg->name);
2847                 return err;
2848         }
2849
2850         /* If applicable, check that the algorithm generated the correct IV */
2851         if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) {
2852                 pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n",
2853                        driver, op, vec_name, cfg->name);
2854                 hexdump(iv, ivsize);
2855                 return -EINVAL;
2856         }
2857
2858         return 0;
2859 }
2860
2861 static int test_skcipher_vec(const char *driver, int enc,
2862                              const struct cipher_testvec *vec,
2863                              unsigned int vec_num,
2864                              struct skcipher_request *req,
2865                              struct cipher_test_sglists *tsgls)
2866 {
2867         char vec_name[16];
2868         unsigned int i;
2869         int err;
2870
2871         if (fips_enabled && vec->fips_skip)
2872                 return 0;
2873
2874         sprintf(vec_name, "%u", vec_num);
2875
2876         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2877                 err = test_skcipher_vec_cfg(driver, enc, vec, vec_name,
2878                                             &default_cipher_testvec_configs[i],
2879                                             req, tsgls);
2880                 if (err)
2881                         return err;
2882         }
2883
2884 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2885         if (!noextratests) {
2886                 struct testvec_config cfg;
2887                 char cfgname[TESTVEC_CONFIG_NAMELEN];
2888
2889                 for (i = 0; i < fuzz_iterations; i++) {
2890                         generate_random_testvec_config(&cfg, cfgname,
2891                                                        sizeof(cfgname));
2892                         err = test_skcipher_vec_cfg(driver, enc, vec, vec_name,
2893                                                     &cfg, req, tsgls);
2894                         if (err)
2895                                 return err;
2896                         cond_resched();
2897                 }
2898         }
2899 #endif
2900         return 0;
2901 }
2902
2903 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2904 /*
2905  * Generate a symmetric cipher test vector from the given implementation.
2906  * Assumes the buffers in 'vec' were already allocated.
2907  */
2908 static void generate_random_cipher_testvec(struct skcipher_request *req,
2909                                            struct cipher_testvec *vec,
2910                                            unsigned int maxdatasize,
2911                                            char *name, size_t max_namelen)
2912 {
2913         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2914         const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
2915         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2916         struct scatterlist src, dst;
2917         u8 iv[MAX_IVLEN];
2918         DECLARE_CRYPTO_WAIT(wait);
2919
2920         /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
2921         vec->klen = maxkeysize;
2922         if (prandom_u32() % 4 == 0)
2923                 vec->klen = prandom_u32() % (maxkeysize + 1);
2924         generate_random_bytes((u8 *)vec->key, vec->klen);
2925         vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen);
2926
2927         /* IV */
2928         generate_random_bytes((u8 *)vec->iv, ivsize);
2929
2930         /* Plaintext */
2931         vec->len = generate_random_length(maxdatasize);
2932         generate_random_bytes((u8 *)vec->ptext, vec->len);
2933
2934         /* If the key couldn't be set, no need to continue to encrypt. */
2935         if (vec->setkey_error)
2936                 goto done;
2937
2938         /* Ciphertext */
2939         sg_init_one(&src, vec->ptext, vec->len);
2940         sg_init_one(&dst, vec->ctext, vec->len);
2941         memcpy(iv, vec->iv, ivsize);
2942         skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
2943         skcipher_request_set_crypt(req, &src, &dst, vec->len, iv);
2944         vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
2945         if (vec->crypt_error != 0) {
2946                 /*
2947                  * The only acceptable error here is for an invalid length, so
2948                  * skcipher decryption should fail with the same error too.
2949                  * We'll test for this.  But to keep the API usage well-defined,
2950                  * explicitly initialize the ciphertext buffer too.
2951                  */
2952                 memset((u8 *)vec->ctext, 0, vec->len);
2953         }
2954 done:
2955         snprintf(name, max_namelen, "\"random: len=%u klen=%u\"",
2956                  vec->len, vec->klen);
2957 }
2958
2959 /*
2960  * Test the skcipher algorithm represented by @req against the corresponding
2961  * generic implementation, if one is available.
2962  */
2963 static int test_skcipher_vs_generic_impl(const char *driver,
2964                                          const char *generic_driver,
2965                                          struct skcipher_request *req,
2966                                          struct cipher_test_sglists *tsgls)
2967 {
2968         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2969         const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
2970         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2971         const unsigned int blocksize = crypto_skcipher_blocksize(tfm);
2972         const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
2973         const char *algname = crypto_skcipher_alg(tfm)->base.cra_name;
2974         char _generic_driver[CRYPTO_MAX_ALG_NAME];
2975         struct crypto_skcipher *generic_tfm = NULL;
2976         struct skcipher_request *generic_req = NULL;
2977         unsigned int i;
2978         struct cipher_testvec vec = { 0 };
2979         char vec_name[64];
2980         struct testvec_config *cfg;
2981         char cfgname[TESTVEC_CONFIG_NAMELEN];
2982         int err;
2983
2984         if (noextratests)
2985                 return 0;
2986
2987         /* Keywrap isn't supported here yet as it handles its IV differently. */
2988         if (strncmp(algname, "kw(", 3) == 0)
2989                 return 0;
2990
2991         if (!generic_driver) { /* Use default naming convention? */
2992                 err = build_generic_driver_name(algname, _generic_driver);
2993                 if (err)
2994                         return err;
2995                 generic_driver = _generic_driver;
2996         }
2997
2998         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
2999                 return 0;
3000
3001         generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0);
3002         if (IS_ERR(generic_tfm)) {
3003                 err = PTR_ERR(generic_tfm);
3004                 if (err == -ENOENT) {
3005                         pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n",
3006                                 driver, generic_driver);
3007                         return 0;
3008                 }
3009                 pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n",
3010                        generic_driver, algname, err);
3011                 return err;
3012         }
3013
3014         cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
3015         if (!cfg) {
3016                 err = -ENOMEM;
3017                 goto out;
3018         }
3019
3020         generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
3021         if (!generic_req) {
3022                 err = -ENOMEM;
3023                 goto out;
3024         }
3025
3026         /* Check the algorithm properties for consistency. */
3027
3028         if (crypto_skcipher_min_keysize(tfm) !=
3029             crypto_skcipher_min_keysize(generic_tfm)) {
3030                 pr_err("alg: skcipher: min keysize for %s (%u) doesn't match generic impl (%u)\n",
3031                        driver, crypto_skcipher_min_keysize(tfm),
3032                        crypto_skcipher_min_keysize(generic_tfm));
3033                 err = -EINVAL;
3034                 goto out;
3035         }
3036
3037         if (maxkeysize != crypto_skcipher_max_keysize(generic_tfm)) {
3038                 pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n",
3039                        driver, maxkeysize,
3040                        crypto_skcipher_max_keysize(generic_tfm));
3041                 err = -EINVAL;
3042                 goto out;
3043         }
3044
3045         if (ivsize != crypto_skcipher_ivsize(generic_tfm)) {
3046                 pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n",
3047                        driver, ivsize, crypto_skcipher_ivsize(generic_tfm));
3048                 err = -EINVAL;
3049                 goto out;
3050         }
3051
3052         if (blocksize != crypto_skcipher_blocksize(generic_tfm)) {
3053                 pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n",
3054                        driver, blocksize,
3055                        crypto_skcipher_blocksize(generic_tfm));
3056                 err = -EINVAL;
3057                 goto out;
3058         }
3059
3060         /*
3061          * Now generate test vectors using the generic implementation, and test
3062          * the other implementation against them.
3063          */
3064
3065         vec.key = kmalloc(maxkeysize, GFP_KERNEL);
3066         vec.iv = kmalloc(ivsize, GFP_KERNEL);
3067         vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);
3068         vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);
3069         if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) {
3070                 err = -ENOMEM;
3071                 goto out;
3072         }
3073
3074         for (i = 0; i < fuzz_iterations * 8; i++) {
3075                 generate_random_cipher_testvec(generic_req, &vec, maxdatasize,
3076                                                vec_name, sizeof(vec_name));
3077                 generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
3078
3079                 err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name,
3080                                             cfg, req, tsgls);
3081                 if (err)
3082                         goto out;
3083                 err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name,
3084                                             cfg, req, tsgls);
3085                 if (err)
3086                         goto out;
3087                 cond_resched();
3088         }
3089         err = 0;
3090 out:
3091         kfree(cfg);
3092         kfree(vec.key);
3093         kfree(vec.iv);
3094         kfree(vec.ptext);
3095         kfree(vec.ctext);
3096         crypto_free_skcipher(generic_tfm);
3097         skcipher_request_free(generic_req);
3098         return err;
3099 }
3100 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
3101 static int test_skcipher_vs_generic_impl(const char *driver,
3102                                          const char *generic_driver,
3103                                          struct skcipher_request *req,
3104                                          struct cipher_test_sglists *tsgls)
3105 {
3106         return 0;
3107 }
3108 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
3109
3110 static int test_skcipher(const char *driver, int enc,
3111                          const struct cipher_test_suite *suite,
3112                          struct skcipher_request *req,
3113                          struct cipher_test_sglists *tsgls)
3114 {
3115         unsigned int i;
3116         int err;
3117
3118         for (i = 0; i < suite->count; i++) {
3119                 err = test_skcipher_vec(driver, enc, &suite->vecs[i], i, req,
3120                                         tsgls);
3121                 if (err)
3122                         return err;
3123                 cond_resched();
3124         }
3125         return 0;
3126 }
3127
3128 static int alg_test_skcipher(const struct alg_test_desc *desc,
3129                              const char *driver, u32 type, u32 mask)
3130 {
3131         const struct cipher_test_suite *suite = &desc->suite.cipher;
3132         struct crypto_skcipher *tfm;
3133         struct skcipher_request *req = NULL;
3134         struct cipher_test_sglists *tsgls = NULL;
3135         int err;
3136
3137         if (suite->count <= 0) {
3138                 pr_err("alg: skcipher: empty test suite for %s\n", driver);
3139                 return -EINVAL;
3140         }
3141
3142         tfm = crypto_alloc_skcipher(driver, type, mask);
3143         if (IS_ERR(tfm)) {
3144                 pr_err("alg: skcipher: failed to allocate transform for %s: %ld\n",
3145                        driver, PTR_ERR(tfm));
3146                 return PTR_ERR(tfm);
3147         }
3148
3149         req = skcipher_request_alloc(tfm, GFP_KERNEL);
3150         if (!req) {
3151                 pr_err("alg: skcipher: failed to allocate request for %s\n",
3152                        driver);
3153                 err = -ENOMEM;
3154                 goto out;
3155         }
3156
3157         tsgls = alloc_cipher_test_sglists();
3158         if (!tsgls) {
3159                 pr_err("alg: skcipher: failed to allocate test buffers for %s\n",
3160                        driver);
3161                 err = -ENOMEM;
3162                 goto out;
3163         }
3164
3165         err = test_skcipher(driver, ENCRYPT, suite, req, tsgls);
3166         if (err)
3167                 goto out;
3168
3169         err = test_skcipher(driver, DECRYPT, suite, req, tsgls);
3170         if (err)
3171                 goto out;
3172
3173         err = test_skcipher_vs_generic_impl(driver, desc->generic_driver, req,
3174                                             tsgls);
3175 out:
3176         free_cipher_test_sglists(tsgls);
3177         skcipher_request_free(req);
3178         crypto_free_skcipher(tfm);
3179         return err;
3180 }
3181
3182 static int test_comp(struct crypto_comp *tfm,
3183                      const struct comp_testvec *ctemplate,
3184                      const struct comp_testvec *dtemplate,
3185                      int ctcount, int dtcount)
3186 {
3187         const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm));
3188         char *output, *decomp_output;
3189         unsigned int i;
3190         int ret;
3191
3192         output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3193         if (!output)
3194                 return -ENOMEM;
3195
3196         decomp_output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3197         if (!decomp_output) {
3198                 kfree(output);
3199                 return -ENOMEM;
3200         }
3201
3202         for (i = 0; i < ctcount; i++) {
3203                 int ilen;
3204                 unsigned int dlen = COMP_BUF_SIZE;
3205
3206                 memset(output, 0, COMP_BUF_SIZE);
3207                 memset(decomp_output, 0, COMP_BUF_SIZE);
3208
3209                 ilen = ctemplate[i].inlen;
3210                 ret = crypto_comp_compress(tfm, ctemplate[i].input,
3211                                            ilen, output, &dlen);
3212                 if (ret) {
3213                         printk(KERN_ERR "alg: comp: compression failed "
3214                                "on test %d for %s: ret=%d\n", i + 1, algo,
3215                                -ret);
3216                         goto out;
3217                 }
3218
3219                 ilen = dlen;
3220                 dlen = COMP_BUF_SIZE;
3221                 ret = crypto_comp_decompress(tfm, output,
3222                                              ilen, decomp_output, &dlen);
3223                 if (ret) {
3224                         pr_err("alg: comp: compression failed: decompress: on test %d for %s failed: ret=%d\n",
3225                                i + 1, algo, -ret);
3226                         goto out;
3227                 }
3228
3229                 if (dlen != ctemplate[i].inlen) {
3230                         printk(KERN_ERR "alg: comp: Compression test %d "
3231                                "failed for %s: output len = %d\n", i + 1, algo,
3232                                dlen);
3233                         ret = -EINVAL;
3234                         goto out;
3235                 }
3236
3237                 if (memcmp(decomp_output, ctemplate[i].input,
3238                            ctemplate[i].inlen)) {
3239                         pr_err("alg: comp: compression failed: output differs: on test %d for %s\n",
3240                                i + 1, algo);
3241                         hexdump(decomp_output, dlen);
3242                         ret = -EINVAL;
3243                         goto out;
3244                 }
3245         }
3246
3247         for (i = 0; i < dtcount; i++) {
3248                 int ilen;
3249                 unsigned int dlen = COMP_BUF_SIZE;
3250
3251                 memset(decomp_output, 0, COMP_BUF_SIZE);
3252
3253                 ilen = dtemplate[i].inlen;
3254                 ret = crypto_comp_decompress(tfm, dtemplate[i].input,
3255                                              ilen, decomp_output, &dlen);
3256                 if (ret) {
3257                         printk(KERN_ERR "alg: comp: decompression failed "
3258                                "on test %d for %s: ret=%d\n", i + 1, algo,
3259                                -ret);
3260                         goto out;
3261                 }
3262
3263                 if (dlen != dtemplate[i].outlen) {
3264                         printk(KERN_ERR "alg: comp: Decompression test %d "
3265                                "failed for %s: output len = %d\n", i + 1, algo,
3266                                dlen);
3267                         ret = -EINVAL;
3268                         goto out;
3269                 }
3270
3271                 if (memcmp(decomp_output, dtemplate[i].output, dlen)) {
3272                         printk(KERN_ERR "alg: comp: Decompression test %d "
3273                                "failed for %s\n", i + 1, algo);
3274                         hexdump(decomp_output, dlen);
3275                         ret = -EINVAL;
3276                         goto out;
3277                 }
3278         }
3279
3280         ret = 0;
3281
3282 out:
3283         kfree(decomp_output);
3284         kfree(output);
3285         return ret;
3286 }
3287
3288 static int test_acomp(struct crypto_acomp *tfm,
3289                               const struct comp_testvec *ctemplate,
3290                       const struct comp_testvec *dtemplate,
3291                       int ctcount, int dtcount)
3292 {
3293         const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));
3294         unsigned int i;
3295         char *output, *decomp_out;
3296         int ret;
3297         struct scatterlist src, dst;
3298         struct acomp_req *req;
3299         struct crypto_wait wait;
3300
3301         output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3302         if (!output)
3303                 return -ENOMEM;
3304
3305         decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3306         if (!decomp_out) {
3307                 kfree(output);
3308                 return -ENOMEM;
3309         }
3310
3311         for (i = 0; i < ctcount; i++) {
3312                 unsigned int dlen = COMP_BUF_SIZE;
3313                 int ilen = ctemplate[i].inlen;
3314                 void *input_vec;
3315
3316                 input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL);
3317                 if (!input_vec) {
3318                         ret = -ENOMEM;
3319                         goto out;
3320                 }
3321
3322                 memset(output, 0, dlen);
3323                 crypto_init_wait(&wait);
3324                 sg_init_one(&src, input_vec, ilen);
3325                 sg_init_one(&dst, output, dlen);
3326
3327                 req = acomp_request_alloc(tfm);
3328                 if (!req) {
3329                         pr_err("alg: acomp: request alloc failed for %s\n",
3330                                algo);
3331                         kfree(input_vec);
3332                         ret = -ENOMEM;
3333                         goto out;
3334                 }
3335
3336                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
3337                 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3338                                            crypto_req_done, &wait);
3339
3340                 ret = crypto_wait_req(crypto_acomp_compress(req), &wait);
3341                 if (ret) {
3342                         pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3343                                i + 1, algo, -ret);
3344                         kfree(input_vec);
3345                         acomp_request_free(req);
3346                         goto out;
3347                 }
3348
3349                 ilen = req->dlen;
3350                 dlen = COMP_BUF_SIZE;
3351                 sg_init_one(&src, output, ilen);
3352                 sg_init_one(&dst, decomp_out, dlen);
3353                 crypto_init_wait(&wait);
3354                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
3355
3356                 ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3357                 if (ret) {
3358                         pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3359                                i + 1, algo, -ret);
3360                         kfree(input_vec);
3361                         acomp_request_free(req);
3362                         goto out;
3363                 }
3364
3365                 if (req->dlen != ctemplate[i].inlen) {
3366                         pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n",
3367                                i + 1, algo, req->dlen);
3368                         ret = -EINVAL;
3369                         kfree(input_vec);
3370                         acomp_request_free(req);
3371                         goto out;
3372                 }
3373
3374                 if (memcmp(input_vec, decomp_out, req->dlen)) {
3375                         pr_err("alg: acomp: Compression test %d failed for %s\n",
3376                                i + 1, algo);
3377                         hexdump(output, req->dlen);
3378                         ret = -EINVAL;
3379                         kfree(input_vec);
3380                         acomp_request_free(req);
3381                         goto out;
3382                 }
3383
3384                 kfree(input_vec);
3385                 acomp_request_free(req);
3386         }
3387
3388         for (i = 0; i < dtcount; i++) {
3389                 unsigned int dlen = COMP_BUF_SIZE;
3390                 int ilen = dtemplate[i].inlen;
3391                 void *input_vec;
3392
3393                 input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL);
3394                 if (!input_vec) {
3395                         ret = -ENOMEM;
3396                         goto out;
3397                 }
3398
3399                 memset(output, 0, dlen);
3400                 crypto_init_wait(&wait);
3401                 sg_init_one(&src, input_vec, ilen);
3402                 sg_init_one(&dst, output, dlen);
3403
3404                 req = acomp_request_alloc(tfm);
3405                 if (!req) {
3406                         pr_err("alg: acomp: request alloc failed for %s\n",
3407                                algo);
3408                         kfree(input_vec);
3409                         ret = -ENOMEM;
3410                         goto out;
3411                 }
3412
3413                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
3414                 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3415                                            crypto_req_done, &wait);
3416
3417                 ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3418                 if (ret) {
3419                         pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n",
3420                                i + 1, algo, -ret);
3421                         kfree(input_vec);
3422                         acomp_request_free(req);
3423                         goto out;
3424                 }
3425
3426                 if (req->dlen != dtemplate[i].outlen) {
3427                         pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n",
3428                                i + 1, algo, req->dlen);
3429                         ret = -EINVAL;
3430                         kfree(input_vec);
3431                         acomp_request_free(req);
3432                         goto out;
3433                 }
3434
3435                 if (memcmp(output, dtemplate[i].output, req->dlen)) {
3436                         pr_err("alg: acomp: Decompression test %d failed for %s\n",
3437                                i + 1, algo);
3438                         hexdump(output, req->dlen);
3439                         ret = -EINVAL;
3440                         kfree(input_vec);
3441                         acomp_request_free(req);
3442                         goto out;
3443                 }
3444
3445                 kfree(input_vec);
3446                 acomp_request_free(req);
3447         }
3448
3449         ret = 0;
3450
3451 out:
3452         kfree(decomp_out);
3453         kfree(output);
3454         return ret;
3455 }
3456
3457 static int test_cprng(struct crypto_rng *tfm,
3458                       const struct cprng_testvec *template,
3459                       unsigned int tcount)
3460 {
3461         const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm));
3462         int err = 0, i, j, seedsize;
3463         u8 *seed;
3464         char result[32];
3465
3466         seedsize = crypto_rng_seedsize(tfm);
3467
3468         seed = kmalloc(seedsize, GFP_KERNEL);
3469         if (!seed) {
3470                 printk(KERN_ERR "alg: cprng: Failed to allocate seed space "
3471                        "for %s\n", algo);
3472                 return -ENOMEM;
3473         }
3474
3475         for (i = 0; i < tcount; i++) {
3476                 memset(result, 0, 32);
3477
3478                 memcpy(seed, template[i].v, template[i].vlen);
3479                 memcpy(seed + template[i].vlen, template[i].key,
3480                        template[i].klen);
3481                 memcpy(seed + template[i].vlen + template[i].klen,
3482                        template[i].dt, template[i].dtlen);
3483
3484                 err = crypto_rng_reset(tfm, seed, seedsize);
3485                 if (err) {
3486                         printk(KERN_ERR "alg: cprng: Failed to reset rng "
3487                                "for %s\n", algo);
3488                         goto out;
3489                 }
3490
3491                 for (j = 0; j < template[i].loops; j++) {
3492                         err = crypto_rng_get_bytes(tfm, result,
3493                                                    template[i].rlen);
3494                         if (err < 0) {
3495                                 printk(KERN_ERR "alg: cprng: Failed to obtain "
3496                                        "the correct amount of random data for "
3497                                        "%s (requested %d)\n", algo,
3498                                        template[i].rlen);
3499                                 goto out;
3500                         }
3501                 }
3502
3503                 err = memcmp(result, template[i].result,
3504                              template[i].rlen);
3505                 if (err) {
3506                         printk(KERN_ERR "alg: cprng: Test %d failed for %s\n",
3507                                i, algo);
3508                         hexdump(result, template[i].rlen);
3509                         err = -EINVAL;
3510                         goto out;
3511                 }
3512         }
3513
3514 out:
3515         kfree(seed);
3516         return err;
3517 }
3518
3519 static int alg_test_cipher(const struct alg_test_desc *desc,
3520                            const char *driver, u32 type, u32 mask)
3521 {
3522         const struct cipher_test_suite *suite = &desc->suite.cipher;
3523         struct crypto_cipher *tfm;
3524         int err;
3525
3526         tfm = crypto_alloc_cipher(driver, type, mask);
3527         if (IS_ERR(tfm)) {
3528                 printk(KERN_ERR "alg: cipher: Failed to load transform for "
3529                        "%s: %ld\n", driver, PTR_ERR(tfm));
3530                 return PTR_ERR(tfm);
3531         }
3532
3533         err = test_cipher(tfm, ENCRYPT, suite->vecs, suite->count);
3534         if (!err)
3535                 err = test_cipher(tfm, DECRYPT, suite->vecs, suite->count);
3536
3537         crypto_free_cipher(tfm);
3538         return err;
3539 }
3540
3541 static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
3542                          u32 type, u32 mask)
3543 {
3544         struct crypto_comp *comp;
3545         struct crypto_acomp *acomp;
3546         int err;
3547         u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK;
3548
3549         if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) {
3550                 acomp = crypto_alloc_acomp(driver, type, mask);
3551                 if (IS_ERR(acomp)) {
3552                         pr_err("alg: acomp: Failed to load transform for %s: %ld\n",
3553                                driver, PTR_ERR(acomp));
3554                         return PTR_ERR(acomp);
3555                 }
3556                 err = test_acomp(acomp, desc->suite.comp.comp.vecs,
3557                                  desc->suite.comp.decomp.vecs,
3558                                  desc->suite.comp.comp.count,
3559                                  desc->suite.comp.decomp.count);
3560                 crypto_free_acomp(acomp);
3561         } else {
3562                 comp = crypto_alloc_comp(driver, type, mask);
3563                 if (IS_ERR(comp)) {
3564                         pr_err("alg: comp: Failed to load transform for %s: %ld\n",
3565                                driver, PTR_ERR(comp));
3566                         return PTR_ERR(comp);
3567                 }
3568
3569                 err = test_comp(comp, desc->suite.comp.comp.vecs,
3570                                 desc->suite.comp.decomp.vecs,
3571                                 desc->suite.comp.comp.count,
3572                                 desc->suite.comp.decomp.count);
3573
3574                 crypto_free_comp(comp);
3575         }
3576         return err;
3577 }
3578
3579 static int alg_test_crc32c(const struct alg_test_desc *desc,
3580                            const char *driver, u32 type, u32 mask)
3581 {
3582         struct crypto_shash *tfm;
3583         __le32 val;
3584         int err;
3585
3586         err = alg_test_hash(desc, driver, type, mask);
3587         if (err)
3588                 return err;
3589
3590         tfm = crypto_alloc_shash(driver, type, mask);
3591         if (IS_ERR(tfm)) {
3592                 if (PTR_ERR(tfm) == -ENOENT) {
3593                         /*
3594                          * This crc32c implementation is only available through
3595                          * ahash API, not the shash API, so the remaining part
3596                          * of the test is not applicable to it.
3597                          */
3598                         return 0;
3599                 }
3600                 printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: "
3601                        "%ld\n", driver, PTR_ERR(tfm));
3602                 return PTR_ERR(tfm);
3603         }
3604
3605         do {
3606                 SHASH_DESC_ON_STACK(shash, tfm);
3607                 u32 *ctx = (u32 *)shash_desc_ctx(shash);
3608
3609                 shash->tfm = tfm;
3610
3611                 *ctx = 420553207;
3612                 err = crypto_shash_final(shash, (u8 *)&val);
3613                 if (err) {
3614                         printk(KERN_ERR "alg: crc32c: Operation failed for "
3615                                "%s: %d\n", driver, err);
3616                         break;
3617                 }
3618
3619                 if (val != cpu_to_le32(~420553207)) {
3620                         pr_err("alg: crc32c: Test failed for %s: %u\n",
3621                                driver, le32_to_cpu(val));
3622                         err = -EINVAL;
3623                 }
3624         } while (0);
3625
3626         crypto_free_shash(tfm);
3627
3628         return err;
3629 }
3630
3631 static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver,
3632                           u32 type, u32 mask)
3633 {
3634         struct crypto_rng *rng;
3635         int err;
3636
3637         rng = crypto_alloc_rng(driver, type, mask);
3638         if (IS_ERR(rng)) {
3639                 printk(KERN_ERR "alg: cprng: Failed to load transform for %s: "
3640                        "%ld\n", driver, PTR_ERR(rng));
3641                 return PTR_ERR(rng);
3642         }
3643
3644         err = test_cprng(rng, desc->suite.cprng.vecs, desc->suite.cprng.count);
3645
3646         crypto_free_rng(rng);
3647
3648         return err;
3649 }
3650
3651
3652 static int drbg_cavs_test(const struct drbg_testvec *test, int pr,
3653                           const char *driver, u32 type, u32 mask)
3654 {
3655         int ret = -EAGAIN;
3656         struct crypto_rng *drng;
3657         struct drbg_test_data test_data;
3658         struct drbg_string addtl, pers, testentropy;
3659         unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL);
3660
3661         if (!buf)
3662                 return -ENOMEM;
3663
3664         drng = crypto_alloc_rng(driver, type, mask);
3665         if (IS_ERR(drng)) {
3666                 printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "
3667                        "%s\n", driver);
3668                 kzfree(buf);
3669                 return -ENOMEM;
3670         }
3671
3672         test_data.testentropy = &testentropy;
3673         drbg_string_fill(&testentropy, test->entropy, test->entropylen);
3674         drbg_string_fill(&pers, test->pers, test->perslen);
3675         ret = crypto_drbg_reset_test(drng, &pers, &test_data);
3676         if (ret) {
3677                 printk(KERN_ERR "alg: drbg: Failed to reset rng\n");
3678                 goto outbuf;
3679         }
3680
3681         drbg_string_fill(&addtl, test->addtla, test->addtllen);
3682         if (pr) {
3683                 drbg_string_fill(&testentropy, test->entpra, test->entprlen);
3684                 ret = crypto_drbg_get_bytes_addtl_test(drng,
3685                         buf, test->expectedlen, &addtl, &test_data);
3686         } else {
3687                 ret = crypto_drbg_get_bytes_addtl(drng,
3688                         buf, test->expectedlen, &addtl);
3689         }
3690         if (ret < 0) {
3691                 printk(KERN_ERR "alg: drbg: could not obtain random data for "
3692                        "driver %s\n", driver);
3693                 goto outbuf;
3694         }
3695
3696         drbg_string_fill(&addtl, test->addtlb, test->addtllen);
3697         if (pr) {
3698                 drbg_string_fill(&testentropy, test->entprb, test->entprlen);
3699                 ret = crypto_drbg_get_bytes_addtl_test(drng,
3700                         buf, test->expectedlen, &addtl, &test_data);
3701         } else {
3702                 ret = crypto_drbg_get_bytes_addtl(drng,
3703                         buf, test->expectedlen, &addtl);
3704         }
3705         if (ret < 0) {
3706                 printk(KERN_ERR "alg: drbg: could not obtain random data for "
3707                        "driver %s\n", driver);
3708                 goto outbuf;
3709         }
3710
3711         ret = memcmp(test->expected, buf, test->expectedlen);
3712
3713 outbuf:
3714         crypto_free_rng(drng);
3715         kzfree(buf);
3716         return ret;
3717 }
3718
3719
3720 static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver,
3721                          u32 type, u32 mask)
3722 {
3723         int err = 0;
3724         int pr = 0;
3725         int i = 0;
3726         const struct drbg_testvec *template = desc->suite.drbg.vecs;
3727         unsigned int tcount = desc->suite.drbg.count;
3728
3729         if (0 == memcmp(driver, "drbg_pr_", 8))
3730                 pr = 1;
3731
3732         for (i = 0; i < tcount; i++) {
3733                 err = drbg_cavs_test(&template[i], pr, driver, type, mask);
3734                 if (err) {
3735                         printk(KERN_ERR "alg: drbg: Test %d failed for %s\n",
3736                                i, driver);
3737                         err = -EINVAL;
3738                         break;
3739                 }
3740         }
3741         return err;
3742
3743 }
3744
3745 static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
3746                        const char *alg)
3747 {
3748         struct kpp_request *req;
3749         void *input_buf = NULL;
3750         void *output_buf = NULL;
3751         void *a_public = NULL;
3752         void *a_ss = NULL;
3753         void *shared_secret = NULL;
3754         struct crypto_wait wait;
3755         unsigned int out_len_max;
3756         int err = -ENOMEM;
3757         struct scatterlist src, dst;
3758
3759         req = kpp_request_alloc(tfm, GFP_KERNEL);
3760         if (!req)
3761                 return err;
3762
3763         crypto_init_wait(&wait);
3764
3765         err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size);
3766         if (err < 0)
3767                 goto free_req;
3768
3769         out_len_max = crypto_kpp_maxsize(tfm);
3770         output_buf = kzalloc(out_len_max, GFP_KERNEL);
3771         if (!output_buf) {
3772                 err = -ENOMEM;
3773                 goto free_req;
3774         }
3775
3776         /* Use appropriate parameter as base */
3777         kpp_request_set_input(req, NULL, 0);
3778         sg_init_one(&dst, output_buf, out_len_max);
3779         kpp_request_set_output(req, &dst, out_len_max);
3780         kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3781                                  crypto_req_done, &wait);
3782
3783         /* Compute party A's public key */
3784         err = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait);
3785         if (err) {
3786                 pr_err("alg: %s: Party A: generate public key test failed. err %d\n",
3787                        alg, err);
3788                 goto free_output;
3789         }
3790
3791         if (vec->genkey) {
3792                 /* Save party A's public key */
3793                 a_public = kmemdup(sg_virt(req->dst), out_len_max, GFP_KERNEL);
3794                 if (!a_public) {
3795                         err = -ENOMEM;
3796                         goto free_output;
3797                 }
3798         } else {
3799                 /* Verify calculated public key */
3800                 if (memcmp(vec->expected_a_public, sg_virt(req->dst),
3801                            vec->expected_a_public_size)) {
3802                         pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n",
3803                                alg);
3804                         err = -EINVAL;
3805                         goto free_output;
3806                 }
3807         }
3808
3809         /* Calculate shared secret key by using counter part (b) public key. */
3810         input_buf = kmemdup(vec->b_public, vec->b_public_size, GFP_KERNEL);
3811         if (!input_buf) {
3812                 err = -ENOMEM;
3813                 goto free_output;
3814         }
3815
3816         sg_init_one(&src, input_buf, vec->b_public_size);
3817         sg_init_one(&dst, output_buf, out_len_max);
3818         kpp_request_set_input(req, &src, vec->b_public_size);
3819         kpp_request_set_output(req, &dst, out_len_max);
3820         kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3821                                  crypto_req_done, &wait);
3822         err = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait);
3823         if (err) {
3824                 pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n",
3825                        alg, err);
3826                 goto free_all;
3827         }
3828
3829         if (vec->genkey) {
3830                 /* Save the shared secret obtained by party A */
3831                 a_ss = kmemdup(sg_virt(req->dst), vec->expected_ss_size, GFP_KERNEL);
3832                 if (!a_ss) {
3833                         err = -ENOMEM;
3834                         goto free_all;
3835                 }
3836
3837                 /*
3838                  * Calculate party B's shared secret by using party A's
3839                  * public key.
3840                  */
3841                 err = crypto_kpp_set_secret(tfm, vec->b_secret,
3842                                             vec->b_secret_size);
3843                 if (err < 0)
3844                         goto free_all;
3845
3846                 sg_init_one(&src, a_public, vec->expected_a_public_size);
3847                 sg_init_one(&dst, output_buf, out_len_max);
3848                 kpp_request_set_input(req, &src, vec->expected_a_public_size);
3849                 kpp_request_set_output(req, &dst, out_len_max);
3850                 kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3851                                          crypto_req_done, &wait);
3852                 err = crypto_wait_req(crypto_kpp_compute_shared_secret(req),
3853                                       &wait);
3854                 if (err) {
3855                         pr_err("alg: %s: Party B: compute shared secret failed. err %d\n",
3856                                alg, err);
3857                         goto free_all;
3858                 }
3859
3860                 shared_secret = a_ss;
3861         } else {
3862                 shared_secret = (void *)vec->expected_ss;
3863         }
3864
3865         /*
3866          * verify shared secret from which the user will derive
3867          * secret key by executing whatever hash it has chosen
3868          */
3869         if (memcmp(shared_secret, sg_virt(req->dst),
3870                    vec->expected_ss_size)) {
3871                 pr_err("alg: %s: compute shared secret test failed. Invalid output\n",
3872                        alg);
3873                 err = -EINVAL;
3874         }
3875
3876 free_all:
3877         kfree(a_ss);
3878         kfree(input_buf);
3879 free_output:
3880         kfree(a_public);
3881         kfree(output_buf);
3882 free_req:
3883         kpp_request_free(req);
3884         return err;
3885 }
3886
3887 static int test_kpp(struct crypto_kpp *tfm, const char *alg,
3888                     const struct kpp_testvec *vecs, unsigned int tcount)
3889 {
3890         int ret, i;
3891
3892         for (i = 0; i < tcount; i++) {
3893                 ret = do_test_kpp(tfm, vecs++, alg);
3894                 if (ret) {
3895                         pr_err("alg: %s: test failed on vector %d, err=%d\n",
3896                                alg, i + 1, ret);
3897                         return ret;
3898                 }
3899         }
3900         return 0;
3901 }
3902
3903 static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,
3904                         u32 type, u32 mask)
3905 {
3906         struct crypto_kpp *tfm;
3907         int err = 0;
3908
3909         tfm = crypto_alloc_kpp(driver, type, mask);
3910         if (IS_ERR(tfm)) {
3911                 pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",
3912                        driver, PTR_ERR(tfm));
3913                 return PTR_ERR(tfm);
3914         }
3915         if (desc->suite.kpp.vecs)
3916                 err = test_kpp(tfm, desc->alg, desc->suite.kpp.vecs,
3917                                desc->suite.kpp.count);
3918
3919         crypto_free_kpp(tfm);
3920         return err;
3921 }
3922
3923 static u8 *test_pack_u32(u8 *dst, u32 val)
3924 {
3925         memcpy(dst, &val, sizeof(val));
3926         return dst + sizeof(val);
3927 }
3928
3929 static int test_akcipher_one(struct crypto_akcipher *tfm,
3930                              const struct akcipher_testvec *vecs)
3931 {
3932         char *xbuf[XBUFSIZE];
3933         struct akcipher_request *req;
3934         void *outbuf_enc = NULL;
3935         void *outbuf_dec = NULL;
3936         struct crypto_wait wait;
3937         unsigned int out_len_max, out_len = 0;
3938         int err = -ENOMEM;
3939         struct scatterlist src, dst, src_tab[3];
3940         const char *m, *c;
3941         unsigned int m_size, c_size;
3942         const char *op;
3943         u8 *key, *ptr;
3944
3945         if (testmgr_alloc_buf(xbuf))
3946                 return err;
3947
3948         req = akcipher_request_alloc(tfm, GFP_KERNEL);
3949         if (!req)
3950                 goto free_xbuf;
3951
3952         crypto_init_wait(&wait);
3953
3954         key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len,
3955                       GFP_KERNEL);
3956         if (!key)
3957                 goto free_xbuf;
3958         memcpy(key, vecs->key, vecs->key_len);
3959         ptr = key + vecs->key_len;
3960         ptr = test_pack_u32(ptr, vecs->algo);
3961         ptr = test_pack_u32(ptr, vecs->param_len);
3962         memcpy(ptr, vecs->params, vecs->param_len);
3963
3964         if (vecs->public_key_vec)
3965                 err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len);
3966         else
3967                 err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len);
3968         if (err)
3969                 goto free_req;
3970
3971         /*
3972          * First run test which do not require a private key, such as
3973          * encrypt or verify.
3974          */
3975         err = -ENOMEM;
3976         out_len_max = crypto_akcipher_maxsize(tfm);
3977         outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
3978         if (!outbuf_enc)
3979                 goto free_req;
3980
3981         if (!vecs->siggen_sigver_test) {
3982                 m = vecs->m;
3983                 m_size = vecs->m_size;
3984                 c = vecs->c;
3985                 c_size = vecs->c_size;
3986                 op = "encrypt";
3987         } else {
3988                 /* Swap args so we could keep plaintext (digest)
3989                  * in vecs->m, and cooked signature in vecs->c.
3990                  */
3991                 m = vecs->c; /* signature */
3992                 m_size = vecs->c_size;
3993                 c = vecs->m; /* digest */
3994                 c_size = vecs->m_size;
3995                 op = "verify";
3996         }
3997
3998         if (WARN_ON(m_size > PAGE_SIZE))
3999                 goto free_all;
4000         memcpy(xbuf[0], m, m_size);
4001
4002         sg_init_table(src_tab, 3);
4003         sg_set_buf(&src_tab[0], xbuf[0], 8);
4004         sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8);
4005         if (vecs->siggen_sigver_test) {
4006                 if (WARN_ON(c_size > PAGE_SIZE))
4007                         goto free_all;
4008                 memcpy(xbuf[1], c, c_size);
4009                 sg_set_buf(&src_tab[2], xbuf[1], c_size);
4010                 akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size);
4011         } else {
4012                 sg_init_one(&dst, outbuf_enc, out_len_max);
4013                 akcipher_request_set_crypt(req, src_tab, &dst, m_size,
4014                                            out_len_max);
4015         }
4016         akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
4017                                       crypto_req_done, &wait);
4018
4019         err = crypto_wait_req(vecs->siggen_sigver_test ?
4020                               /* Run asymmetric signature verification */
4021                               crypto_akcipher_verify(req) :
4022                               /* Run asymmetric encrypt */
4023                               crypto_akcipher_encrypt(req), &wait);
4024         if (err) {
4025                 pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
4026                 goto free_all;
4027         }
4028         if (!vecs->siggen_sigver_test) {
4029                 if (req->dst_len != c_size) {
4030                         pr_err("alg: akcipher: %s test failed. Invalid output len\n",
4031                                op);
4032                         err = -EINVAL;
4033                         goto free_all;
4034                 }
4035                 /* verify that encrypted message is equal to expected */
4036                 if (memcmp(c, outbuf_enc, c_size) != 0) {
4037                         pr_err("alg: akcipher: %s test failed. Invalid output\n",
4038                                op);
4039                         hexdump(outbuf_enc, c_size);
4040                         err = -EINVAL;
4041                         goto free_all;
4042                 }
4043         }
4044
4045         /*
4046          * Don't invoke (decrypt or sign) test which require a private key
4047          * for vectors with only a public key.
4048          */
4049         if (vecs->public_key_vec) {
4050                 err = 0;
4051                 goto free_all;
4052         }
4053         outbuf_dec = kzalloc(out_len_max, GFP_KERNEL);
4054         if (!outbuf_dec) {
4055                 err = -ENOMEM;
4056                 goto free_all;
4057         }
4058
4059         op = vecs->siggen_sigver_test ? "sign" : "decrypt";
4060         if (WARN_ON(c_size > PAGE_SIZE))
4061                 goto free_all;
4062         memcpy(xbuf[0], c, c_size);
4063
4064         sg_init_one(&src, xbuf[0], c_size);
4065         sg_init_one(&dst, outbuf_dec, out_len_max);
4066         crypto_init_wait(&wait);
4067         akcipher_request_set_crypt(req, &src, &dst, c_size, out_len_max);
4068
4069         err = crypto_wait_req(vecs->siggen_sigver_test ?
4070                               /* Run asymmetric signature generation */
4071                               crypto_akcipher_sign(req) :
4072                               /* Run asymmetric decrypt */
4073                               crypto_akcipher_decrypt(req), &wait);
4074         if (err) {
4075                 pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
4076                 goto free_all;
4077         }
4078         out_len = req->dst_len;
4079         if (out_len < m_size) {
4080                 pr_err("alg: akcipher: %s test failed. Invalid output len %u\n",
4081                        op, out_len);
4082                 err = -EINVAL;
4083                 goto free_all;
4084         }
4085         /* verify that decrypted message is equal to the original msg */
4086         if (memchr_inv(outbuf_dec, 0, out_len - m_size) ||
4087             memcmp(m, outbuf_dec + out_len - m_size, m_size)) {
4088                 pr_err("alg: akcipher: %s test failed. Invalid output\n", op);
4089                 hexdump(outbuf_dec, out_len);
4090                 err = -EINVAL;
4091         }
4092 free_all:
4093         kfree(outbuf_dec);
4094         kfree(outbuf_enc);
4095 free_req:
4096         akcipher_request_free(req);
4097         kfree(key);
4098 free_xbuf:
4099         testmgr_free_buf(xbuf);
4100         return err;
4101 }
4102
4103 static int test_akcipher(struct crypto_akcipher *tfm, const char *alg,
4104                          const struct akcipher_testvec *vecs,
4105                          unsigned int tcount)
4106 {
4107         const char *algo =
4108                 crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm));
4109         int ret, i;
4110
4111         for (i = 0; i < tcount; i++) {
4112                 ret = test_akcipher_one(tfm, vecs++);
4113                 if (!ret)
4114                         continue;
4115
4116                 pr_err("alg: akcipher: test %d failed for %s, err=%d\n",
4117                        i + 1, algo, ret);
4118                 return ret;
4119         }
4120         return 0;
4121 }
4122
4123 static int alg_test_akcipher(const struct alg_test_desc *desc,
4124                              const char *driver, u32 type, u32 mask)
4125 {
4126         struct crypto_akcipher *tfm;
4127         int err = 0;
4128
4129         tfm = crypto_alloc_akcipher(driver, type, mask);
4130         if (IS_ERR(tfm)) {
4131                 pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n",
4132                        driver, PTR_ERR(tfm));
4133                 return PTR_ERR(tfm);
4134         }
4135         if (desc->suite.akcipher.vecs)
4136                 err = test_akcipher(tfm, desc->alg, desc->suite.akcipher.vecs,
4137                                     desc->suite.akcipher.count);
4138
4139         crypto_free_akcipher(tfm);
4140         return err;
4141 }
4142
4143 static int alg_test_null(const struct alg_test_desc *desc,
4144                              const char *driver, u32 type, u32 mask)
4145 {
4146         return 0;
4147 }
4148
4149 #define ____VECS(tv)    .vecs = tv, .count = ARRAY_SIZE(tv)
4150 #define __VECS(tv)      { ____VECS(tv) }
4151
4152 /* Please keep this list sorted by algorithm name. */
4153 static const struct alg_test_desc alg_test_descs[] = {
4154         {
4155                 .alg = "adiantum(xchacha12,aes)",
4156                 .generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)",
4157                 .test = alg_test_skcipher,
4158                 .suite = {
4159                         .cipher = __VECS(adiantum_xchacha12_aes_tv_template)
4160                 },
4161         }, {
4162                 .alg = "adiantum(xchacha20,aes)",
4163                 .generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)",
4164                 .test = alg_test_skcipher,
4165                 .suite = {
4166                         .cipher = __VECS(adiantum_xchacha20_aes_tv_template)
4167                 },
4168         }, {
4169                 .alg = "aegis128",
4170                 .test = alg_test_aead,
4171                 .suite = {
4172                         .aead = __VECS(aegis128_tv_template)
4173                 }
4174         }, {
4175                 .alg = "ansi_cprng",
4176                 .test = alg_test_cprng,
4177                 .suite = {
4178                         .cprng = __VECS(ansi_cprng_aes_tv_template)
4179                 }
4180         }, {
4181                 .alg = "authenc(hmac(md5),ecb(cipher_null))",
4182                 .test = alg_test_aead,
4183                 .suite = {
4184                         .aead = __VECS(hmac_md5_ecb_cipher_null_tv_template)
4185                 }
4186         }, {
4187                 .alg = "authenc(hmac(sha1),cbc(aes))",
4188                 .test = alg_test_aead,
4189                 .fips_allowed = 1,
4190                 .suite = {
4191                         .aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
4192                 }
4193         }, {
4194                 .alg = "authenc(hmac(sha1),cbc(des))",
4195                 .test = alg_test_aead,
4196                 .suite = {
4197                         .aead = __VECS(hmac_sha1_des_cbc_tv_temp)
4198                 }
4199         }, {
4200                 .alg = "authenc(hmac(sha1),cbc(des3_ede))",
4201                 .test = alg_test_aead,
4202                 .fips_allowed = 1,
4203                 .suite = {
4204                         .aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
4205                 }
4206         }, {
4207                 .alg = "authenc(hmac(sha1),ctr(aes))",
4208                 .test = alg_test_null,
4209                 .fips_allowed = 1,
4210         }, {
4211                 .alg = "authenc(hmac(sha1),ecb(cipher_null))",
4212                 .test = alg_test_aead,
4213                 .suite = {
4214                         .aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)
4215                 }
4216         }, {
4217                 .alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4218                 .test = alg_test_null,
4219                 .fips_allowed = 1,
4220         }, {
4221                 .alg = "authenc(hmac(sha224),cbc(des))",
4222                 .test = alg_test_aead,
4223                 .suite = {
4224                         .aead = __VECS(hmac_sha224_des_cbc_tv_temp)
4225                 }
4226         }, {
4227                 .alg = "authenc(hmac(sha224),cbc(des3_ede))",
4228                 .test = alg_test_aead,
4229                 .fips_allowed = 1,
4230                 .suite = {
4231                         .aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
4232                 }
4233         }, {
4234                 .alg = "authenc(hmac(sha256),cbc(aes))",
4235                 .test = alg_test_aead,
4236                 .fips_allowed = 1,
4237                 .suite = {
4238                         .aead = __VECS(hmac_sha256_aes_cbc_tv_temp)
4239                 }
4240         }, {
4241                 .alg = "authenc(hmac(sha256),cbc(des))",
4242                 .test = alg_test_aead,
4243                 .suite = {
4244                         .aead = __VECS(hmac_sha256_des_cbc_tv_temp)
4245                 }
4246         }, {
4247                 .alg = "authenc(hmac(sha256),cbc(des3_ede))",
4248                 .test = alg_test_aead,
4249                 .fips_allowed = 1,
4250                 .suite = {
4251                         .aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
4252                 }
4253         }, {
4254                 .alg = "authenc(hmac(sha256),ctr(aes))",
4255                 .test = alg_test_null,
4256                 .fips_allowed = 1,
4257         }, {
4258                 .alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4259                 .test = alg_test_null,
4260                 .fips_allowed = 1,
4261         }, {
4262                 .alg = "authenc(hmac(sha384),cbc(des))",
4263                 .test = alg_test_aead,
4264                 .suite = {
4265                         .aead = __VECS(hmac_sha384_des_cbc_tv_temp)
4266                 }
4267         }, {
4268                 .alg = "authenc(hmac(sha384),cbc(des3_ede))",
4269                 .test = alg_test_aead,
4270                 .fips_allowed = 1,
4271                 .suite = {
4272                         .aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
4273                 }
4274         }, {
4275                 .alg = "authenc(hmac(sha384),ctr(aes))",
4276                 .test = alg_test_null,
4277                 .fips_allowed = 1,
4278         }, {
4279                 .alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4280                 .test = alg_test_null,
4281                 .fips_allowed = 1,
4282         }, {
4283                 .alg = "authenc(hmac(sha512),cbc(aes))",
4284                 .fips_allowed = 1,
4285                 .test = alg_test_aead,
4286                 .suite = {
4287                         .aead = __VECS(hmac_sha512_aes_cbc_tv_temp)
4288                 }
4289         }, {
4290                 .alg = "authenc(hmac(sha512),cbc(des))",
4291                 .test = alg_test_aead,
4292                 .suite = {
4293                         .aead = __VECS(hmac_sha512_des_cbc_tv_temp)
4294                 }
4295         }, {
4296                 .alg = "authenc(hmac(sha512),cbc(des3_ede))",
4297                 .test = alg_test_aead,
4298                 .fips_allowed = 1,
4299                 .suite = {
4300                         .aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
4301                 }
4302         }, {
4303                 .alg = "authenc(hmac(sha512),ctr(aes))",
4304                 .test = alg_test_null,
4305                 .fips_allowed = 1,
4306         }, {
4307                 .alg = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4308                 .test = alg_test_null,
4309                 .fips_allowed = 1,
4310         }, {
4311                 .alg = "blake2b-160",
4312                 .test = alg_test_hash,
4313                 .fips_allowed = 0,
4314                 .suite = {
4315                         .hash = __VECS(blake2b_160_tv_template)
4316                 }
4317         }, {
4318                 .alg = "blake2b-256",
4319                 .test = alg_test_hash,
4320                 .fips_allowed = 0,
4321                 .suite = {
4322                         .hash = __VECS(blake2b_256_tv_template)
4323                 }
4324         }, {
4325                 .alg = "blake2b-384",
4326                 .test = alg_test_hash,
4327                 .fips_allowed = 0,
4328                 .suite = {
4329                         .hash = __VECS(blake2b_384_tv_template)
4330                 }
4331         }, {
4332                 .alg = "blake2b-512",
4333                 .test = alg_test_hash,
4334                 .fips_allowed = 0,
4335                 .suite = {
4336                         .hash = __VECS(blake2b_512_tv_template)
4337                 }
4338         }, {
4339                 .alg = "blake2s-128",
4340                 .test = alg_test_hash,
4341                 .suite = {
4342                         .hash = __VECS(blakes2s_128_tv_template)
4343                 }
4344         }, {
4345                 .alg = "blake2s-160",
4346                 .test = alg_test_hash,
4347                 .suite = {
4348                         .hash = __VECS(blakes2s_160_tv_template)
4349                 }
4350         }, {
4351                 .alg = "blake2s-224",
4352                 .test = alg_test_hash,
4353                 .suite = {
4354                         .hash = __VECS(blakes2s_224_tv_template)
4355                 }
4356         }, {
4357                 .alg = "blake2s-256",
4358                 .test = alg_test_hash,
4359                 .suite = {
4360                         .hash = __VECS(blakes2s_256_tv_template)
4361                 }
4362         }, {
4363                 .alg = "cbc(aes)",
4364                 .test = alg_test_skcipher,
4365                 .fips_allowed = 1,
4366                 .suite = {
4367                         .cipher = __VECS(aes_cbc_tv_template)
4368                 },
4369         }, {
4370                 .alg = "cbc(anubis)",
4371                 .test = alg_test_skcipher,
4372                 .suite = {
4373                         .cipher = __VECS(anubis_cbc_tv_template)
4374                 },
4375         }, {
4376                 .alg = "cbc(blowfish)",
4377                 .test = alg_test_skcipher,
4378                 .suite = {
4379                         .cipher = __VECS(bf_cbc_tv_template)
4380                 },
4381         }, {
4382                 .alg = "cbc(camellia)",
4383                 .test = alg_test_skcipher,
4384                 .suite = {
4385                         .cipher = __VECS(camellia_cbc_tv_template)
4386                 },
4387         }, {
4388                 .alg = "cbc(cast5)",
4389                 .test = alg_test_skcipher,
4390                 .suite = {
4391                         .cipher = __VECS(cast5_cbc_tv_template)
4392                 },
4393         }, {
4394                 .alg = "cbc(cast6)",
4395                 .test = alg_test_skcipher,
4396                 .suite = {
4397                         .cipher = __VECS(cast6_cbc_tv_template)
4398                 },
4399         }, {
4400                 .alg = "cbc(des)",
4401                 .test = alg_test_skcipher,
4402                 .suite = {
4403                         .cipher = __VECS(des_cbc_tv_template)
4404                 },
4405         }, {
4406                 .alg = "cbc(des3_ede)",
4407                 .test = alg_test_skcipher,
4408                 .fips_allowed = 1,
4409                 .suite = {
4410                         .cipher = __VECS(des3_ede_cbc_tv_template)
4411                 },
4412         }, {
4413                 /* Same as cbc(aes) except the key is stored in
4414                  * hardware secure memory which we reference by index
4415                  */
4416                 .alg = "cbc(paes)",
4417                 .test = alg_test_null,
4418                 .fips_allowed = 1,
4419         }, {
4420                 /* Same as cbc(sm4) except the key is stored in
4421                  * hardware secure memory which we reference by index
4422                  */
4423                 .alg = "cbc(psm4)",
4424                 .test = alg_test_null,
4425         }, {
4426                 .alg = "cbc(serpent)",
4427                 .test = alg_test_skcipher,
4428                 .suite = {
4429                         .cipher = __VECS(serpent_cbc_tv_template)
4430                 },
4431         }, {
4432                 .alg = "cbc(sm4)",
4433                 .test = alg_test_skcipher,
4434                 .suite = {
4435                         .cipher = __VECS(sm4_cbc_tv_template)
4436                 }
4437         }, {
4438                 .alg = "cbc(twofish)",
4439                 .test = alg_test_skcipher,
4440                 .suite = {
4441                         .cipher = __VECS(tf_cbc_tv_template)
4442                 },
4443         }, {
4444 #if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4445                 .alg = "cbc-paes-s390",
4446                 .fips_allowed = 1,
4447                 .test = alg_test_skcipher,
4448                 .suite = {
4449                         .cipher = __VECS(aes_cbc_tv_template)
4450                 }
4451         }, {
4452 #endif
4453                 .alg = "cbcmac(aes)",
4454                 .fips_allowed = 1,
4455                 .test = alg_test_hash,
4456                 .suite = {
4457                         .hash = __VECS(aes_cbcmac_tv_template)
4458                 }
4459         }, {
4460                 .alg = "ccm(aes)",
4461                 .generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))",
4462                 .test = alg_test_aead,
4463                 .fips_allowed = 1,
4464                 .suite = {
4465                         .aead = {
4466                                 ____VECS(aes_ccm_tv_template),
4467                                 .einval_allowed = 1,
4468                         }
4469                 }
4470         }, {
4471                 .alg = "cfb(aes)",
4472                 .test = alg_test_skcipher,
4473                 .fips_allowed = 1,
4474                 .suite = {
4475                         .cipher = __VECS(aes_cfb_tv_template)
4476                 },
4477         }, {
4478                 .alg = "cfb(sm4)",
4479                 .test = alg_test_skcipher,
4480                 .suite = {
4481                         .cipher = __VECS(sm4_cfb_tv_template)
4482                 }
4483         }, {
4484                 .alg = "chacha20",
4485                 .test = alg_test_skcipher,
4486                 .suite = {
4487                         .cipher = __VECS(chacha20_tv_template)
4488                 },
4489         }, {
4490                 .alg = "cmac(aes)",
4491                 .fips_allowed = 1,
4492                 .test = alg_test_hash,
4493                 .suite = {
4494                         .hash = __VECS(aes_cmac128_tv_template)
4495                 }
4496         }, {
4497                 .alg = "cmac(des3_ede)",
4498                 .fips_allowed = 1,
4499                 .test = alg_test_hash,
4500                 .suite = {
4501                         .hash = __VECS(des3_ede_cmac64_tv_template)
4502                 }
4503         }, {
4504                 .alg = "compress_null",
4505                 .test = alg_test_null,
4506         }, {
4507                 .alg = "crc32",
4508                 .test = alg_test_hash,
4509                 .fips_allowed = 1,
4510                 .suite = {
4511                         .hash = __VECS(crc32_tv_template)
4512                 }
4513         }, {
4514                 .alg = "crc32c",
4515                 .test = alg_test_crc32c,
4516                 .fips_allowed = 1,
4517                 .suite = {
4518                         .hash = __VECS(crc32c_tv_template)
4519                 }
4520         }, {
4521                 .alg = "crct10dif",
4522                 .test = alg_test_hash,
4523                 .fips_allowed = 1,
4524                 .suite = {
4525                         .hash = __VECS(crct10dif_tv_template)
4526                 }
4527         }, {
4528                 .alg = "ctr(aes)",
4529                 .test = alg_test_skcipher,
4530                 .fips_allowed = 1,
4531                 .suite = {
4532                         .cipher = __VECS(aes_ctr_tv_template)
4533                 }
4534         }, {
4535                 .alg = "ctr(blowfish)",
4536                 .test = alg_test_skcipher,
4537                 .suite = {
4538                         .cipher = __VECS(bf_ctr_tv_template)
4539                 }
4540         }, {
4541                 .alg = "ctr(camellia)",
4542                 .test = alg_test_skcipher,
4543                 .suite = {
4544                         .cipher = __VECS(camellia_ctr_tv_template)
4545                 }
4546         }, {
4547                 .alg = "ctr(cast5)",
4548                 .test = alg_test_skcipher,
4549                 .suite = {
4550                         .cipher = __VECS(cast5_ctr_tv_template)
4551                 }
4552         }, {
4553                 .alg = "ctr(cast6)",
4554                 .test = alg_test_skcipher,
4555                 .suite = {
4556                         .cipher = __VECS(cast6_ctr_tv_template)
4557                 }
4558         }, {
4559                 .alg = "ctr(des)",
4560                 .test = alg_test_skcipher,
4561                 .suite = {
4562                         .cipher = __VECS(des_ctr_tv_template)
4563                 }
4564         }, {
4565                 .alg = "ctr(des3_ede)",
4566                 .test = alg_test_skcipher,
4567                 .fips_allowed = 1,
4568                 .suite = {
4569                         .cipher = __VECS(des3_ede_ctr_tv_template)
4570                 }
4571         }, {
4572                 /* Same as ctr(aes) except the key is stored in
4573                  * hardware secure memory which we reference by index
4574                  */
4575                 .alg = "ctr(paes)",
4576                 .test = alg_test_null,
4577                 .fips_allowed = 1,
4578         }, {
4579
4580                 /* Same as ctr(sm4) except the key is stored in
4581                  * hardware secure memory which we reference by index
4582                  */
4583                 .alg = "ctr(psm4)",
4584                 .test = alg_test_null,
4585         }, {
4586                 .alg = "ctr(serpent)",
4587                 .test = alg_test_skcipher,
4588                 .suite = {
4589                         .cipher = __VECS(serpent_ctr_tv_template)
4590                 }
4591         }, {
4592                 .alg = "ctr(sm4)",
4593                 .test = alg_test_skcipher,
4594                 .suite = {
4595                         .cipher = __VECS(sm4_ctr_tv_template)
4596                 }
4597         }, {
4598                 .alg = "ctr(twofish)",
4599                 .test = alg_test_skcipher,
4600                 .suite = {
4601                         .cipher = __VECS(tf_ctr_tv_template)
4602                 }
4603         }, {
4604 #if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4605                 .alg = "ctr-paes-s390",
4606                 .fips_allowed = 1,
4607                 .test = alg_test_skcipher,
4608                 .suite = {
4609                         .cipher = __VECS(aes_ctr_tv_template)
4610                 }
4611         }, {
4612 #endif
4613                 .alg = "cts(cbc(aes))",
4614                 .test = alg_test_skcipher,
4615                 .fips_allowed = 1,
4616                 .suite = {
4617                         .cipher = __VECS(cts_mode_tv_template)
4618                 }
4619         }, {
4620                 /* Same as cts(cbc((aes)) except the key is stored in
4621                  * hardware secure memory which we reference by index
4622                  */
4623                 .alg = "cts(cbc(paes))",
4624                 .test = alg_test_null,
4625                 .fips_allowed = 1,
4626         }, {
4627                 .alg = "curve25519",
4628                 .test = alg_test_kpp,
4629                 .suite = {
4630                         .kpp = __VECS(curve25519_tv_template)
4631                 }
4632         }, {
4633                 .alg = "deflate",
4634                 .test = alg_test_comp,
4635                 .fips_allowed = 1,
4636                 .suite = {
4637                         .comp = {
4638                                 .comp = __VECS(deflate_comp_tv_template),
4639                                 .decomp = __VECS(deflate_decomp_tv_template)
4640                         }
4641                 }
4642         }, {
4643                 .alg = "dh",
4644                 .test = alg_test_kpp,
4645                 .fips_allowed = 1,
4646                 .suite = {
4647                         .kpp = __VECS(dh_tv_template)
4648                 }
4649         }, {
4650                 .alg = "digest_null",
4651                 .test = alg_test_null,
4652         }, {
4653                 .alg = "drbg_nopr_ctr_aes128",
4654                 .test = alg_test_drbg,
4655                 .fips_allowed = 1,
4656                 .suite = {
4657                         .drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)
4658                 }
4659         }, {
4660                 .alg = "drbg_nopr_ctr_aes192",
4661                 .test = alg_test_drbg,
4662                 .fips_allowed = 1,
4663                 .suite = {
4664                         .drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)
4665                 }
4666         }, {
4667                 .alg = "drbg_nopr_ctr_aes256",
4668                 .test = alg_test_drbg,
4669                 .fips_allowed = 1,
4670                 .suite = {
4671                         .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
4672                 }
4673         }, {
4674                 /*
4675                  * There is no need to specifically test the DRBG with every
4676                  * backend cipher -- covered by drbg_nopr_hmac_sha256 test
4677                  */
4678                 .alg = "drbg_nopr_hmac_sha1",
4679                 .fips_allowed = 1,
4680                 .test = alg_test_null,
4681         }, {
4682                 .alg = "drbg_nopr_hmac_sha256",
4683                 .test = alg_test_drbg,
4684                 .fips_allowed = 1,
4685                 .suite = {
4686                         .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
4687                 }
4688         }, {
4689                 /* covered by drbg_nopr_hmac_sha256 test */
4690                 .alg = "drbg_nopr_hmac_sha384",
4691                 .fips_allowed = 1,
4692                 .test = alg_test_null,
4693         }, {
4694                 .alg = "drbg_nopr_hmac_sha512",
4695                 .test = alg_test_null,
4696                 .fips_allowed = 1,
4697         }, {
4698                 .alg = "drbg_nopr_sha1",
4699                 .fips_allowed = 1,
4700                 .test = alg_test_null,
4701         }, {
4702                 .alg = "drbg_nopr_sha256",
4703                 .test = alg_test_drbg,
4704                 .fips_allowed = 1,
4705                 .suite = {
4706                         .drbg = __VECS(drbg_nopr_sha256_tv_template)
4707                 }
4708         }, {
4709                 /* covered by drbg_nopr_sha256 test */
4710                 .alg = "drbg_nopr_sha384",
4711                 .fips_allowed = 1,
4712                 .test = alg_test_null,
4713         }, {
4714                 .alg = "drbg_nopr_sha512",
4715                 .fips_allowed = 1,
4716                 .test = alg_test_null,
4717         }, {
4718                 .alg = "drbg_pr_ctr_aes128",
4719                 .test = alg_test_drbg,
4720                 .fips_allowed = 1,
4721                 .suite = {
4722                         .drbg = __VECS(drbg_pr_ctr_aes128_tv_template)
4723                 }
4724         }, {
4725                 /* covered by drbg_pr_ctr_aes128 test */
4726                 .alg = "drbg_pr_ctr_aes192",
4727                 .fips_allowed = 1,
4728                 .test = alg_test_null,
4729         }, {
4730                 .alg = "drbg_pr_ctr_aes256",
4731                 .fips_allowed = 1,
4732                 .test = alg_test_null,
4733         }, {
4734                 .alg = "drbg_pr_hmac_sha1",
4735                 .fips_allowed = 1,
4736                 .test = alg_test_null,
4737         }, {
4738                 .alg = "drbg_pr_hmac_sha256",
4739                 .test = alg_test_drbg,
4740                 .fips_allowed = 1,
4741                 .suite = {
4742                         .drbg = __VECS(drbg_pr_hmac_sha256_tv_template)
4743                 }
4744         }, {
4745                 /* covered by drbg_pr_hmac_sha256 test */
4746                 .alg = "drbg_pr_hmac_sha384",
4747                 .fips_allowed = 1,
4748                 .test = alg_test_null,
4749         }, {
4750                 .alg = "drbg_pr_hmac_sha512",
4751                 .test = alg_test_null,
4752                 .fips_allowed = 1,
4753         }, {
4754                 .alg = "drbg_pr_sha1",
4755                 .fips_allowed = 1,
4756                 .test = alg_test_null,
4757         }, {
4758                 .alg = "drbg_pr_sha256",
4759                 .test = alg_test_drbg,
4760                 .fips_allowed = 1,
4761                 .suite = {
4762                         .drbg = __VECS(drbg_pr_sha256_tv_template)
4763                 }
4764         }, {
4765                 /* covered by drbg_pr_sha256 test */
4766                 .alg = "drbg_pr_sha384",
4767                 .fips_allowed = 1,
4768                 .test = alg_test_null,
4769         }, {
4770                 .alg = "drbg_pr_sha512",
4771                 .fips_allowed = 1,
4772                 .test = alg_test_null,
4773         }, {
4774                 .alg = "ecb(aes)",
4775                 .test = alg_test_skcipher,
4776                 .fips_allowed = 1,
4777                 .suite = {
4778                         .cipher = __VECS(aes_tv_template)
4779                 }
4780         }, {
4781                 .alg = "ecb(anubis)",
4782                 .test = alg_test_skcipher,
4783                 .suite = {
4784                         .cipher = __VECS(anubis_tv_template)
4785                 }
4786         }, {
4787                 .alg = "ecb(arc4)",
4788                 .generic_driver = "ecb(arc4)-generic",
4789                 .test = alg_test_skcipher,
4790                 .suite = {
4791                         .cipher = __VECS(arc4_tv_template)
4792                 }
4793         }, {
4794                 .alg = "ecb(blowfish)",
4795                 .test = alg_test_skcipher,
4796                 .suite = {
4797                         .cipher = __VECS(bf_tv_template)
4798                 }
4799         }, {
4800                 .alg = "ecb(camellia)",
4801                 .test = alg_test_skcipher,
4802                 .suite = {
4803                         .cipher = __VECS(camellia_tv_template)
4804                 }
4805         }, {
4806                 .alg = "ecb(cast5)",
4807                 .test = alg_test_skcipher,
4808                 .suite = {
4809                         .cipher = __VECS(cast5_tv_template)
4810                 }
4811         }, {
4812                 .alg = "ecb(cast6)",
4813                 .test = alg_test_skcipher,
4814                 .suite = {
4815                         .cipher = __VECS(cast6_tv_template)
4816                 }
4817         }, {
4818                 .alg = "ecb(cipher_null)",
4819                 .test = alg_test_null,
4820                 .fips_allowed = 1,
4821         }, {
4822                 .alg = "ecb(des)",
4823                 .test = alg_test_skcipher,
4824                 .suite = {
4825                         .cipher = __VECS(des_tv_template)
4826                 }
4827         }, {
4828                 .alg = "ecb(des3_ede)",
4829                 .test = alg_test_skcipher,
4830                 .fips_allowed = 1,
4831                 .suite = {
4832                         .cipher = __VECS(des3_ede_tv_template)
4833                 }
4834         }, {
4835                 .alg = "ecb(fcrypt)",
4836                 .test = alg_test_skcipher,
4837                 .suite = {
4838                         .cipher = {
4839                                 .vecs = fcrypt_pcbc_tv_template,
4840                                 .count = 1
4841                         }
4842                 }
4843         }, {
4844                 .alg = "ecb(khazad)",
4845                 .test = alg_test_skcipher,
4846                 .suite = {
4847                         .cipher = __VECS(khazad_tv_template)
4848                 }
4849         }, {
4850                 /* Same as ecb(aes) except the key is stored in
4851                  * hardware secure memory which we reference by index
4852                  */
4853                 .alg = "ecb(paes)",
4854                 .test = alg_test_null,
4855                 .fips_allowed = 1,
4856         }, {
4857                 .alg = "ecb(seed)",
4858                 .test = alg_test_skcipher,
4859                 .suite = {
4860                         .cipher = __VECS(seed_tv_template)
4861                 }
4862         }, {
4863                 .alg = "ecb(serpent)",
4864                 .test = alg_test_skcipher,
4865                 .suite = {
4866                         .cipher = __VECS(serpent_tv_template)
4867                 }
4868         }, {
4869                 .alg = "ecb(sm4)",
4870                 .test = alg_test_skcipher,
4871                 .suite = {
4872                         .cipher = __VECS(sm4_tv_template)
4873                 }
4874         }, {
4875                 .alg = "ecb(tea)",
4876                 .test = alg_test_skcipher,
4877                 .suite = {
4878                         .cipher = __VECS(tea_tv_template)
4879                 }
4880         }, {
4881                 .alg = "ecb(tnepres)",
4882                 .test = alg_test_skcipher,
4883                 .suite = {
4884                         .cipher = __VECS(tnepres_tv_template)
4885                 }
4886         }, {
4887                 .alg = "ecb(twofish)",
4888                 .test = alg_test_skcipher,
4889                 .suite = {
4890                         .cipher = __VECS(tf_tv_template)
4891                 }
4892         }, {
4893                 .alg = "ecb(xeta)",
4894                 .test = alg_test_skcipher,
4895                 .suite = {
4896                         .cipher = __VECS(xeta_tv_template)
4897                 }
4898         }, {
4899                 .alg = "ecb(xtea)",
4900                 .test = alg_test_skcipher,
4901                 .suite = {
4902                         .cipher = __VECS(xtea_tv_template)
4903                 }
4904         }, {
4905 #if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4906                 .alg = "ecb-paes-s390",
4907                 .fips_allowed = 1,
4908                 .test = alg_test_skcipher,
4909                 .suite = {
4910                         .cipher = __VECS(aes_tv_template)
4911                 }
4912         }, {
4913 #endif
4914                 .alg = "ecdh",
4915                 .test = alg_test_kpp,
4916                 .fips_allowed = 1,
4917                 .suite = {
4918                         .kpp = __VECS(ecdh_tv_template)
4919                 }
4920         }, {
4921                 .alg = "ecrdsa",
4922                 .test = alg_test_akcipher,
4923                 .suite = {
4924                         .akcipher = __VECS(ecrdsa_tv_template)
4925                 }
4926         }, {
4927                 .alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)",
4928                 .test = alg_test_aead,
4929                 .fips_allowed = 1,
4930                 .suite = {
4931                         .aead = __VECS(essiv_hmac_sha256_aes_cbc_tv_temp)
4932                 }
4933         }, {
4934                 .alg = "essiv(cbc(aes),sha256)",
4935                 .test = alg_test_skcipher,
4936                 .fips_allowed = 1,
4937                 .suite = {
4938                         .cipher = __VECS(essiv_aes_cbc_tv_template)
4939                 }
4940         }, {
4941                 .alg = "gcm(aes)",
4942                 .generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)",
4943                 .test = alg_test_aead,
4944                 .fips_allowed = 1,
4945                 .suite = {
4946                         .aead = __VECS(aes_gcm_tv_template)
4947                 }
4948         }, {
4949                 .alg = "ghash",
4950                 .test = alg_test_hash,
4951                 .fips_allowed = 1,
4952                 .suite = {
4953                         .hash = __VECS(ghash_tv_template)
4954                 }
4955         }, {
4956                 .alg = "hmac(md5)",
4957                 .test = alg_test_hash,
4958                 .suite = {
4959                         .hash = __VECS(hmac_md5_tv_template)
4960                 }
4961         }, {
4962                 .alg = "hmac(rmd128)",
4963                 .test = alg_test_hash,
4964                 .suite = {
4965                         .hash = __VECS(hmac_rmd128_tv_template)
4966                 }
4967         }, {
4968                 .alg = "hmac(rmd160)",
4969                 .test = alg_test_hash,
4970                 .suite = {
4971                         .hash = __VECS(hmac_rmd160_tv_template)
4972                 }
4973         }, {
4974                 .alg = "hmac(sha1)",
4975                 .test = alg_test_hash,
4976                 .fips_allowed = 1,
4977                 .suite = {
4978                         .hash = __VECS(hmac_sha1_tv_template)
4979                 }
4980         }, {
4981                 .alg = "hmac(sha224)",
4982                 .test = alg_test_hash,
4983                 .fips_allowed = 1,
4984                 .suite = {
4985                         .hash = __VECS(hmac_sha224_tv_template)
4986                 }
4987         }, {
4988                 .alg = "hmac(sha256)",
4989                 .test = alg_test_hash,
4990                 .fips_allowed = 1,
4991                 .suite = {
4992                         .hash = __VECS(hmac_sha256_tv_template)
4993                 }
4994         }, {
4995                 .alg = "hmac(sha3-224)",
4996                 .test = alg_test_hash,
4997                 .fips_allowed = 1,
4998                 .suite = {
4999                         .hash = __VECS(hmac_sha3_224_tv_template)
5000                 }
5001         }, {
5002                 .alg = "hmac(sha3-256)",
5003                 .test = alg_test_hash,
5004                 .fips_allowed = 1,
5005                 .suite = {
5006                         .hash = __VECS(hmac_sha3_256_tv_template)
5007                 }
5008         }, {
5009                 .alg = "hmac(sha3-384)",
5010                 .test = alg_test_hash,
5011                 .fips_allowed = 1,
5012                 .suite = {
5013                         .hash = __VECS(hmac_sha3_384_tv_template)
5014                 }
5015         }, {
5016                 .alg = "hmac(sha3-512)",
5017                 .test = alg_test_hash,
5018                 .fips_allowed = 1,
5019                 .suite = {
5020                         .hash = __VECS(hmac_sha3_512_tv_template)
5021                 }
5022         }, {
5023                 .alg = "hmac(sha384)",
5024                 .test = alg_test_hash,
5025                 .fips_allowed = 1,
5026                 .suite = {
5027                         .hash = __VECS(hmac_sha384_tv_template)
5028                 }
5029         }, {
5030                 .alg = "hmac(sha512)",
5031                 .test = alg_test_hash,
5032                 .fips_allowed = 1,
5033                 .suite = {
5034                         .hash = __VECS(hmac_sha512_tv_template)
5035                 }
5036         }, {
5037                 .alg = "hmac(sm3)",
5038                 .test = alg_test_hash,
5039                 .suite = {
5040                         .hash = __VECS(hmac_sm3_tv_template)
5041                 }
5042         }, {
5043                 .alg = "hmac(streebog256)",
5044                 .test = alg_test_hash,
5045                 .suite = {
5046                         .hash = __VECS(hmac_streebog256_tv_template)
5047                 }
5048         }, {
5049                 .alg = "hmac(streebog512)",
5050                 .test = alg_test_hash,
5051                 .suite = {
5052                         .hash = __VECS(hmac_streebog512_tv_template)
5053                 }
5054         }, {
5055                 .alg = "jitterentropy_rng",
5056                 .fips_allowed = 1,
5057                 .test = alg_test_null,
5058         }, {
5059                 .alg = "kw(aes)",
5060                 .test = alg_test_skcipher,
5061                 .fips_allowed = 1,
5062                 .suite = {
5063                         .cipher = __VECS(aes_kw_tv_template)
5064                 }
5065         }, {
5066                 .alg = "lrw(aes)",
5067                 .generic_driver = "lrw(ecb(aes-generic))",
5068                 .test = alg_test_skcipher,
5069                 .suite = {
5070                         .cipher = __VECS(aes_lrw_tv_template)
5071                 }
5072         }, {
5073                 .alg = "lrw(camellia)",
5074                 .generic_driver = "lrw(ecb(camellia-generic))",
5075                 .test = alg_test_skcipher,
5076                 .suite = {
5077                         .cipher = __VECS(camellia_lrw_tv_template)
5078                 }
5079         }, {
5080                 .alg = "lrw(cast6)",
5081                 .generic_driver = "lrw(ecb(cast6-generic))",
5082                 .test = alg_test_skcipher,
5083                 .suite = {
5084                         .cipher = __VECS(cast6_lrw_tv_template)
5085                 }
5086         }, {
5087                 .alg = "lrw(serpent)",
5088                 .generic_driver = "lrw(ecb(serpent-generic))",
5089                 .test = alg_test_skcipher,
5090                 .suite = {
5091                         .cipher = __VECS(serpent_lrw_tv_template)
5092                 }
5093         }, {
5094                 .alg = "lrw(twofish)",
5095                 .generic_driver = "lrw(ecb(twofish-generic))",
5096                 .test = alg_test_skcipher,
5097                 .suite = {
5098                         .cipher = __VECS(tf_lrw_tv_template)
5099                 }
5100         }, {
5101                 .alg = "lz4",
5102                 .test = alg_test_comp,
5103                 .fips_allowed = 1,
5104                 .suite = {
5105                         .comp = {
5106                                 .comp = __VECS(lz4_comp_tv_template),
5107                                 .decomp = __VECS(lz4_decomp_tv_template)
5108                         }
5109                 }
5110         }, {
5111                 .alg = "lz4hc",
5112                 .test = alg_test_comp,
5113                 .fips_allowed = 1,
5114                 .suite = {
5115                         .comp = {
5116                                 .comp = __VECS(lz4hc_comp_tv_template),
5117                                 .decomp = __VECS(lz4hc_decomp_tv_template)
5118                         }
5119                 }
5120         }, {
5121                 .alg = "lzo",
5122                 .test = alg_test_comp,
5123                 .fips_allowed = 1,
5124                 .suite = {
5125                         .comp = {
5126                                 .comp = __VECS(lzo_comp_tv_template),
5127                                 .decomp = __VECS(lzo_decomp_tv_template)
5128                         }
5129                 }
5130         }, {
5131                 .alg = "lzo-rle",
5132                 .test = alg_test_comp,
5133                 .fips_allowed = 1,
5134                 .suite = {
5135                         .comp = {
5136                                 .comp = __VECS(lzorle_comp_tv_template),
5137                                 .decomp = __VECS(lzorle_decomp_tv_template)
5138                         }
5139                 }
5140         }, {
5141                 .alg = "md4",
5142                 .test = alg_test_hash,
5143                 .suite = {
5144                         .hash = __VECS(md4_tv_template)
5145                 }
5146         }, {
5147                 .alg = "md5",
5148                 .test = alg_test_hash,
5149                 .suite = {
5150                         .hash = __VECS(md5_tv_template)
5151                 }
5152         }, {
5153                 .alg = "michael_mic",
5154                 .test = alg_test_hash,
5155                 .suite = {
5156                         .hash = __VECS(michael_mic_tv_template)
5157                 }
5158         }, {
5159                 .alg = "nhpoly1305",
5160                 .test = alg_test_hash,
5161                 .suite = {
5162                         .hash = __VECS(nhpoly1305_tv_template)
5163                 }
5164         }, {
5165                 .alg = "ofb(aes)",
5166                 .test = alg_test_skcipher,
5167                 .fips_allowed = 1,
5168                 .suite = {
5169                         .cipher = __VECS(aes_ofb_tv_template)
5170                 }
5171         }, {
5172                 /* Same as ofb(aes) except the key is stored in
5173                  * hardware secure memory which we reference by index
5174                  */
5175                 .alg = "ofb(paes)",
5176                 .test = alg_test_null,
5177                 .fips_allowed = 1,
5178         }, {
5179                 .alg = "ofb(sm4)",
5180                 .test = alg_test_skcipher,
5181                 .suite = {
5182                         .cipher = __VECS(sm4_ofb_tv_template)
5183                 }
5184         }, {
5185                 .alg = "pcbc(fcrypt)",
5186                 .test = alg_test_skcipher,
5187                 .suite = {
5188                         .cipher = __VECS(fcrypt_pcbc_tv_template)
5189                 }
5190         }, {
5191                 .alg = "pkcs1pad(rsa,sha224)",
5192                 .test = alg_test_null,
5193                 .fips_allowed = 1,
5194         }, {
5195                 .alg = "pkcs1pad(rsa,sha256)",
5196                 .test = alg_test_akcipher,
5197                 .fips_allowed = 1,
5198                 .suite = {
5199                         .akcipher = __VECS(pkcs1pad_rsa_tv_template)
5200                 }
5201         }, {
5202                 .alg = "pkcs1pad(rsa,sha384)",
5203                 .test = alg_test_null,
5204                 .fips_allowed = 1,
5205         }, {
5206                 .alg = "pkcs1pad(rsa,sha512)",
5207                 .test = alg_test_null,
5208                 .fips_allowed = 1,
5209         }, {
5210                 .alg = "poly1305",
5211                 .test = alg_test_hash,
5212                 .suite = {
5213                         .hash = __VECS(poly1305_tv_template)
5214                 }
5215         }, {
5216                 .alg = "rfc3686(ctr(aes))",
5217                 .test = alg_test_skcipher,
5218                 .fips_allowed = 1,
5219                 .suite = {
5220                         .cipher = __VECS(aes_ctr_rfc3686_tv_template)
5221                 }
5222         }, {
5223                 .alg = "rfc3686(ctr(sm4))",
5224                 .test = alg_test_skcipher,
5225                 .suite = {
5226                         .cipher = __VECS(sm4_ctr_rfc3686_tv_template)
5227                 }
5228         }, {
5229                 .alg = "rfc4106(gcm(aes))",
5230                 .generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))",
5231                 .test = alg_test_aead,
5232                 .fips_allowed = 1,
5233                 .suite = {
5234                         .aead = {
5235                                 ____VECS(aes_gcm_rfc4106_tv_template),
5236                                 .einval_allowed = 1,
5237                                 .aad_iv = 1,
5238                         }
5239                 }
5240         }, {
5241                 .alg = "rfc4309(ccm(aes))",
5242                 .generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))",
5243                 .test = alg_test_aead,
5244                 .fips_allowed = 1,
5245                 .suite = {
5246                         .aead = {
5247                                 ____VECS(aes_ccm_rfc4309_tv_template),
5248                                 .einval_allowed = 1,
5249                                 .aad_iv = 1,
5250                         }
5251                 }
5252         }, {
5253                 .alg = "rfc4543(gcm(aes))",
5254                 .generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))",
5255                 .test = alg_test_aead,
5256                 .suite = {
5257                         .aead = {
5258                                 ____VECS(aes_gcm_rfc4543_tv_template),
5259                                 .einval_allowed = 1,
5260                                 .aad_iv = 1,
5261                         }
5262                 }
5263         }, {
5264                 .alg = "rfc7539(chacha20,poly1305)",
5265                 .test = alg_test_aead,
5266                 .suite = {
5267                         .aead = __VECS(rfc7539_tv_template)
5268                 }
5269         }, {
5270                 .alg = "rfc7539esp(chacha20,poly1305)",
5271                 .test = alg_test_aead,
5272                 .suite = {
5273                         .aead = {
5274                                 ____VECS(rfc7539esp_tv_template),
5275                                 .einval_allowed = 1,
5276                                 .aad_iv = 1,
5277                         }
5278                 }
5279         }, {
5280                 .alg = "rmd128",
5281                 .test = alg_test_hash,
5282                 .suite = {
5283                         .hash = __VECS(rmd128_tv_template)
5284                 }
5285         }, {
5286                 .alg = "rmd160",
5287                 .test = alg_test_hash,
5288                 .suite = {
5289                         .hash = __VECS(rmd160_tv_template)
5290                 }
5291         }, {
5292                 .alg = "rmd256",
5293                 .test = alg_test_hash,
5294                 .suite = {
5295                         .hash = __VECS(rmd256_tv_template)
5296                 }
5297         }, {
5298                 .alg = "rmd320",
5299                 .test = alg_test_hash,
5300                 .suite = {
5301                         .hash = __VECS(rmd320_tv_template)
5302                 }
5303         }, {
5304                 .alg = "rsa",
5305                 .test = alg_test_akcipher,
5306                 .fips_allowed = 1,
5307                 .suite = {
5308                         .akcipher = __VECS(rsa_tv_template)
5309                 }
5310         }, {
5311                 .alg = "salsa20",
5312                 .test = alg_test_skcipher,
5313                 .suite = {
5314                         .cipher = __VECS(salsa20_stream_tv_template)
5315                 }
5316         }, {
5317                 .alg = "sha1",
5318                 .test = alg_test_hash,
5319                 .fips_allowed = 1,
5320                 .suite = {
5321                         .hash = __VECS(sha1_tv_template)
5322                 }
5323         }, {
5324                 .alg = "sha224",
5325                 .test = alg_test_hash,
5326                 .fips_allowed = 1,
5327                 .suite = {
5328                         .hash = __VECS(sha224_tv_template)
5329                 }
5330         }, {
5331                 .alg = "sha256",
5332                 .test = alg_test_hash,
5333                 .fips_allowed = 1,
5334                 .suite = {
5335                         .hash = __VECS(sha256_tv_template)
5336                 }
5337         }, {
5338                 .alg = "sha3-224",
5339                 .test = alg_test_hash,
5340                 .fips_allowed = 1,
5341                 .suite = {
5342                         .hash = __VECS(sha3_224_tv_template)
5343                 }
5344         }, {
5345                 .alg = "sha3-256",
5346                 .test = alg_test_hash,
5347                 .fips_allowed = 1,
5348                 .suite = {
5349                         .hash = __VECS(sha3_256_tv_template)
5350                 }
5351         }, {
5352                 .alg = "sha3-384",
5353                 .test = alg_test_hash,
5354                 .fips_allowed = 1,
5355                 .suite = {
5356                         .hash = __VECS(sha3_384_tv_template)
5357                 }
5358         }, {
5359                 .alg = "sha3-512",
5360                 .test = alg_test_hash,
5361                 .fips_allowed = 1,
5362                 .suite = {
5363                         .hash = __VECS(sha3_512_tv_template)
5364                 }
5365         }, {
5366                 .alg = "sha384",
5367                 .test = alg_test_hash,
5368                 .fips_allowed = 1,
5369                 .suite = {
5370                         .hash = __VECS(sha384_tv_template)
5371                 }
5372         }, {
5373                 .alg = "sha512",
5374                 .test = alg_test_hash,
5375                 .fips_allowed = 1,
5376                 .suite = {
5377                         .hash = __VECS(sha512_tv_template)
5378                 }
5379         }, {
5380                 .alg = "sm3",
5381                 .test = alg_test_hash,
5382                 .suite = {
5383                         .hash = __VECS(sm3_tv_template)
5384                 }
5385         }, {
5386                 .alg = "streebog256",
5387                 .test = alg_test_hash,
5388                 .suite = {
5389                         .hash = __VECS(streebog256_tv_template)
5390                 }
5391         }, {
5392                 .alg = "streebog512",
5393                 .test = alg_test_hash,
5394                 .suite = {
5395                         .hash = __VECS(streebog512_tv_template)
5396                 }
5397         }, {
5398                 .alg = "tgr128",
5399                 .test = alg_test_hash,
5400                 .suite = {
5401                         .hash = __VECS(tgr128_tv_template)
5402                 }
5403         }, {
5404                 .alg = "tgr160",
5405                 .test = alg_test_hash,
5406                 .suite = {
5407                         .hash = __VECS(tgr160_tv_template)
5408                 }
5409         }, {
5410                 .alg = "tgr192",
5411                 .test = alg_test_hash,
5412                 .suite = {
5413                         .hash = __VECS(tgr192_tv_template)
5414                 }
5415         }, {
5416                 .alg = "vmac64(aes)",
5417                 .test = alg_test_hash,
5418                 .suite = {
5419                         .hash = __VECS(vmac64_aes_tv_template)
5420                 }
5421         }, {
5422                 .alg = "wp256",
5423                 .test = alg_test_hash,
5424                 .suite = {
5425                         .hash = __VECS(wp256_tv_template)
5426                 }
5427         }, {
5428                 .alg = "wp384",
5429                 .test = alg_test_hash,
5430                 .suite = {
5431                         .hash = __VECS(wp384_tv_template)
5432                 }
5433         }, {
5434                 .alg = "wp512",
5435                 .test = alg_test_hash,
5436                 .suite = {
5437                         .hash = __VECS(wp512_tv_template)
5438                 }
5439         }, {
5440                 .alg = "xcbc(aes)",
5441                 .test = alg_test_hash,
5442                 .suite = {
5443                         .hash = __VECS(aes_xcbc128_tv_template)
5444                 }
5445         }, {
5446                 .alg = "xchacha12",
5447                 .test = alg_test_skcipher,
5448                 .suite = {
5449                         .cipher = __VECS(xchacha12_tv_template)
5450                 },
5451         }, {
5452                 .alg = "xchacha20",
5453                 .test = alg_test_skcipher,
5454                 .suite = {
5455                         .cipher = __VECS(xchacha20_tv_template)
5456                 },
5457         }, {
5458                 .alg = "xts(aes)",
5459                 .generic_driver = "xts(ecb(aes-generic))",
5460                 .test = alg_test_skcipher,
5461                 .fips_allowed = 1,
5462                 .suite = {
5463                         .cipher = __VECS(aes_xts_tv_template)
5464                 }
5465         }, {
5466                 .alg = "xts(camellia)",
5467                 .generic_driver = "xts(ecb(camellia-generic))",
5468                 .test = alg_test_skcipher,
5469                 .suite = {
5470                         .cipher = __VECS(camellia_xts_tv_template)
5471                 }
5472         }, {
5473                 .alg = "xts(cast6)",
5474                 .generic_driver = "xts(ecb(cast6-generic))",
5475                 .test = alg_test_skcipher,
5476                 .suite = {
5477                         .cipher = __VECS(cast6_xts_tv_template)
5478                 }
5479         }, {
5480                 /* Same as xts(aes) except the key is stored in
5481                  * hardware secure memory which we reference by index
5482                  */
5483                 .alg = "xts(paes)",
5484                 .test = alg_test_null,
5485                 .fips_allowed = 1,
5486         }, {
5487                 .alg = "xts(serpent)",
5488                 .generic_driver = "xts(ecb(serpent-generic))",
5489                 .test = alg_test_skcipher,
5490                 .suite = {
5491                         .cipher = __VECS(serpent_xts_tv_template)
5492                 }
5493         }, {
5494                 .alg = "xts(twofish)",
5495                 .generic_driver = "xts(ecb(twofish-generic))",
5496                 .test = alg_test_skcipher,
5497                 .suite = {
5498                         .cipher = __VECS(tf_xts_tv_template)
5499                 }
5500         }, {
5501 #if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
5502                 .alg = "xts-paes-s390",
5503                 .fips_allowed = 1,
5504                 .test = alg_test_skcipher,
5505                 .suite = {
5506                         .cipher = __VECS(aes_xts_tv_template)
5507                 }
5508         }, {
5509 #endif
5510                 .alg = "xts4096(paes)",
5511                 .test = alg_test_null,
5512                 .fips_allowed = 1,
5513         }, {
5514                 .alg = "xts512(paes)",
5515                 .test = alg_test_null,
5516                 .fips_allowed = 1,
5517         }, {
5518                 .alg = "xxhash64",
5519                 .test = alg_test_hash,
5520                 .fips_allowed = 1,
5521                 .suite = {
5522                         .hash = __VECS(xxhash64_tv_template)
5523                 }
5524         }, {
5525                 .alg = "zlib-deflate",
5526                 .test = alg_test_comp,
5527                 .fips_allowed = 1,
5528                 .suite = {
5529                         .comp = {
5530                                 .comp = __VECS(zlib_deflate_comp_tv_template),
5531                                 .decomp = __VECS(zlib_deflate_decomp_tv_template)
5532                         }
5533                 }
5534         }, {
5535                 .alg = "zstd",
5536                 .test = alg_test_comp,
5537                 .fips_allowed = 1,
5538                 .suite = {
5539                         .comp = {
5540                                 .comp = __VECS(zstd_comp_tv_template),
5541                                 .decomp = __VECS(zstd_decomp_tv_template)
5542                         }
5543                 }
5544         }
5545 };
5546
5547 static void alg_check_test_descs_order(void)
5548 {
5549         int i;
5550
5551         for (i = 1; i < ARRAY_SIZE(alg_test_descs); i++) {
5552                 int diff = strcmp(alg_test_descs[i - 1].alg,
5553                                   alg_test_descs[i].alg);
5554
5555                 if (WARN_ON(diff > 0)) {
5556                         pr_warn("testmgr: alg_test_descs entries in wrong order: '%s' before '%s'\n",
5557                                 alg_test_descs[i - 1].alg,
5558                                 alg_test_descs[i].alg);
5559                 }
5560
5561                 if (WARN_ON(diff == 0)) {
5562                         pr_warn("testmgr: duplicate alg_test_descs entry: '%s'\n",
5563                                 alg_test_descs[i].alg);
5564                 }
5565         }
5566 }
5567
5568 static void alg_check_testvec_configs(void)
5569 {
5570         int i;
5571
5572         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++)
5573                 WARN_ON(!valid_testvec_config(
5574                                 &default_cipher_testvec_configs[i]));
5575
5576         for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++)
5577                 WARN_ON(!valid_testvec_config(
5578                                 &default_hash_testvec_configs[i]));
5579 }
5580
5581 static void testmgr_onetime_init(void)
5582 {
5583         alg_check_test_descs_order();
5584         alg_check_testvec_configs();
5585
5586 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
5587         pr_warn("alg: extra crypto tests enabled.  This is intended for developer use only.\n");
5588 #endif
5589 }
5590
5591 static int alg_find_test(const char *alg)
5592 {
5593         int start = 0;
5594         int end = ARRAY_SIZE(alg_test_descs);
5595
5596         while (start < end) {
5597                 int i = (start + end) / 2;
5598                 int diff = strcmp(alg_test_descs[i].alg, alg);
5599
5600                 if (diff > 0) {
5601                         end = i;
5602                         continue;
5603                 }
5604
5605                 if (diff < 0) {
5606                         start = i + 1;
5607                         continue;
5608                 }
5609
5610                 return i;
5611         }
5612
5613         return -1;
5614 }
5615
5616 int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
5617 {
5618         int i;
5619         int j;
5620         int rc;
5621
5622         if (!fips_enabled && notests) {
5623                 printk_once(KERN_INFO "alg: self-tests disabled\n");
5624                 return 0;
5625         }
5626
5627         DO_ONCE(testmgr_onetime_init);
5628
5629         if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) {
5630                 char nalg[CRYPTO_MAX_ALG_NAME];
5631
5632                 if (snprintf(nalg, sizeof(nalg), "ecb(%s)", alg) >=
5633                     sizeof(nalg))
5634                         return -ENAMETOOLONG;
5635
5636                 i = alg_find_test(nalg);
5637                 if (i < 0)
5638                         goto notest;
5639
5640                 if (fips_enabled && !alg_test_descs[i].fips_allowed)
5641                         goto non_fips_alg;
5642
5643                 rc = alg_test_cipher(alg_test_descs + i, driver, type, mask);
5644                 goto test_done;
5645         }
5646
5647         i = alg_find_test(alg);
5648         j = alg_find_test(driver);
5649         if (i < 0 && j < 0)
5650                 goto notest;
5651
5652         if (fips_enabled && ((i >= 0 && !alg_test_descs[i].fips_allowed) ||
5653                              (j >= 0 && !alg_test_descs[j].fips_allowed)))
5654                 goto non_fips_alg;
5655
5656         rc = 0;
5657         if (i >= 0)
5658                 rc |= alg_test_descs[i].test(alg_test_descs + i, driver,
5659                                              type, mask);
5660         if (j >= 0 && j != i)
5661                 rc |= alg_test_descs[j].test(alg_test_descs + j, driver,
5662                                              type, mask);
5663
5664 test_done:
5665         if (rc && (fips_enabled || panic_on_fail)) {
5666                 fips_fail_notify();
5667                 panic("alg: self-tests for %s (%s) failed in %s mode!\n",
5668                       driver, alg, fips_enabled ? "fips" : "panic_on_fail");
5669         }
5670
5671         if (fips_enabled && !rc)
5672                 pr_info("alg: self-tests for %s (%s) passed\n", driver, alg);
5673
5674         return rc;
5675
5676 notest:
5677         printk(KERN_INFO "alg: No test for %s (%s)\n", alg, driver);
5678         return 0;
5679 non_fips_alg:
5680         return -EINVAL;
5681 }
5682
5683 #endif /* CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */
5684
5685 EXPORT_SYMBOL_GPL(alg_test);