crypto/testmgr.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Algorithm testing framework and tests.
   4  *
   5  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
   6  * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
   7  * Copyright (c) 2007 Nokia Siemens Networks
   8  * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
   9  * Copyright (c) 2019 Google LLC
  10  *
  11  * Updated RFC4106 AES-GCM testing.
  12  *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
  13  *             Adrian Hoban <adrian.hoban@intel.com>
  14  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  15  *             Tadeusz Struk (tadeusz.struk@intel.com)
  16  *    Copyright (c) 2010, Intel Corporation.
  17  */
  18
  19 #include <crypto/aead.h>
  20 #include <crypto/hash.h>
  21 #include <crypto/skcipher.h>
  22 #include <linux/err.h>
  23 #include <linux/fips.h>
  24 #include <linux/module.h>
  25 #include <linux/once.h>
  26 #include <linux/random.h>
  27 #include <linux/scatterlist.h>
  28 #include <linux/slab.h>
  29 #include <linux/string.h>
  30 #include <crypto/rng.h>
  31 #include <crypto/drbg.h>
  32 #include <crypto/akcipher.h>
  33 #include <crypto/kpp.h>
  34 #include <crypto/acompress.h>
  35 #include <crypto/internal/simd.h>
  36
  37 #include "internal.h"
  38
  39 static bool notests;
  40 module_param(notests, bool, 0644);
  41 MODULE_PARM_DESC(notests, "disable crypto self-tests");
  42
  43 static bool panic_on_fail;
  44 module_param(panic_on_fail, bool, 0444);
  45
  46 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
  47 static bool noextratests;
  48 module_param(noextratests, bool, 0644);
  49 MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests");
  50
  51 static unsigned int fuzz_iterations = 100;
  52 module_param(fuzz_iterations, uint, 0644);
  53 MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");
  54
  55 DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test);
  56 EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test);
  57 #endif
  58
  59 #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS
  60
  61 /* a perfect nop */
  62 int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
  63 {
  64         return 0;
  65 }
  66
  67 #else
  68
  69 #include "testmgr.h"
  70
  71 /*
  72  * Need slab memory for testing (size in number of pages).
  73  */
  74 #define XBUFSIZE        8
  75
  76 /*
  77 * Used by test_cipher()
  78 */
  79 #define ENCRYPT 1
  80 #define DECRYPT 0
  81
  82 struct aead_test_suite {
  83         const struct aead_testvec *vecs;
  84         unsigned int count;
  85 };
  86
  87 struct cipher_test_suite {
  88         const struct cipher_testvec *vecs;
  89         unsigned int count;
  90 };
  91
  92 struct comp_test_suite {
  93         struct {
  94                 const struct comp_testvec *vecs;
  95                 unsigned int count;
  96         } comp, decomp;
  97 };
  98
  99 struct hash_test_suite {
 100         const struct hash_testvec *vecs;
 101         unsigned int count;
 102 };
 103
 104 struct cprng_test_suite {
 105         const struct cprng_testvec *vecs;
 106         unsigned int count;
 107 };
 108
 109 struct drbg_test_suite {
 110         const struct drbg_testvec *vecs;
 111         unsigned int count;
 112 };
 113
 114 struct akcipher_test_suite {
 115         const struct akcipher_testvec *vecs;
 116         unsigned int count;
 117 };
 118
 119 struct kpp_test_suite {
 120         const struct kpp_testvec *vecs;
 121         unsigned int count;
 122 };
 123
 124 struct alg_test_desc {
 125         const char *alg;
 126         const char *generic_driver;
 127         int (*test)(const struct alg_test_desc *desc, const char *driver,
 128                     u32 type, u32 mask);
 129         int fips_allowed;       /* set if alg is allowed in fips mode */
 130
 131         union {
 132                 struct aead_test_suite aead;
 133                 struct cipher_test_suite cipher;
 134                 struct comp_test_suite comp;
 135                 struct hash_test_suite hash;
 136                 struct cprng_test_suite cprng;
 137                 struct drbg_test_suite drbg;
 138                 struct akcipher_test_suite akcipher;
 139                 struct kpp_test_suite kpp;
 140         } suite;
 141 };
 142
 143 static void hexdump(unsigned char *buf, unsigned int len)
 144 {
 145         print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
 146                         16, 1,
 147                         buf, len, false);
 148 }
 149
 150 static int __testmgr_alloc_buf(char *buf[XBUFSIZE], int order)
 151 {
 152         int i;
 153
 154         for (i = 0; i < XBUFSIZE; i++) {
 155                 buf[i] = (char *)__get_free_pages(GFP_KERNEL, order);
 156                 if (!buf[i])
 157                         goto err_free_buf;
 158         }
 159
 160         return 0;
 161
 162 err_free_buf:
 163         while (i-- > 0)
 164                 free_pages((unsigned long)buf[i], order);
 165
 166         return -ENOMEM;
 167 }
 168
 169 static int testmgr_alloc_buf(char *buf[XBUFSIZE])
 170 {
 171         return __testmgr_alloc_buf(buf, 0);
 172 }
 173
 174 static void __testmgr_free_buf(char *buf[XBUFSIZE], int order)
 175 {
 176         int i;
 177
 178         for (i = 0; i < XBUFSIZE; i++)
 179                 free_pages((unsigned long)buf[i], order);
 180 }
 181
 182 static void testmgr_free_buf(char *buf[XBUFSIZE])
 183 {
 184         __testmgr_free_buf(buf, 0);
 185 }
 186
 187 #define TESTMGR_POISON_BYTE     0xfe
 188 #define TESTMGR_POISON_LEN      16
 189
 190 static inline void testmgr_poison(void *addr, size_t len)
 191 {
 192         memset(addr, TESTMGR_POISON_BYTE, len);
 193 }
 194
 195 /* Is the memory region still fully poisoned? */
 196 static inline bool testmgr_is_poison(const void *addr, size_t len)
 197 {
 198         return memchr_inv(addr, TESTMGR_POISON_BYTE, len) == NULL;
 199 }
 200
 201 /* flush type for hash algorithms */
 202 enum flush_type {
 203         /* merge with update of previous buffer(s) */
 204         FLUSH_TYPE_NONE = 0,
 205
 206         /* update with previous buffer(s) before doing this one */
 207         FLUSH_TYPE_FLUSH,
 208
 209         /* likewise, but also export and re-import the intermediate state */
 210         FLUSH_TYPE_REIMPORT,
 211 };
 212
 213 /* finalization function for hash algorithms */
 214 enum finalization_type {
 215         FINALIZATION_TYPE_FINAL,        /* use final() */
 216         FINALIZATION_TYPE_FINUP,        /* use finup() */
 217         FINALIZATION_TYPE_DIGEST,       /* use digest() */
 218 };
 219
 220 #define TEST_SG_TOTAL   10000
 221
 222 /**
 223  * struct test_sg_division - description of a scatterlist entry
 224  *
 225  * This struct describes one entry of a scatterlist being constructed to check a
 226  * crypto test vector.
 227  *
 228  * @proportion_of_total: length of this chunk relative to the total length,
 229  *                       given as a proportion out of TEST_SG_TOTAL so that it
 230  *                       scales to fit any test vector
 231  * @offset: byte offset into a 2-page buffer at which this chunk will start
 232  * @offset_relative_to_alignmask: if true, add the algorithm's alignmask to the
 233  *                                @offset
 234  * @flush_type: for hashes, whether an update() should be done now vs.
 235  *              continuing to accumulate data
 236  * @nosimd: if doing the pending update(), do it with SIMD disabled?
 237  */
 238 struct test_sg_division {
 239         unsigned int proportion_of_total;
 240         unsigned int offset;
 241         bool offset_relative_to_alignmask;
 242         enum flush_type flush_type;
 243         bool nosimd;
 244 };
 245
 246 /**
 247  * struct testvec_config - configuration for testing a crypto test vector
 248  *
 249  * This struct describes the data layout and other parameters with which each
 250  * crypto test vector can be tested.
 251  *
 252  * @name: name of this config, logged for debugging purposes if a test fails
 253  * @inplace: operate on the data in-place, if applicable for the algorithm type?
 254  * @req_flags: extra request_flags, e.g. CRYPTO_TFM_REQ_MAY_SLEEP
 255  * @src_divs: description of how to arrange the source scatterlist
 256  * @dst_divs: description of how to arrange the dst scatterlist, if applicable
 257  *            for the algorithm type.  Defaults to @src_divs if unset.
 258  * @iv_offset: misalignment of the IV in the range [0..MAX_ALGAPI_ALIGNMASK+1],
 259  *             where 0 is aligned to a 2*(MAX_ALGAPI_ALIGNMASK+1) byte boundary
 260  * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 261  *                                   the @iv_offset
 262  * @finalization_type: what finalization function to use for hashes
 263  * @nosimd: execute with SIMD disabled?  Requires !CRYPTO_TFM_REQ_MAY_SLEEP.
 264  */
 265 struct testvec_config {
 266         const char *name;
 267         bool inplace;
 268         u32 req_flags;
 269         struct test_sg_division src_divs[XBUFSIZE];
 270         struct test_sg_division dst_divs[XBUFSIZE];
 271         unsigned int iv_offset;
 272         bool iv_offset_relative_to_alignmask;
 273         enum finalization_type finalization_type;
 274         bool nosimd;
 275 };
 276
 277 #define TESTVEC_CONFIG_NAMELEN  192
 278
 279 /*
 280  * The following are the lists of testvec_configs to test for each algorithm
 281  * type when the basic crypto self-tests are enabled, i.e. when
 282  * CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is unset.  They aim to provide good test
 283  * coverage, while keeping the test time much shorter than the full fuzz tests
 284  * so that the basic tests can be enabled in a wider range of circumstances.
 285  */
 286
 287 /* Configs for skciphers and aeads */
 288 static const struct testvec_config default_cipher_testvec_configs[] = {
 289         {
 290                 .name = "in-place",
 291                 .inplace = true,
 292                 .src_divs = { { .proportion_of_total = 10000 } },
 293         }, {
 294                 .name = "out-of-place",
 295                 .src_divs = { { .proportion_of_total = 10000 } },
 296         }, {
 297                 .name = "unaligned buffer, offset=1",
 298                 .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 299                 .iv_offset = 1,
 300         }, {
 301                 .name = "buffer aligned only to alignmask",
 302                 .src_divs = {
 303                         {
 304                                 .proportion_of_total = 10000,
 305                                 .offset = 1,
 306                                 .offset_relative_to_alignmask = true,
 307                         },
 308                 },
 309                 .iv_offset = 1,
 310                 .iv_offset_relative_to_alignmask = true,
 311         }, {
 312                 .name = "two even aligned splits",
 313                 .src_divs = {
 314                         { .proportion_of_total = 5000 },
 315                         { .proportion_of_total = 5000 },
 316                 },
 317         }, {
 318                 .name = "uneven misaligned splits, may sleep",
 319                 .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 320                 .src_divs = {
 321                         { .proportion_of_total = 1900, .offset = 33 },
 322                         { .proportion_of_total = 3300, .offset = 7  },
 323                         { .proportion_of_total = 4800, .offset = 18 },
 324                 },
 325                 .iv_offset = 3,
 326         }, {
 327                 .name = "misaligned splits crossing pages, inplace",
 328                 .inplace = true,
 329                 .src_divs = {
 330                         {
 331                                 .proportion_of_total = 7500,
 332                                 .offset = PAGE_SIZE - 32
 333                         }, {
 334                                 .proportion_of_total = 2500,
 335                                 .offset = PAGE_SIZE - 7
 336                         },
 337                 },
 338         }
 339 };
 340
 341 static const struct testvec_config default_hash_testvec_configs[] = {
 342         {
 343                 .name = "init+update+final aligned buffer",
 344                 .src_divs = { { .proportion_of_total = 10000 } },
 345                 .finalization_type = FINALIZATION_TYPE_FINAL,
 346         }, {
 347                 .name = "init+finup aligned buffer",
 348                 .src_divs = { { .proportion_of_total = 10000 } },
 349                 .finalization_type = FINALIZATION_TYPE_FINUP,
 350         }, {
 351                 .name = "digest aligned buffer",
 352                 .src_divs = { { .proportion_of_total = 10000 } },
 353                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 354         }, {
 355                 .name = "init+update+final misaligned buffer",
 356                 .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 357                 .finalization_type = FINALIZATION_TYPE_FINAL,
 358         }, {
 359                 .name = "digest buffer aligned only to alignmask",
 360                 .src_divs = {
 361                         {
 362                                 .proportion_of_total = 10000,
 363                                 .offset = 1,
 364                                 .offset_relative_to_alignmask = true,
 365                         },
 366                 },
 367                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 368         }, {
 369                 .name = "init+update+update+final two even splits",
 370                 .src_divs = {
 371                         { .proportion_of_total = 5000 },
 372                         {
 373                                 .proportion_of_total = 5000,
 374                                 .flush_type = FLUSH_TYPE_FLUSH,
 375                         },
 376                 },
 377                 .finalization_type = FINALIZATION_TYPE_FINAL,
 378         }, {
 379                 .name = "digest uneven misaligned splits, may sleep",
 380                 .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 381                 .src_divs = {
 382                         { .proportion_of_total = 1900, .offset = 33 },
 383                         { .proportion_of_total = 3300, .offset = 7  },
 384                         { .proportion_of_total = 4800, .offset = 18 },
 385                 },
 386                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 387         }, {
 388                 .name = "digest misaligned splits crossing pages",
 389                 .src_divs = {
 390                         {
 391                                 .proportion_of_total = 7500,
 392                                 .offset = PAGE_SIZE - 32,
 393                         }, {
 394                                 .proportion_of_total = 2500,
 395                                 .offset = PAGE_SIZE - 7,
 396                         },
 397                 },
 398                 .finalization_type = FINALIZATION_TYPE_DIGEST,
 399         }, {
 400                 .name = "import/export",
 401                 .src_divs = {
 402                         {
 403                                 .proportion_of_total = 6500,
 404                                 .flush_type = FLUSH_TYPE_REIMPORT,
 405                         }, {
 406                                 .proportion_of_total = 3500,
 407                                 .flush_type = FLUSH_TYPE_REIMPORT,
 408                         },
 409                 },
 410                 .finalization_type = FINALIZATION_TYPE_FINAL,
 411         }
 412 };
 413
 414 static unsigned int count_test_sg_divisions(const struct test_sg_division *divs)
 415 {
 416         unsigned int remaining = TEST_SG_TOTAL;
 417         unsigned int ndivs = 0;
 418
 419         do {
 420                 remaining -= divs[ndivs++].proportion_of_total;
 421         } while (remaining);
 422
 423         return ndivs;
 424 }
 425
 426 #define SGDIVS_HAVE_FLUSHES     BIT(0)
 427 #define SGDIVS_HAVE_NOSIMD      BIT(1)
 428
 429 static bool valid_sg_divisions(const struct test_sg_division *divs,
 430                                unsigned int count, int *flags_ret)
 431 {
 432         unsigned int total = 0;
 433         unsigned int i;
 434
 435         for (i = 0; i < count && total != TEST_SG_TOTAL; i++) {
 436                 if (divs[i].proportion_of_total <= 0 ||
 437                     divs[i].proportion_of_total > TEST_SG_TOTAL - total)
 438                         return false;
 439                 total += divs[i].proportion_of_total;
 440                 if (divs[i].flush_type != FLUSH_TYPE_NONE)
 441                         *flags_ret |= SGDIVS_HAVE_FLUSHES;
 442                 if (divs[i].nosimd)
 443                         *flags_ret |= SGDIVS_HAVE_NOSIMD;
 444         }
 445         return total == TEST_SG_TOTAL &&
 446                 memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL;
 447 }
 448
 449 /*
 450  * Check whether the given testvec_config is valid.  This isn't strictly needed
 451  * since every testvec_config should be valid, but check anyway so that people
 452  * don't unknowingly add broken configs that don't do what they wanted.
 453  */
 454 static bool valid_testvec_config(const struct testvec_config *cfg)
 455 {
 456         int flags = 0;
 457
 458         if (cfg->name == NULL)
 459                 return false;
 460
 461         if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs),
 462                                 &flags))
 463                 return false;
 464
 465         if (cfg->dst_divs[0].proportion_of_total) {
 466                 if (!valid_sg_divisions(cfg->dst_divs,
 467                                         ARRAY_SIZE(cfg->dst_divs), &flags))
 468                         return false;
 469         } else {
 470                 if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs)))
 471                         return false;
 472                 /* defaults to dst_divs=src_divs */
 473         }
 474
 475         if (cfg->iv_offset +
 476             (cfg->iv_offset_relative_to_alignmask ? MAX_ALGAPI_ALIGNMASK : 0) >
 477             MAX_ALGAPI_ALIGNMASK + 1)
 478                 return false;
 479
 480         if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) &&
 481             cfg->finalization_type == FINALIZATION_TYPE_DIGEST)
 482                 return false;
 483
 484         if ((cfg->nosimd || (flags & SGDIVS_HAVE_NOSIMD)) &&
 485             (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP))
 486                 return false;
 487
 488         return true;
 489 }
 490
 491 struct test_sglist {
 492         char *bufs[XBUFSIZE];
 493         struct scatterlist sgl[XBUFSIZE];
 494         struct scatterlist sgl_saved[XBUFSIZE];
 495         struct scatterlist *sgl_ptr;
 496         unsigned int nents;
 497 };
 498
 499 static int init_test_sglist(struct test_sglist *tsgl)
 500 {
 501         return __testmgr_alloc_buf(tsgl->bufs, 1 /* two pages per buffer */);
 502 }
 503
 504 static void destroy_test_sglist(struct test_sglist *tsgl)
 505 {
 506         return __testmgr_free_buf(tsgl->bufs, 1 /* two pages per buffer */);
 507 }
 508
 509 /**
 510  * build_test_sglist() - build a scatterlist for a crypto test
 511  *
 512  * @tsgl: the scatterlist to build.  @tsgl->bufs[] contains an array of 2-page
 513  *        buffers which the scatterlist @tsgl->sgl[] will be made to point into.
 514  * @divs: the layout specification on which the scatterlist will be based
 515  * @alignmask: the algorithm's alignmask
 516  * @total_len: the total length of the scatterlist to build in bytes
 517  * @data: if non-NULL, the buffers will be filled with this data until it ends.
 518  *        Otherwise the buffers will be poisoned.  In both cases, some bytes
 519  *        past the end of each buffer will be poisoned to help detect overruns.
 520  * @out_divs: if non-NULL, the test_sg_division to which each scatterlist entry
 521  *            corresponds will be returned here.  This will match @divs except
 522  *            that divisions resolving to a length of 0 are omitted as they are
 523  *            not included in the scatterlist.
 524  *
 525  * Return: 0 or a -errno value
 526  */
 527 static int build_test_sglist(struct test_sglist *tsgl,
 528                              const struct test_sg_division *divs,
 529                              const unsigned int alignmask,
 530                              const unsigned int total_len,
 531                              struct iov_iter *data,
 532                              const struct test_sg_division *out_divs[XBUFSIZE])
 533 {
 534         struct {
 535                 const struct test_sg_division *div;
 536                 size_t length;
 537         } partitions[XBUFSIZE];
 538         const unsigned int ndivs = count_test_sg_divisions(divs);
 539         unsigned int len_remaining = total_len;
 540         unsigned int i;
 541
 542         BUILD_BUG_ON(ARRAY_SIZE(partitions) != ARRAY_SIZE(tsgl->sgl));
 543         if (WARN_ON(ndivs > ARRAY_SIZE(partitions)))
 544                 return -EINVAL;
 545
 546         /* Calculate the (div, length) pairs */
 547         tsgl->nents = 0;
 548         for (i = 0; i < ndivs; i++) {
 549                 unsigned int len_this_sg =
 550                         min(len_remaining,
 551                             (total_len * divs[i].proportion_of_total +
 552                              TEST_SG_TOTAL / 2) / TEST_SG_TOTAL);
 553
 554                 if (len_this_sg != 0) {
 555                         partitions[tsgl->nents].div = &divs[i];
 556                         partitions[tsgl->nents].length = len_this_sg;
 557                         tsgl->nents++;
 558                         len_remaining -= len_this_sg;
 559                 }
 560         }
 561         if (tsgl->nents == 0) {
 562                 partitions[tsgl->nents].div = &divs[0];
 563                 partitions[tsgl->nents].length = 0;
 564                 tsgl->nents++;
 565         }
 566         partitions[tsgl->nents - 1].length += len_remaining;
 567
 568         /* Set up the sgl entries and fill the data or poison */
 569         sg_init_table(tsgl->sgl, tsgl->nents);
 570         for (i = 0; i < tsgl->nents; i++) {
 571                 unsigned int offset = partitions[i].div->offset;
 572                 void *addr;
 573
 574                 if (partitions[i].div->offset_relative_to_alignmask)
 575                         offset += alignmask;
 576
 577                 while (offset + partitions[i].length + TESTMGR_POISON_LEN >
 578                        2 * PAGE_SIZE) {
 579                         if (WARN_ON(offset <= 0))
 580                                 return -EINVAL;
 581                         offset /= 2;
 582                 }
 583
 584                 addr = &tsgl->bufs[i][offset];
 585                 sg_set_buf(&tsgl->sgl[i], addr, partitions[i].length);
 586
 587                 if (out_divs)
 588                         out_divs[i] = partitions[i].div;
 589
 590                 if (data) {
 591                         size_t copy_len, copied;
 592
 593                         copy_len = min(partitions[i].length, data->count);
 594                         copied = copy_from_iter(addr, copy_len, data);
 595                         if (WARN_ON(copied != copy_len))
 596                                 return -EINVAL;
 597                         testmgr_poison(addr + copy_len, partitions[i].length +
 598                                        TESTMGR_POISON_LEN - copy_len);
 599                 } else {
 600                         testmgr_poison(addr, partitions[i].length +
 601                                        TESTMGR_POISON_LEN);
 602                 }
 603         }
 604
 605         sg_mark_end(&tsgl->sgl[tsgl->nents - 1]);
 606         tsgl->sgl_ptr = tsgl->sgl;
 607         memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0]));
 608         return 0;
 609 }
 610
 611 /*
 612  * Verify that a scatterlist crypto operation produced the correct output.
 613  *
 614  * @tsgl: scatterlist containing the actual output
 615  * @expected_output: buffer containing the expected output
 616  * @len_to_check: length of @expected_output in bytes
 617  * @unchecked_prefix_len: number of ignored bytes in @tsgl prior to real result
 618  * @check_poison: verify that the poison bytes after each chunk are intact?
 619  *
 620  * Return: 0 if correct, -EINVAL if incorrect, -EOVERFLOW if buffer overrun.
 621  */
 622 static int verify_correct_output(const struct test_sglist *tsgl,
 623                                  const char *expected_output,
 624                                  unsigned int len_to_check,
 625                                  unsigned int unchecked_prefix_len,
 626                                  bool check_poison)
 627 {
 628         unsigned int i;
 629
 630         for (i = 0; i < tsgl->nents; i++) {
 631                 struct scatterlist *sg = &tsgl->sgl_ptr[i];
 632                 unsigned int len = sg->length;
 633                 unsigned int offset = sg->offset;
 634                 const char *actual_output;
 635
 636                 if (unchecked_prefix_len) {
 637                         if (unchecked_prefix_len >= len) {
 638                                 unchecked_prefix_len -= len;
 639                                 continue;
 640                         }
 641                         offset += unchecked_prefix_len;
 642                         len -= unchecked_prefix_len;
 643                         unchecked_prefix_len = 0;
 644                 }
 645                 len = min(len, len_to_check);
 646                 actual_output = page_address(sg_page(sg)) + offset;
 647                 if (memcmp(expected_output, actual_output, len) != 0)
 648                         return -EINVAL;
 649                 if (check_poison &&
 650                     !testmgr_is_poison(actual_output + len, TESTMGR_POISON_LEN))
 651                         return -EOVERFLOW;
 652                 len_to_check -= len;
 653                 expected_output += len;
 654         }
 655         if (WARN_ON(len_to_check != 0))
 656                 return -EINVAL;
 657         return 0;
 658 }
 659
 660 static bool is_test_sglist_corrupted(const struct test_sglist *tsgl)
 661 {
 662         unsigned int i;
 663
 664         for (i = 0; i < tsgl->nents; i++) {
 665                 if (tsgl->sgl[i].page_link != tsgl->sgl_saved[i].page_link)
 666                         return true;
 667                 if (tsgl->sgl[i].offset != tsgl->sgl_saved[i].offset)
 668                         return true;
 669                 if (tsgl->sgl[i].length != tsgl->sgl_saved[i].length)
 670                         return true;
 671         }
 672         return false;
 673 }
 674
 675 struct cipher_test_sglists {
 676         struct test_sglist src;
 677         struct test_sglist dst;
 678 };
 679
 680 static struct cipher_test_sglists *alloc_cipher_test_sglists(void)
 681 {
 682         struct cipher_test_sglists *tsgls;
 683
 684         tsgls = kmalloc(sizeof(*tsgls), GFP_KERNEL);
 685         if (!tsgls)
 686                 return NULL;
 687
 688         if (init_test_sglist(&tsgls->src) != 0)
 689                 goto fail_kfree;
 690         if (init_test_sglist(&tsgls->dst) != 0)
 691                 goto fail_destroy_src;
 692
 693         return tsgls;
 694
 695 fail_destroy_src:
 696         destroy_test_sglist(&tsgls->src);
 697 fail_kfree:
 698         kfree(tsgls);
 699         return NULL;
 700 }
 701
 702 static void free_cipher_test_sglists(struct cipher_test_sglists *tsgls)
 703 {
 704         if (tsgls) {
 705                 destroy_test_sglist(&tsgls->src);
 706                 destroy_test_sglist(&tsgls->dst);
 707                 kfree(tsgls);
 708         }
 709 }
 710
 711 /* Build the src and dst scatterlists for an skcipher or AEAD test */
 712 static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls,
 713                                      const struct testvec_config *cfg,
 714                                      unsigned int alignmask,
 715                                      unsigned int src_total_len,
 716                                      unsigned int dst_total_len,
 717                                      const struct kvec *inputs,
 718                                      unsigned int nr_inputs)
 719 {
 720         struct iov_iter input;
 721         int err;
 722
 723         iov_iter_kvec(&input, WRITE, inputs, nr_inputs, src_total_len);
 724         err = build_test_sglist(&tsgls->src, cfg->src_divs, alignmask,
 725                                 cfg->inplace ?
 726                                         max(dst_total_len, src_total_len) :
 727                                         src_total_len,
 728                                 &input, NULL);
 729         if (err)
 730                 return err;
 731
 732         if (cfg->inplace) {
 733                 tsgls->dst.sgl_ptr = tsgls->src.sgl;
 734                 tsgls->dst.nents = tsgls->src.nents;
 735                 return 0;
 736         }
 737         return build_test_sglist(&tsgls->dst,
 738                                  cfg->dst_divs[0].proportion_of_total ?
 739                                         cfg->dst_divs : cfg->src_divs,
 740                                  alignmask, dst_total_len, NULL, NULL);
 741 }
 742
 743 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
 744
 745 /* Generate a random length in range [0, max_len], but prefer smaller values */
 746 static unsigned int generate_random_length(unsigned int max_len)
 747 {
 748         unsigned int len = prandom_u32() % (max_len + 1);
 749
 750         switch (prandom_u32() % 4) {
 751         case 0:
 752                 return len % 64;
 753         case 1:
 754                 return len % 256;
 755         case 2:
 756                 return len % 1024;
 757         default:
 758                 return len;
 759         }
 760 }
 761
 762 /* Sometimes make some random changes to the given data buffer */
 763 static void mutate_buffer(u8 *buf, size_t count)
 764 {
 765         size_t num_flips;
 766         size_t i;
 767         size_t pos;
 768
 769         /* Sometimes flip some bits */
 770         if (prandom_u32() % 4 == 0) {
 771                 num_flips = min_t(size_t, 1 << (prandom_u32() % 8), count * 8);
 772                 for (i = 0; i < num_flips; i++) {
 773                         pos = prandom_u32() % (count * 8);
 774                         buf[pos / 8] ^= 1 << (pos % 8);
 775                 }
 776         }
 777
 778         /* Sometimes flip some bytes */
 779         if (prandom_u32() % 4 == 0) {
 780                 num_flips = min_t(size_t, 1 << (prandom_u32() % 8), count);
 781                 for (i = 0; i < num_flips; i++)
 782                         buf[prandom_u32() % count] ^= 0xff;
 783         }
 784 }
 785
 786 /* Randomly generate 'count' bytes, but sometimes make them "interesting" */
 787 static void generate_random_bytes(u8 *buf, size_t count)
 788 {
 789         u8 b;
 790         u8 increment;
 791         size_t i;
 792
 793         if (count == 0)
 794                 return;
 795
 796         switch (prandom_u32() % 8) { /* Choose a generation strategy */
 797         case 0:
 798         case 1:
 799                 /* All the same byte, plus optional mutations */
 800                 switch (prandom_u32() % 4) {
 801                 case 0:
 802                         b = 0x00;
 803                         break;
 804                 case 1:
 805                         b = 0xff;
 806                         break;
 807                 default:
 808                         b = (u8)prandom_u32();
 809                         break;
 810                 }
 811                 memset(buf, b, count);
 812                 mutate_buffer(buf, count);
 813                 break;
 814         case 2:
 815                 /* Ascending or descending bytes, plus optional mutations */
 816                 increment = (u8)prandom_u32();
 817                 b = (u8)prandom_u32();
 818                 for (i = 0; i < count; i++, b += increment)
 819                         buf[i] = b;
 820                 mutate_buffer(buf, count);
 821                 break;
 822         default:
 823                 /* Fully random bytes */
 824                 for (i = 0; i < count; i++)
 825                         buf[i] = (u8)prandom_u32();
 826         }
 827 }
 828
 829 static char *generate_random_sgl_divisions(struct test_sg_division *divs,
 830                                            size_t max_divs, char *p, char *end,
 831                                            bool gen_flushes, u32 req_flags)
 832 {
 833         struct test_sg_division *div = divs;
 834         unsigned int remaining = TEST_SG_TOTAL;
 835
 836         do {
 837                 unsigned int this_len;
 838                 const char *flushtype_str;
 839
 840                 if (div == &divs[max_divs - 1] || prandom_u32() % 2 == 0)
 841                         this_len = remaining;
 842                 else
 843                         this_len = 1 + (prandom_u32() % remaining);
 844                 div->proportion_of_total = this_len;
 845
 846                 if (prandom_u32() % 4 == 0)
 847                         div->offset = (PAGE_SIZE - 128) + (prandom_u32() % 128);
 848                 else if (prandom_u32() % 2 == 0)
 849                         div->offset = prandom_u32() % 32;
 850                 else
 851                         div->offset = prandom_u32() % PAGE_SIZE;
 852                 if (prandom_u32() % 8 == 0)
 853                         div->offset_relative_to_alignmask = true;
 854
 855                 div->flush_type = FLUSH_TYPE_NONE;
 856                 if (gen_flushes) {
 857                         switch (prandom_u32() % 4) {
 858                         case 0:
 859                                 div->flush_type = FLUSH_TYPE_REIMPORT;
 860                                 break;
 861                         case 1:
 862                                 div->flush_type = FLUSH_TYPE_FLUSH;
 863                                 break;
 864                         }
 865                 }
 866
 867                 if (div->flush_type != FLUSH_TYPE_NONE &&
 868                     !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
 869                     prandom_u32() % 2 == 0)
 870                         div->nosimd = true;
 871
 872                 switch (div->flush_type) {
 873                 case FLUSH_TYPE_FLUSH:
 874                         if (div->nosimd)
 875                                 flushtype_str = "<flush,nosimd>";
 876                         else
 877                                 flushtype_str = "<flush>";
 878                         break;
 879                 case FLUSH_TYPE_REIMPORT:
 880                         if (div->nosimd)
 881                                 flushtype_str = "<reimport,nosimd>";
 882                         else
 883                                 flushtype_str = "<reimport>";
 884                         break;
 885                 default:
 886                         flushtype_str = "";
 887                         break;
 888                 }
 889
 890                 BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */
 891                 p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str,
 892                                this_len / 100, this_len % 100,
 893                                div->offset_relative_to_alignmask ?
 894                                         "alignmask" : "",
 895                                div->offset, this_len == remaining ? "" : ", ");
 896                 remaining -= this_len;
 897                 div++;
 898         } while (remaining);
 899
 900         return p;
 901 }
 902
 903 /* Generate a random testvec_config for fuzz testing */
 904 static void generate_random_testvec_config(struct testvec_config *cfg,
 905                                            char *name, size_t max_namelen)
 906 {
 907         char *p = name;
 908         char * const end = name + max_namelen;
 909
 910         memset(cfg, 0, sizeof(*cfg));
 911
 912         cfg->name = name;
 913
 914         p += scnprintf(p, end - p, "random:");
 915
 916         if (prandom_u32() % 2 == 0) {
 917                 cfg->inplace = true;
 918                 p += scnprintf(p, end - p, " inplace");
 919         }
 920
 921         if (prandom_u32() % 2 == 0) {
 922                 cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP;
 923                 p += scnprintf(p, end - p, " may_sleep");
 924         }
 925
 926         switch (prandom_u32() % 4) {
 927         case 0:
 928                 cfg->finalization_type = FINALIZATION_TYPE_FINAL;
 929                 p += scnprintf(p, end - p, " use_final");
 930                 break;
 931         case 1:
 932                 cfg->finalization_type = FINALIZATION_TYPE_FINUP;
 933                 p += scnprintf(p, end - p, " use_finup");
 934                 break;
 935         default:
 936                 cfg->finalization_type = FINALIZATION_TYPE_DIGEST;
 937                 p += scnprintf(p, end - p, " use_digest");
 938                 break;
 939         }
 940
 941         if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
 942             prandom_u32() % 2 == 0) {
 943                 cfg->nosimd = true;
 944                 p += scnprintf(p, end - p, " nosimd");
 945         }
 946
 947         p += scnprintf(p, end - p, " src_divs=[");
 948         p = generate_random_sgl_divisions(cfg->src_divs,
 949                                           ARRAY_SIZE(cfg->src_divs), p, end,
 950                                           (cfg->finalization_type !=
 951                                            FINALIZATION_TYPE_DIGEST),
 952                                           cfg->req_flags);
 953         p += scnprintf(p, end - p, "]");
 954
 955         if (!cfg->inplace && prandom_u32() % 2 == 0) {
 956                 p += scnprintf(p, end - p, " dst_divs=[");
 957                 p = generate_random_sgl_divisions(cfg->dst_divs,
 958                                                   ARRAY_SIZE(cfg->dst_divs),
 959                                                   p, end, false,
 960                                                   cfg->req_flags);
 961                 p += scnprintf(p, end - p, "]");
 962         }
 963
 964         if (prandom_u32() % 2 == 0) {
 965                 cfg->iv_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
 966                 p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset);
 967         }
 968
 969         WARN_ON_ONCE(!valid_testvec_config(cfg));
 970 }
 971
 972 static void crypto_disable_simd_for_test(void)
 973 {
 974         preempt_disable();
 975         __this_cpu_write(crypto_simd_disabled_for_test, true);
 976 }
 977
 978 static void crypto_reenable_simd_for_test(void)
 979 {
 980         __this_cpu_write(crypto_simd_disabled_for_test, false);
 981         preempt_enable();
 982 }
 983
 984 /*
 985  * Given an algorithm name, build the name of the generic implementation of that
 986  * algorithm, assuming the usual naming convention.  Specifically, this appends
 987  * "-generic" to every part of the name that is not a template name.  Examples:
 988  *
 989  *      aes => aes-generic
 990  *      cbc(aes) => cbc(aes-generic)
 991  *      cts(cbc(aes)) => cts(cbc(aes-generic))
 992  *      rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic)
 993  *
 994  * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long
 995  */
 996 static int build_generic_driver_name(const char *algname,
 997                                      char driver_name[CRYPTO_MAX_ALG_NAME])
 998 {
 999         const char *in = algname;
1000         char *out = driver_name;
1001         size_t len = strlen(algname);
1002
1003         if (len >= CRYPTO_MAX_ALG_NAME)
1004                 goto too_long;
1005         do {
1006                 const char *in_saved = in;
1007
1008                 while (*in && *in != '(' && *in != ')' && *in != ',')
1009                         *out++ = *in++;
1010                 if (*in != '(' && in > in_saved) {
1011                         len += 8;
1012                         if (len >= CRYPTO_MAX_ALG_NAME)
1013                                 goto too_long;
1014                         memcpy(out, "-generic", 8);
1015                         out += 8;
1016                 }
1017         } while ((*out++ = *in++) != '\0');
1018         return 0;
1019
1020 too_long:
1021         pr_err("alg: generic driver name for \"%s\" would be too long\n",
1022                algname);
1023         return -ENAMETOOLONG;
1024 }
1025 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1026 static void crypto_disable_simd_for_test(void)
1027 {
1028 }
1029
1030 static void crypto_reenable_simd_for_test(void)
1031 {
1032 }
1033 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1034
1035 static int do_ahash_op(int (*op)(struct ahash_request *req),
1036                        struct ahash_request *req,
1037                        struct crypto_wait *wait, bool nosimd)
1038 {
1039         int err;
1040
1041         if (nosimd)
1042                 crypto_disable_simd_for_test();
1043
1044         err = op(req);
1045
1046         if (nosimd)
1047                 crypto_reenable_simd_for_test();
1048
1049         return crypto_wait_req(err, wait);
1050 }
1051
1052 static int check_nonfinal_hash_op(const char *op, int err,
1053                                   u8 *result, unsigned int digestsize,
1054                                   const char *driver, const char *vec_name,
1055                                   const struct testvec_config *cfg)
1056 {
1057         if (err) {
1058                 pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1059                        driver, op, err, vec_name, cfg->name);
1060                 return err;
1061         }
1062         if (!testmgr_is_poison(result, digestsize)) {
1063                 pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
1064                        driver, op, vec_name, cfg->name);
1065                 return -EINVAL;
1066         }
1067         return 0;
1068 }
1069
1070 static int test_hash_vec_cfg(const char *driver,
1071                              const struct hash_testvec *vec,
1072                              const char *vec_name,
1073                              const struct testvec_config *cfg,
1074                              struct ahash_request *req,
1075                              struct test_sglist *tsgl,
1076                              u8 *hashstate)
1077 {
1078         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1079         const unsigned int alignmask = crypto_ahash_alignmask(tfm);
1080         const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1081         const unsigned int statesize = crypto_ahash_statesize(tfm);
1082         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1083         const struct test_sg_division *divs[XBUFSIZE];
1084         DECLARE_CRYPTO_WAIT(wait);
1085         struct kvec _input;
1086         struct iov_iter input;
1087         unsigned int i;
1088         struct scatterlist *pending_sgl;
1089         unsigned int pending_len;
1090         u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1091         int err;
1092
1093         /* Set the key, if specified */
1094         if (vec->ksize) {
1095                 err = crypto_ahash_setkey(tfm, vec->key, vec->ksize);
1096                 if (err) {
1097                         if (err == vec->setkey_error)
1098                                 return 0;
1099                         pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1100                                driver, vec_name, vec->setkey_error, err,
1101                                crypto_ahash_get_flags(tfm));
1102                         return err;
1103                 }
1104                 if (vec->setkey_error) {
1105                         pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1106                                driver, vec_name, vec->setkey_error);
1107                         return -EINVAL;
1108                 }
1109         }
1110
1111         /* Build the scatterlist for the source data */
1112         _input.iov_base = (void *)vec->plaintext;
1113         _input.iov_len = vec->psize;
1114         iov_iter_kvec(&input, WRITE, &_input, 1, vec->psize);
1115         err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
1116                                 &input, divs);
1117         if (err) {
1118                 pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1119                        driver, vec_name, cfg->name);
1120                 return err;
1121         }
1122
1123         /* Do the actual hashing */
1124
1125         testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1126         testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1127
1128         if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1129             vec->digest_error) {
1130                 /* Just using digest() */
1131                 ahash_request_set_callback(req, req_flags, crypto_req_done,
1132                                            &wait);
1133                 ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize);
1134                 err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd);
1135                 if (err) {
1136                         if (err == vec->digest_error)
1137                                 return 0;
1138                         pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1139                                driver, vec_name, vec->digest_error, err,
1140                                cfg->name);
1141                         return err;
1142                 }
1143                 if (vec->digest_error) {
1144                         pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1145                                driver, vec_name, vec->digest_error, cfg->name);
1146                         return -EINVAL;
1147                 }
1148                 goto result_ready;
1149         }
1150
1151         /* Using init(), zero or more update(), then final() or finup() */
1152
1153         ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1154         ahash_request_set_crypt(req, NULL, result, 0);
1155         err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);
1156         err = check_nonfinal_hash_op("init", err, result, digestsize,
1157                                      driver, vec_name, cfg);
1158         if (err)
1159                 return err;
1160
1161         pending_sgl = NULL;
1162         pending_len = 0;
1163         for (i = 0; i < tsgl->nents; i++) {
1164                 if (divs[i]->flush_type != FLUSH_TYPE_NONE &&
1165                     pending_sgl != NULL) {
1166                         /* update() with the pending data */
1167                         ahash_request_set_callback(req, req_flags,
1168                                                    crypto_req_done, &wait);
1169                         ahash_request_set_crypt(req, pending_sgl, result,
1170                                                 pending_len);
1171                         err = do_ahash_op(crypto_ahash_update, req, &wait,
1172                                           divs[i]->nosimd);
1173                         err = check_nonfinal_hash_op("update", err,
1174                                                      result, digestsize,
1175                                                      driver, vec_name, cfg);
1176                         if (err)
1177                                 return err;
1178                         pending_sgl = NULL;
1179                         pending_len = 0;
1180                 }
1181                 if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1182                         /* Test ->export() and ->import() */
1183                         testmgr_poison(hashstate + statesize,
1184                                        TESTMGR_POISON_LEN);
1185                         err = crypto_ahash_export(req, hashstate);
1186                         err = check_nonfinal_hash_op("export", err,
1187                                                      result, digestsize,
1188                                                      driver, vec_name, cfg);
1189                         if (err)
1190                                 return err;
1191                         if (!testmgr_is_poison(hashstate + statesize,
1192                                                TESTMGR_POISON_LEN)) {
1193                                 pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1194                                        driver, vec_name, cfg->name);
1195                                 return -EOVERFLOW;
1196                         }
1197
1198                         testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1199                         err = crypto_ahash_import(req, hashstate);
1200                         err = check_nonfinal_hash_op("import", err,
1201                                                      result, digestsize,
1202                                                      driver, vec_name, cfg);
1203                         if (err)
1204                                 return err;
1205                 }
1206                 if (pending_sgl == NULL)
1207                         pending_sgl = &tsgl->sgl[i];
1208                 pending_len += tsgl->sgl[i].length;
1209         }
1210
1211         ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1212         ahash_request_set_crypt(req, pending_sgl, result, pending_len);
1213         if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
1214                 /* finish with update() and final() */
1215                 err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);
1216                 err = check_nonfinal_hash_op("update", err, result, digestsize,
1217                                              driver, vec_name, cfg);
1218                 if (err)
1219                         return err;
1220                 err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);
1221                 if (err) {
1222                         pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
1223                                driver, err, vec_name, cfg->name);
1224                         return err;
1225                 }
1226         } else {
1227                 /* finish with finup() */
1228                 err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);
1229                 if (err) {
1230                         pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
1231                                driver, err, vec_name, cfg->name);
1232                         return err;
1233                 }
1234         }
1235
1236 result_ready:
1237         /* Check that the algorithm produced the correct digest */
1238         if (memcmp(result, vec->digest, digestsize) != 0) {
1239                 pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
1240                        driver, vec_name, cfg->name);
1241                 return -EINVAL;
1242         }
1243         if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
1244                 pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
1245                        driver, vec_name, cfg->name);
1246                 return -EOVERFLOW;
1247         }
1248
1249         return 0;
1250 }
1251
1252 static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
1253                          unsigned int vec_num, struct ahash_request *req,
1254                          struct test_sglist *tsgl, u8 *hashstate)
1255 {
1256         char vec_name[16];
1257         unsigned int i;
1258         int err;
1259
1260         sprintf(vec_name, "%u", vec_num);
1261
1262         for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
1263                 err = test_hash_vec_cfg(driver, vec, vec_name,
1264                                         &default_hash_testvec_configs[i],
1265                                         req, tsgl, hashstate);
1266                 if (err)
1267                         return err;
1268         }
1269
1270 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1271         if (!noextratests) {
1272                 struct testvec_config cfg;
1273                 char cfgname[TESTVEC_CONFIG_NAMELEN];
1274
1275                 for (i = 0; i < fuzz_iterations; i++) {
1276                         generate_random_testvec_config(&cfg, cfgname,
1277                                                        sizeof(cfgname));
1278                         err = test_hash_vec_cfg(driver, vec, vec_name, &cfg,
1279                                                 req, tsgl, hashstate);
1280                         if (err)
1281                                 return err;
1282                 }
1283         }
1284 #endif
1285         return 0;
1286 }
1287
1288 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1289 /*
1290  * Generate a hash test vector from the given implementation.
1291  * Assumes the buffers in 'vec' were already allocated.
1292  */
1293 static void generate_random_hash_testvec(struct crypto_shash *tfm,
1294                                          struct hash_testvec *vec,
1295                                          unsigned int maxkeysize,
1296                                          unsigned int maxdatasize,
1297                                          char *name, size_t max_namelen)
1298 {
1299         SHASH_DESC_ON_STACK(desc, tfm);
1300
1301         /* Data */
1302         vec->psize = generate_random_length(maxdatasize);
1303         generate_random_bytes((u8 *)vec->plaintext, vec->psize);
1304
1305         /*
1306          * Key: length in range [1, maxkeysize], but usually choose maxkeysize.
1307          * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0.
1308          */
1309         vec->setkey_error = 0;
1310         vec->ksize = 0;
1311         if (maxkeysize) {
1312                 vec->ksize = maxkeysize;
1313                 if (prandom_u32() % 4 == 0)
1314                         vec->ksize = 1 + (prandom_u32() % maxkeysize);
1315                 generate_random_bytes((u8 *)vec->key, vec->ksize);
1316
1317                 vec->setkey_error = crypto_shash_setkey(tfm, vec->key,
1318                                                         vec->ksize);
1319                 /* If the key couldn't be set, no need to continue to digest. */
1320                 if (vec->setkey_error)
1321                         goto done;
1322         }
1323
1324         /* Digest */
1325         desc->tfm = tfm;
1326         vec->digest_error = crypto_shash_digest(desc, vec->plaintext,
1327                                                 vec->psize, (u8 *)vec->digest);
1328 done:
1329         snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"",
1330                  vec->psize, vec->ksize);
1331 }
1332
1333 /*
1334  * Test the hash algorithm represented by @req against the corresponding generic
1335  * implementation, if one is available.
1336  */
1337 static int test_hash_vs_generic_impl(const char *driver,
1338                                      const char *generic_driver,
1339                                      unsigned int maxkeysize,
1340                                      struct ahash_request *req,
1341                                      struct test_sglist *tsgl,
1342                                      u8 *hashstate)
1343 {
1344         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1345         const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1346         const unsigned int blocksize = crypto_ahash_blocksize(tfm);
1347         const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
1348         const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
1349         char _generic_driver[CRYPTO_MAX_ALG_NAME];
1350         struct crypto_shash *generic_tfm = NULL;
1351         unsigned int i;
1352         struct hash_testvec vec = { 0 };
1353         char vec_name[64];
1354         struct testvec_config cfg;
1355         char cfgname[TESTVEC_CONFIG_NAMELEN];
1356         int err;
1357
1358         if (noextratests)
1359                 return 0;
1360
1361         if (!generic_driver) { /* Use default naming convention? */
1362                 err = build_generic_driver_name(algname, _generic_driver);
1363                 if (err)
1364                         return err;
1365                 generic_driver = _generic_driver;
1366         }
1367
1368         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
1369                 return 0;
1370
1371         generic_tfm = crypto_alloc_shash(generic_driver, 0, 0);
1372         if (IS_ERR(generic_tfm)) {
1373                 err = PTR_ERR(generic_tfm);
1374                 if (err == -ENOENT) {
1375                         pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n",
1376                                 driver, generic_driver);
1377                         return 0;
1378                 }
1379                 pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n",
1380                        generic_driver, algname, err);
1381                 return err;
1382         }
1383
1384         /* Check the algorithm properties for consistency. */
1385
1386         if (digestsize != crypto_shash_digestsize(generic_tfm)) {
1387                 pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n",
1388                        driver, digestsize,
1389                        crypto_shash_digestsize(generic_tfm));
1390                 err = -EINVAL;
1391                 goto out;
1392         }
1393
1394         if (blocksize != crypto_shash_blocksize(generic_tfm)) {
1395                 pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n",
1396                        driver, blocksize, crypto_shash_blocksize(generic_tfm));
1397                 err = -EINVAL;
1398                 goto out;
1399         }
1400
1401         /*
1402          * Now generate test vectors using the generic implementation, and test
1403          * the other implementation against them.
1404          */
1405
1406         vec.key = kmalloc(maxkeysize, GFP_KERNEL);
1407         vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL);
1408         vec.digest = kmalloc(digestsize, GFP_KERNEL);
1409         if (!vec.key || !vec.plaintext || !vec.digest) {
1410                 err = -ENOMEM;
1411                 goto out;
1412         }
1413
1414         for (i = 0; i < fuzz_iterations * 8; i++) {
1415                 generate_random_hash_testvec(generic_tfm, &vec,
1416                                              maxkeysize, maxdatasize,
1417                                              vec_name, sizeof(vec_name));
1418                 generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
1419
1420                 err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg,
1421                                         req, tsgl, hashstate);
1422                 if (err)
1423                         goto out;
1424                 cond_resched();
1425         }
1426         err = 0;
1427 out:
1428         kfree(vec.key);
1429         kfree(vec.plaintext);
1430         kfree(vec.digest);
1431         crypto_free_shash(generic_tfm);
1432         return err;
1433 }
1434 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1435 static int test_hash_vs_generic_impl(const char *driver,
1436                                      const char *generic_driver,
1437                                      unsigned int maxkeysize,
1438                                      struct ahash_request *req,
1439                                      struct test_sglist *tsgl,
1440                                      u8 *hashstate)
1441 {
1442         return 0;
1443 }
1444 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1445
1446 static int __alg_test_hash(const struct hash_testvec *vecs,
1447                            unsigned int num_vecs, const char *driver,
1448                            u32 type, u32 mask,
1449                            const char *generic_driver, unsigned int maxkeysize)
1450 {
1451         struct crypto_ahash *tfm;
1452         struct ahash_request *req = NULL;
1453         struct test_sglist *tsgl = NULL;
1454         u8 *hashstate = NULL;
1455         unsigned int i;
1456         int err;
1457
1458         tfm = crypto_alloc_ahash(driver, type, mask);
1459         if (IS_ERR(tfm)) {
1460                 pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
1461                        driver, PTR_ERR(tfm));
1462                 return PTR_ERR(tfm);
1463         }
1464
1465         req = ahash_request_alloc(tfm, GFP_KERNEL);
1466         if (!req) {
1467                 pr_err("alg: hash: failed to allocate request for %s\n",
1468                        driver);
1469                 err = -ENOMEM;
1470                 goto out;
1471         }
1472
1473         tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
1474         if (!tsgl || init_test_sglist(tsgl) != 0) {
1475                 pr_err("alg: hash: failed to allocate test buffers for %s\n",
1476                        driver);
1477                 kfree(tsgl);
1478                 tsgl = NULL;
1479                 err = -ENOMEM;
1480                 goto out;
1481         }
1482
1483         hashstate = kmalloc(crypto_ahash_statesize(tfm) + TESTMGR_POISON_LEN,
1484                             GFP_KERNEL);
1485         if (!hashstate) {
1486                 pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
1487                        driver);
1488                 err = -ENOMEM;
1489                 goto out;
1490         }
1491
1492         for (i = 0; i < num_vecs; i++) {
1493                 err = test_hash_vec(driver, &vecs[i], i, req, tsgl, hashstate);
1494                 if (err)
1495                         goto out;
1496         }
1497         err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req,
1498                                         tsgl, hashstate);
1499 out:
1500         kfree(hashstate);
1501         if (tsgl) {
1502                 destroy_test_sglist(tsgl);
1503                 kfree(tsgl);
1504         }
1505         ahash_request_free(req);
1506         crypto_free_ahash(tfm);
1507         return err;
1508 }
1509
1510 static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
1511                          u32 type, u32 mask)
1512 {
1513         const struct hash_testvec *template = desc->suite.hash.vecs;
1514         unsigned int tcount = desc->suite.hash.count;
1515         unsigned int nr_unkeyed, nr_keyed;
1516         unsigned int maxkeysize = 0;
1517         int err;
1518
1519         /*
1520          * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
1521          * first, before setting a key on the tfm.  To make this easier, we
1522          * require that the unkeyed test vectors (if any) are listed first.
1523          */
1524
1525         for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
1526                 if (template[nr_unkeyed].ksize)
1527                         break;
1528         }
1529         for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
1530                 if (!template[nr_unkeyed + nr_keyed].ksize) {
1531                         pr_err("alg: hash: test vectors for %s out of order, "
1532                                "unkeyed ones must come first\n", desc->alg);
1533                         return -EINVAL;
1534                 }
1535                 maxkeysize = max_t(unsigned int, maxkeysize,
1536                                    template[nr_unkeyed + nr_keyed].ksize);
1537         }
1538
1539         err = 0;
1540         if (nr_unkeyed) {
1541                 err = __alg_test_hash(template, nr_unkeyed, driver, type, mask,
1542                                       desc->generic_driver, maxkeysize);
1543                 template += nr_unkeyed;
1544         }
1545
1546         if (!err && nr_keyed)
1547                 err = __alg_test_hash(template, nr_keyed, driver, type, mask,
1548                                       desc->generic_driver, maxkeysize);
1549
1550         return err;
1551 }
1552
1553 static int test_aead_vec_cfg(const char *driver, int enc,
1554                              const struct aead_testvec *vec,
1555                              const char *vec_name,
1556                              const struct testvec_config *cfg,
1557                              struct aead_request *req,
1558                              struct cipher_test_sglists *tsgls)
1559 {
1560         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1561         const unsigned int alignmask = crypto_aead_alignmask(tfm);
1562         const unsigned int ivsize = crypto_aead_ivsize(tfm);
1563         const unsigned int authsize = vec->clen - vec->plen;
1564         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1565         const char *op = enc ? "encryption" : "decryption";
1566         DECLARE_CRYPTO_WAIT(wait);
1567         u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
1568         u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
1569                  cfg->iv_offset +
1570                  (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
1571         struct kvec input[2];
1572         int expected_error;
1573         int err;
1574
1575         /* Set the key */
1576         if (vec->wk)
1577                 crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
1578         else
1579                 crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
1580         err = crypto_aead_setkey(tfm, vec->key, vec->klen);
1581         if (err && err != vec->setkey_error) {
1582                 pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1583                        driver, vec_name, vec->setkey_error, err,
1584                        crypto_aead_get_flags(tfm));
1585                 return err;
1586         }
1587         if (!err && vec->setkey_error) {
1588                 pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1589                        driver, vec_name, vec->setkey_error);
1590                 return -EINVAL;
1591         }
1592
1593         /* Set the authentication tag size */
1594         err = crypto_aead_setauthsize(tfm, authsize);
1595         if (err && err != vec->setauthsize_error) {
1596                 pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n",
1597                        driver, vec_name, vec->setauthsize_error, err);
1598                 return err;
1599         }
1600         if (!err && vec->setauthsize_error) {
1601                 pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n",
1602                        driver, vec_name, vec->setauthsize_error);
1603                 return -EINVAL;
1604         }
1605
1606         if (vec->setkey_error || vec->setauthsize_error)
1607                 return 0;
1608
1609         /* The IV must be copied to a buffer, as the algorithm may modify it */
1610         if (WARN_ON(ivsize > MAX_IVLEN))
1611                 return -EINVAL;
1612         if (vec->iv)
1613                 memcpy(iv, vec->iv, ivsize);
1614         else
1615                 memset(iv, 0, ivsize);
1616
1617         /* Build the src/dst scatterlists */
1618         input[0].iov_base = (void *)vec->assoc;
1619         input[0].iov_len = vec->alen;
1620         input[1].iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
1621         input[1].iov_len = enc ? vec->plen : vec->clen;
1622         err = build_cipher_test_sglists(tsgls, cfg, alignmask,
1623                                         vec->alen + (enc ? vec->plen :
1624                                                      vec->clen),
1625                                         vec->alen + (enc ? vec->clen :
1626                                                      vec->plen),
1627                                         input, 2);
1628         if (err) {
1629                 pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
1630                        driver, op, vec_name, cfg->name);
1631                 return err;
1632         }
1633
1634         /* Do the actual encryption or decryption */
1635         testmgr_poison(req->__ctx, crypto_aead_reqsize(tfm));
1636         aead_request_set_callback(req, req_flags, crypto_req_done, &wait);
1637         aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
1638                                enc ? vec->plen : vec->clen, iv);
1639         aead_request_set_ad(req, vec->alen);
1640         if (cfg->nosimd)
1641                 crypto_disable_simd_for_test();
1642         err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
1643         if (cfg->nosimd)
1644                 crypto_reenable_simd_for_test();
1645         err = crypto_wait_req(err, &wait);
1646
1647         /* Check that the algorithm didn't overwrite things it shouldn't have */
1648         if (req->cryptlen != (enc ? vec->plen : vec->clen) ||
1649             req->assoclen != vec->alen ||
1650             req->iv != iv ||
1651             req->src != tsgls->src.sgl_ptr ||
1652             req->dst != tsgls->dst.sgl_ptr ||
1653             crypto_aead_reqtfm(req) != tfm ||
1654             req->base.complete != crypto_req_done ||
1655             req->base.flags != req_flags ||
1656             req->base.data != &wait) {
1657                 pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
1658                        driver, op, vec_name, cfg->name);
1659                 if (req->cryptlen != (enc ? vec->plen : vec->clen))
1660                         pr_err("alg: aead: changed 'req->cryptlen'\n");
1661                 if (req->assoclen != vec->alen)
1662                         pr_err("alg: aead: changed 'req->assoclen'\n");
1663                 if (req->iv != iv)
1664                         pr_err("alg: aead: changed 'req->iv'\n");
1665                 if (req->src != tsgls->src.sgl_ptr)
1666                         pr_err("alg: aead: changed 'req->src'\n");
1667                 if (req->dst != tsgls->dst.sgl_ptr)
1668                         pr_err("alg: aead: changed 'req->dst'\n");
1669                 if (crypto_aead_reqtfm(req) != tfm)
1670                         pr_err("alg: aead: changed 'req->base.tfm'\n");
1671                 if (req->base.complete != crypto_req_done)
1672                         pr_err("alg: aead: changed 'req->base.complete'\n");
1673                 if (req->base.flags != req_flags)
1674                         pr_err("alg: aead: changed 'req->base.flags'\n");
1675                 if (req->base.data != &wait)
1676                         pr_err("alg: aead: changed 'req->base.data'\n");
1677                 return -EINVAL;
1678         }
1679         if (is_test_sglist_corrupted(&tsgls->src)) {
1680                 pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
1681                        driver, op, vec_name, cfg->name);
1682                 return -EINVAL;
1683         }
1684         if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
1685             is_test_sglist_corrupted(&tsgls->dst)) {
1686                 pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
1687                        driver, op, vec_name, cfg->name);
1688                 return -EINVAL;
1689         }
1690
1691         /* Check for success or failure */
1692         expected_error = vec->novrfy ? -EBADMSG : vec->crypt_error;
1693         if (err) {
1694                 if (err == expected_error)
1695                         return 0;
1696                 pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1697                        driver, op, vec_name, expected_error, err, cfg->name);
1698                 return err;
1699         }
1700         if (expected_error) {
1701                 pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1702                        driver, op, vec_name, expected_error, cfg->name);
1703                 return -EINVAL;
1704         }
1705
1706         /* Check for the correct output (ciphertext or plaintext) */
1707         err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
1708                                     enc ? vec->clen : vec->plen,
1709                                     vec->alen, enc || !cfg->inplace);
1710         if (err == -EOVERFLOW) {
1711                 pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
1712                        driver, op, vec_name, cfg->name);
1713                 return err;
1714         }
1715         if (err) {
1716                 pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
1717                        driver, op, vec_name, cfg->name);
1718                 return err;
1719         }
1720
1721         return 0;
1722 }
1723
1724 static int test_aead_vec(const char *driver, int enc,
1725                          const struct aead_testvec *vec, unsigned int vec_num,
1726                          struct aead_request *req,
1727                          struct cipher_test_sglists *tsgls)
1728 {
1729         char vec_name[16];
1730         unsigned int i;
1731         int err;
1732
1733         if (enc && vec->novrfy)
1734                 return 0;
1735
1736         sprintf(vec_name, "%u", vec_num);
1737
1738         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
1739                 err = test_aead_vec_cfg(driver, enc, vec, vec_name,
1740                                         &default_cipher_testvec_configs[i],
1741                                         req, tsgls);
1742                 if (err)
1743                         return err;
1744         }
1745
1746 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1747         if (!noextratests) {
1748                 struct testvec_config cfg;
1749                 char cfgname[TESTVEC_CONFIG_NAMELEN];
1750
1751                 for (i = 0; i < fuzz_iterations; i++) {
1752                         generate_random_testvec_config(&cfg, cfgname,
1753                                                        sizeof(cfgname));
1754                         err = test_aead_vec_cfg(driver, enc, vec, vec_name,
1755                                                 &cfg, req, tsgls);
1756                         if (err)
1757                                 return err;
1758                 }
1759         }
1760 #endif
1761         return 0;
1762 }
1763
1764 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1765 /*
1766  * Generate an AEAD test vector from the given implementation.
1767  * Assumes the buffers in 'vec' were already allocated.
1768  */
1769 static void generate_random_aead_testvec(struct aead_request *req,
1770                                          struct aead_testvec *vec,
1771                                          unsigned int maxkeysize,
1772                                          unsigned int maxdatasize,
1773                                          char *name, size_t max_namelen)
1774 {
1775         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1776         const unsigned int ivsize = crypto_aead_ivsize(tfm);
1777         unsigned int maxauthsize = crypto_aead_alg(tfm)->maxauthsize;
1778         unsigned int authsize;
1779         unsigned int total_len;
1780         int i;
1781         struct scatterlist src[2], dst;
1782         u8 iv[MAX_IVLEN];
1783         DECLARE_CRYPTO_WAIT(wait);
1784
1785         /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
1786         vec->klen = maxkeysize;
1787         if (prandom_u32() % 4 == 0)
1788                 vec->klen = prandom_u32() % (maxkeysize + 1);
1789         generate_random_bytes((u8 *)vec->key, vec->klen);
1790         vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen);
1791
1792         /* IV */
1793         generate_random_bytes((u8 *)vec->iv, ivsize);
1794
1795         /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */
1796         authsize = maxauthsize;
1797         if (prandom_u32() % 4 == 0)
1798                 authsize = prandom_u32() % (maxauthsize + 1);
1799         if (WARN_ON(authsize > maxdatasize))
1800                 authsize = maxdatasize;
1801         maxdatasize -= authsize;
1802         vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize);
1803
1804         /* Plaintext and associated data */
1805         total_len = generate_random_length(maxdatasize);
1806         if (prandom_u32() % 4 == 0)
1807                 vec->alen = 0;
1808         else
1809                 vec->alen = generate_random_length(total_len);
1810         vec->plen = total_len - vec->alen;
1811         generate_random_bytes((u8 *)vec->assoc, vec->alen);
1812         generate_random_bytes((u8 *)vec->ptext, vec->plen);
1813
1814         vec->clen = vec->plen + authsize;
1815
1816         /*
1817          * If the key or authentication tag size couldn't be set, no need to
1818          * continue to encrypt.
1819          */
1820         if (vec->setkey_error || vec->setauthsize_error)
1821                 goto done;
1822
1823         /* Ciphertext */
1824         sg_init_table(src, 2);
1825         i = 0;
1826         if (vec->alen)
1827                 sg_set_buf(&src[i++], vec->assoc, vec->alen);
1828         if (vec->plen)
1829                 sg_set_buf(&src[i++], vec->ptext, vec->plen);
1830         sg_init_one(&dst, vec->ctext, vec->alen + vec->clen);
1831         memcpy(iv, vec->iv, ivsize);
1832         aead_request_set_callback(req, 0, crypto_req_done, &wait);
1833         aead_request_set_crypt(req, src, &dst, vec->plen, iv);
1834         aead_request_set_ad(req, vec->alen);
1835         vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req), &wait);
1836         if (vec->crypt_error == 0)
1837                 memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen);
1838 done:
1839         snprintf(name, max_namelen,
1840                  "\"random: alen=%u plen=%u authsize=%u klen=%u\"",
1841                  vec->alen, vec->plen, authsize, vec->klen);
1842 }
1843
1844 /*
1845  * Test the AEAD algorithm represented by @req against the corresponding generic
1846  * implementation, if one is available.
1847  */
1848 static int test_aead_vs_generic_impl(const char *driver,
1849                                      const struct alg_test_desc *test_desc,
1850                                      struct aead_request *req,
1851                                      struct cipher_test_sglists *tsgls)
1852 {
1853         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1854         const unsigned int ivsize = crypto_aead_ivsize(tfm);
1855         const unsigned int maxauthsize = crypto_aead_alg(tfm)->maxauthsize;
1856         const unsigned int blocksize = crypto_aead_blocksize(tfm);
1857         const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
1858         const char *algname = crypto_aead_alg(tfm)->base.cra_name;
1859         const char *generic_driver = test_desc->generic_driver;
1860         char _generic_driver[CRYPTO_MAX_ALG_NAME];
1861         struct crypto_aead *generic_tfm = NULL;
1862         struct aead_request *generic_req = NULL;
1863         unsigned int maxkeysize;
1864         unsigned int i;
1865         struct aead_testvec vec = { 0 };
1866         char vec_name[64];
1867         struct testvec_config cfg;
1868         char cfgname[TESTVEC_CONFIG_NAMELEN];
1869         int err;
1870
1871         if (noextratests)
1872                 return 0;
1873
1874         if (!generic_driver) { /* Use default naming convention? */
1875                 err = build_generic_driver_name(algname, _generic_driver);
1876                 if (err)
1877                         return err;
1878                 generic_driver = _generic_driver;
1879         }
1880
1881         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
1882                 return 0;
1883
1884         generic_tfm = crypto_alloc_aead(generic_driver, 0, 0);
1885         if (IS_ERR(generic_tfm)) {
1886                 err = PTR_ERR(generic_tfm);
1887                 if (err == -ENOENT) {
1888                         pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n",
1889                                 driver, generic_driver);
1890                         return 0;
1891                 }
1892                 pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n",
1893                        generic_driver, algname, err);
1894                 return err;
1895         }
1896
1897         generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);
1898         if (!generic_req) {
1899                 err = -ENOMEM;
1900                 goto out;
1901         }
1902
1903         /* Check the algorithm properties for consistency. */
1904
1905         if (maxauthsize != crypto_aead_alg(generic_tfm)->maxauthsize) {
1906                 pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n",
1907                        driver, maxauthsize,
1908                        crypto_aead_alg(generic_tfm)->maxauthsize);
1909                 err = -EINVAL;
1910                 goto out;
1911         }
1912
1913         if (ivsize != crypto_aead_ivsize(generic_tfm)) {
1914                 pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n",
1915                        driver, ivsize, crypto_aead_ivsize(generic_tfm));
1916                 err = -EINVAL;
1917                 goto out;
1918         }
1919
1920         if (blocksize != crypto_aead_blocksize(generic_tfm)) {
1921                 pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n",
1922                        driver, blocksize, crypto_aead_blocksize(generic_tfm));
1923                 err = -EINVAL;
1924                 goto out;
1925         }
1926
1927         /*
1928          * Now generate test vectors using the generic implementation, and test
1929          * the other implementation against them.
1930          */
1931
1932         maxkeysize = 0;
1933         for (i = 0; i < test_desc->suite.aead.count; i++)
1934                 maxkeysize = max_t(unsigned int, maxkeysize,
1935                                    test_desc->suite.aead.vecs[i].klen);
1936
1937         vec.key = kmalloc(maxkeysize, GFP_KERNEL);
1938         vec.iv = kmalloc(ivsize, GFP_KERNEL);
1939         vec.assoc = kmalloc(maxdatasize, GFP_KERNEL);
1940         vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);
1941         vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);
1942         if (!vec.key || !vec.iv || !vec.assoc || !vec.ptext || !vec.ctext) {
1943                 err = -ENOMEM;
1944                 goto out;
1945         }
1946
1947         for (i = 0; i < fuzz_iterations * 8; i++) {
1948                 generate_random_aead_testvec(generic_req, &vec,
1949                                              maxkeysize, maxdatasize,
1950                                              vec_name, sizeof(vec_name));
1951                 generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
1952
1953                 err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg,
1954                                         req, tsgls);
1955                 if (err)
1956                         goto out;
1957                 err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg,
1958                                         req, tsgls);
1959                 if (err)
1960                         goto out;
1961                 cond_resched();
1962         }
1963         err = 0;
1964 out:
1965         kfree(vec.key);
1966         kfree(vec.iv);
1967         kfree(vec.assoc);
1968         kfree(vec.ptext);
1969         kfree(vec.ctext);
1970         crypto_free_aead(generic_tfm);
1971         aead_request_free(generic_req);
1972         return err;
1973 }
1974 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1975 static int test_aead_vs_generic_impl(const char *driver,
1976                                      const struct alg_test_desc *test_desc,
1977                                      struct aead_request *req,
1978                                      struct cipher_test_sglists *tsgls)
1979 {
1980         return 0;
1981 }
1982 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1983
1984 static int test_aead(const char *driver, int enc,
1985                      const struct aead_test_suite *suite,
1986                      struct aead_request *req,
1987                      struct cipher_test_sglists *tsgls)
1988 {
1989         unsigned int i;
1990         int err;
1991
1992         for (i = 0; i < suite->count; i++) {
1993                 err = test_aead_vec(driver, enc, &suite->vecs[i], i, req,
1994                                     tsgls);
1995                 if (err)
1996                         return err;
1997         }
1998         return 0;
1999 }
2000
2001 static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,
2002                          u32 type, u32 mask)
2003 {
2004         const struct aead_test_suite *suite = &desc->suite.aead;
2005         struct crypto_aead *tfm;
2006         struct aead_request *req = NULL;
2007         struct cipher_test_sglists *tsgls = NULL;
2008         int err;
2009
2010         if (suite->count <= 0) {
2011                 pr_err("alg: aead: empty test suite for %s\n", driver);
2012                 return -EINVAL;
2013         }
2014
2015         tfm = crypto_alloc_aead(driver, type, mask);
2016         if (IS_ERR(tfm)) {
2017                 pr_err("alg: aead: failed to allocate transform for %s: %ld\n",
2018                        driver, PTR_ERR(tfm));
2019                 return PTR_ERR(tfm);
2020         }
2021
2022         req = aead_request_alloc(tfm, GFP_KERNEL);
2023         if (!req) {
2024                 pr_err("alg: aead: failed to allocate request for %s\n",
2025                        driver);
2026                 err = -ENOMEM;
2027                 goto out;
2028         }
2029
2030         tsgls = alloc_cipher_test_sglists();
2031         if (!tsgls) {
2032                 pr_err("alg: aead: failed to allocate test buffers for %s\n",
2033                        driver);
2034                 err = -ENOMEM;
2035                 goto out;
2036         }
2037
2038         err = test_aead(driver, ENCRYPT, suite, req, tsgls);
2039         if (err)
2040                 goto out;
2041
2042         err = test_aead(driver, DECRYPT, suite, req, tsgls);
2043         if (err)
2044                 goto out;
2045
2046         err = test_aead_vs_generic_impl(driver, desc, req, tsgls);
2047 out:
2048         free_cipher_test_sglists(tsgls);
2049         aead_request_free(req);
2050         crypto_free_aead(tfm);
2051         return err;
2052 }
2053
2054 static int test_cipher(struct crypto_cipher *tfm, int enc,
2055                        const struct cipher_testvec *template,
2056                        unsigned int tcount)
2057 {
2058         const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm));
2059         unsigned int i, j, k;
2060         char *q;
2061         const char *e;
2062         const char *input, *result;
2063         void *data;
2064         char *xbuf[XBUFSIZE];
2065         int ret = -ENOMEM;
2066
2067         if (testmgr_alloc_buf(xbuf))
2068                 goto out_nobuf;
2069
2070         if (enc == ENCRYPT)
2071                 e = "encryption";
2072         else
2073                 e = "decryption";
2074
2075         j = 0;
2076         for (i = 0; i < tcount; i++) {
2077
2078                 if (fips_enabled && template[i].fips_skip)
2079                         continue;
2080
2081                 input  = enc ? template[i].ptext : template[i].ctext;
2082                 result = enc ? template[i].ctext : template[i].ptext;
2083                 j++;
2084
2085                 ret = -EINVAL;
2086                 if (WARN_ON(template[i].len > PAGE_SIZE))
2087                         goto out;
2088
2089                 data = xbuf[0];
2090                 memcpy(data, input, template[i].len);
2091
2092                 crypto_cipher_clear_flags(tfm, ~0);
2093                 if (template[i].wk)
2094                         crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2095
2096                 ret = crypto_cipher_setkey(tfm, template[i].key,
2097                                            template[i].klen);
2098                 if (ret) {
2099                         if (ret == template[i].setkey_error)
2100                                 continue;
2101                         pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n",
2102                                algo, j, template[i].setkey_error, ret,
2103                                crypto_cipher_get_flags(tfm));
2104                         goto out;
2105                 }
2106                 if (template[i].setkey_error) {
2107                         pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n",
2108                                algo, j, template[i].setkey_error);
2109                         ret = -EINVAL;
2110                         goto out;
2111                 }
2112
2113                 for (k = 0; k < template[i].len;
2114                      k += crypto_cipher_blocksize(tfm)) {
2115                         if (enc)
2116                                 crypto_cipher_encrypt_one(tfm, data + k,
2117                                                           data + k);
2118                         else
2119                                 crypto_cipher_decrypt_one(tfm, data + k,
2120                                                           data + k);
2121                 }
2122
2123                 q = data;
2124                 if (memcmp(q, result, template[i].len)) {
2125                         printk(KERN_ERR "alg: cipher: Test %d failed "
2126                                "on %s for %s\n", j, e, algo);
2127                         hexdump(q, template[i].len);
2128                         ret = -EINVAL;
2129                         goto out;
2130                 }
2131         }
2132
2133         ret = 0;
2134
2135 out:
2136         testmgr_free_buf(xbuf);
2137 out_nobuf:
2138         return ret;
2139 }
2140
2141 static int test_skcipher_vec_cfg(const char *driver, int enc,
2142                                  const struct cipher_testvec *vec,
2143                                  const char *vec_name,
2144                                  const struct testvec_config *cfg,
2145                                  struct skcipher_request *req,
2146                                  struct cipher_test_sglists *tsgls)
2147 {
2148         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2149         const unsigned int alignmask = crypto_skcipher_alignmask(tfm);
2150         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2151         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2152         const char *op = enc ? "encryption" : "decryption";
2153         DECLARE_CRYPTO_WAIT(wait);
2154         u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2155         u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2156                  cfg->iv_offset +
2157                  (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2158         struct kvec input;
2159         int err;
2160
2161         /* Set the key */
2162         if (vec->wk)
2163                 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2164         else
2165                 crypto_skcipher_clear_flags(tfm,
2166                                             CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2167         err = crypto_skcipher_setkey(tfm, vec->key, vec->klen);
2168         if (err) {
2169                 if (err == vec->setkey_error)
2170                         return 0;
2171                 pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2172                        driver, vec_name, vec->setkey_error, err,
2173                        crypto_skcipher_get_flags(tfm));
2174                 return err;
2175         }
2176         if (vec->setkey_error) {
2177                 pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2178                        driver, vec_name, vec->setkey_error);
2179                 return -EINVAL;
2180         }
2181
2182         /* The IV must be copied to a buffer, as the algorithm may modify it */
2183         if (ivsize) {
2184                 if (WARN_ON(ivsize > MAX_IVLEN))
2185                         return -EINVAL;
2186                 if (vec->generates_iv && !enc)
2187                         memcpy(iv, vec->iv_out, ivsize);
2188                 else if (vec->iv)
2189                         memcpy(iv, vec->iv, ivsize);
2190                 else
2191                         memset(iv, 0, ivsize);
2192         } else {
2193                 if (vec->generates_iv) {
2194                         pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n",
2195                                driver, vec_name);
2196                         return -EINVAL;
2197                 }
2198                 iv = NULL;
2199         }
2200
2201         /* Build the src/dst scatterlists */
2202         input.iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2203         input.iov_len = vec->len;
2204         err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2205                                         vec->len, vec->len, &input, 1);
2206         if (err) {
2207                 pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2208                        driver, op, vec_name, cfg->name);
2209                 return err;
2210         }
2211
2212         /* Do the actual encryption or decryption */
2213         testmgr_poison(req->__ctx, crypto_skcipher_reqsize(tfm));
2214         skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait);
2215         skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2216                                    vec->len, iv);
2217         if (cfg->nosimd)
2218                 crypto_disable_simd_for_test();
2219         err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
2220         if (cfg->nosimd)
2221                 crypto_reenable_simd_for_test();
2222         err = crypto_wait_req(err, &wait);
2223
2224         /* Check that the algorithm didn't overwrite things it shouldn't have */
2225         if (req->cryptlen != vec->len ||
2226             req->iv != iv ||
2227             req->src != tsgls->src.sgl_ptr ||
2228             req->dst != tsgls->dst.sgl_ptr ||
2229             crypto_skcipher_reqtfm(req) != tfm ||
2230             req->base.complete != crypto_req_done ||
2231             req->base.flags != req_flags ||
2232             req->base.data != &wait) {
2233                 pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2234                        driver, op, vec_name, cfg->name);
2235                 if (req->cryptlen != vec->len)
2236                         pr_err("alg: skcipher: changed 'req->cryptlen'\n");
2237                 if (req->iv != iv)
2238                         pr_err("alg: skcipher: changed 'req->iv'\n");
2239                 if (req->src != tsgls->src.sgl_ptr)
2240                         pr_err("alg: skcipher: changed 'req->src'\n");
2241                 if (req->dst != tsgls->dst.sgl_ptr)
2242                         pr_err("alg: skcipher: changed 'req->dst'\n");
2243                 if (crypto_skcipher_reqtfm(req) != tfm)
2244                         pr_err("alg: skcipher: changed 'req->base.tfm'\n");
2245                 if (req->base.complete != crypto_req_done)
2246                         pr_err("alg: skcipher: changed 'req->base.complete'\n");
2247                 if (req->base.flags != req_flags)
2248                         pr_err("alg: skcipher: changed 'req->base.flags'\n");
2249                 if (req->base.data != &wait)
2250                         pr_err("alg: skcipher: changed 'req->base.data'\n");
2251                 return -EINVAL;
2252         }
2253         if (is_test_sglist_corrupted(&tsgls->src)) {
2254                 pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2255                        driver, op, vec_name, cfg->name);
2256                 return -EINVAL;
2257         }
2258         if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2259             is_test_sglist_corrupted(&tsgls->dst)) {
2260                 pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2261                        driver, op, vec_name, cfg->name);
2262                 return -EINVAL;
2263         }
2264
2265         /* Check for success or failure */
2266         if (err) {
2267                 if (err == vec->crypt_error)
2268                         return 0;
2269                 pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
2270                        driver, op, vec_name, vec->crypt_error, err, cfg->name);
2271                 return err;
2272         }
2273         if (vec->crypt_error) {
2274                 pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
2275                        driver, op, vec_name, vec->crypt_error, cfg->name);
2276                 return -EINVAL;
2277         }
2278
2279         /* Check for the correct output (ciphertext or plaintext) */
2280         err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2281                                     vec->len, 0, true);
2282         if (err == -EOVERFLOW) {
2283                 pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2284                        driver, op, vec_name, cfg->name);
2285                 return err;
2286         }
2287         if (err) {
2288                 pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2289                        driver, op, vec_name, cfg->name);
2290                 return err;
2291         }
2292
2293         /* If applicable, check that the algorithm generated the correct IV */
2294         if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) {
2295                 pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n",
2296                        driver, op, vec_name, cfg->name);
2297                 hexdump(iv, ivsize);
2298                 return -EINVAL;
2299         }
2300
2301         return 0;
2302 }
2303
2304 static int test_skcipher_vec(const char *driver, int enc,
2305                              const struct cipher_testvec *vec,
2306                              unsigned int vec_num,
2307                              struct skcipher_request *req,
2308                              struct cipher_test_sglists *tsgls)
2309 {
2310         char vec_name[16];
2311         unsigned int i;
2312         int err;
2313
2314         if (fips_enabled && vec->fips_skip)
2315                 return 0;
2316
2317         sprintf(vec_name, "%u", vec_num);
2318
2319         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2320                 err = test_skcipher_vec_cfg(driver, enc, vec, vec_name,
2321                                             &default_cipher_testvec_configs[i],
2322                                             req, tsgls);
2323                 if (err)
2324                         return err;
2325         }
2326
2327 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2328         if (!noextratests) {
2329                 struct testvec_config cfg;
2330                 char cfgname[TESTVEC_CONFIG_NAMELEN];
2331
2332                 for (i = 0; i < fuzz_iterations; i++) {
2333                         generate_random_testvec_config(&cfg, cfgname,
2334                                                        sizeof(cfgname));
2335                         err = test_skcipher_vec_cfg(driver, enc, vec, vec_name,
2336                                                     &cfg, req, tsgls);
2337                         if (err)
2338                                 return err;
2339                 }
2340         }
2341 #endif
2342         return 0;
2343 }
2344
2345 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2346 /*
2347  * Generate a symmetric cipher test vector from the given implementation.
2348  * Assumes the buffers in 'vec' were already allocated.
2349  */
2350 static void generate_random_cipher_testvec(struct skcipher_request *req,
2351                                            struct cipher_testvec *vec,
2352                                            unsigned int maxdatasize,
2353                                            char *name, size_t max_namelen)
2354 {
2355         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2356         const unsigned int maxkeysize = tfm->keysize;
2357         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2358         struct scatterlist src, dst;
2359         u8 iv[MAX_IVLEN];
2360         DECLARE_CRYPTO_WAIT(wait);
2361
2362         /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
2363         vec->klen = maxkeysize;
2364         if (prandom_u32() % 4 == 0)
2365                 vec->klen = prandom_u32() % (maxkeysize + 1);
2366         generate_random_bytes((u8 *)vec->key, vec->klen);
2367         vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen);
2368
2369         /* IV */
2370         generate_random_bytes((u8 *)vec->iv, ivsize);
2371
2372         /* Plaintext */
2373         vec->len = generate_random_length(maxdatasize);
2374         generate_random_bytes((u8 *)vec->ptext, vec->len);
2375
2376         /* If the key couldn't be set, no need to continue to encrypt. */
2377         if (vec->setkey_error)
2378                 goto done;
2379
2380         /* Ciphertext */
2381         sg_init_one(&src, vec->ptext, vec->len);
2382         sg_init_one(&dst, vec->ctext, vec->len);
2383         memcpy(iv, vec->iv, ivsize);
2384         skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
2385         skcipher_request_set_crypt(req, &src, &dst, vec->len, iv);
2386         vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
2387 done:
2388         snprintf(name, max_namelen, "\"random: len=%u klen=%u\"",
2389                  vec->len, vec->klen);
2390 }
2391
2392 /*
2393  * Test the skcipher algorithm represented by @req against the corresponding
2394  * generic implementation, if one is available.
2395  */
2396 static int test_skcipher_vs_generic_impl(const char *driver,
2397                                          const char *generic_driver,
2398                                          struct skcipher_request *req,
2399                                          struct cipher_test_sglists *tsgls)
2400 {
2401         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2402         const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2403         const unsigned int blocksize = crypto_skcipher_blocksize(tfm);
2404         const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
2405         const char *algname = crypto_skcipher_alg(tfm)->base.cra_name;
2406         char _generic_driver[CRYPTO_MAX_ALG_NAME];
2407         struct crypto_skcipher *generic_tfm = NULL;
2408         struct skcipher_request *generic_req = NULL;
2409         unsigned int i;
2410         struct cipher_testvec vec = { 0 };
2411         char vec_name[64];
2412         struct testvec_config cfg;
2413         char cfgname[TESTVEC_CONFIG_NAMELEN];
2414         int err;
2415
2416         if (noextratests)
2417                 return 0;
2418
2419         /* Keywrap isn't supported here yet as it handles its IV differently. */
2420         if (strncmp(algname, "kw(", 3) == 0)
2421                 return 0;
2422
2423         if (!generic_driver) { /* Use default naming convention? */
2424                 err = build_generic_driver_name(algname, _generic_driver);
2425                 if (err)
2426                         return err;
2427                 generic_driver = _generic_driver;
2428         }
2429
2430         if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
2431                 return 0;
2432
2433         generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0);
2434         if (IS_ERR(generic_tfm)) {
2435                 err = PTR_ERR(generic_tfm);
2436                 if (err == -ENOENT) {
2437                         pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n",
2438                                 driver, generic_driver);
2439                         return 0;
2440                 }
2441                 pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n",
2442                        generic_driver, algname, err);
2443                 return err;
2444         }
2445
2446         generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
2447         if (!generic_req) {
2448                 err = -ENOMEM;
2449                 goto out;
2450         }
2451
2452         /* Check the algorithm properties for consistency. */
2453
2454         if (tfm->keysize != generic_tfm->keysize) {
2455                 pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n",
2456                        driver, tfm->keysize, generic_tfm->keysize);
2457                 err = -EINVAL;
2458                 goto out;
2459         }
2460
2461         if (ivsize != crypto_skcipher_ivsize(generic_tfm)) {
2462                 pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n",
2463                        driver, ivsize, crypto_skcipher_ivsize(generic_tfm));
2464                 err = -EINVAL;
2465                 goto out;
2466         }
2467
2468         if (blocksize != crypto_skcipher_blocksize(generic_tfm)) {
2469                 pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n",
2470                        driver, blocksize,
2471                        crypto_skcipher_blocksize(generic_tfm));
2472                 err = -EINVAL;
2473                 goto out;
2474         }
2475
2476         /*
2477          * Now generate test vectors using the generic implementation, and test
2478          * the other implementation against them.
2479          */
2480
2481         vec.key = kmalloc(tfm->keysize, GFP_KERNEL);
2482         vec.iv = kmalloc(ivsize, GFP_KERNEL);
2483         vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);
2484         vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);
2485         if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) {
2486                 err = -ENOMEM;
2487                 goto out;
2488         }
2489
2490         for (i = 0; i < fuzz_iterations * 8; i++) {
2491                 generate_random_cipher_testvec(generic_req, &vec, maxdatasize,
2492                                                vec_name, sizeof(vec_name));
2493                 generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
2494
2495                 err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name,
2496                                             &cfg, req, tsgls);
2497                 if (err)
2498                         goto out;
2499                 err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name,
2500                                             &cfg, req, tsgls);
2501                 if (err)
2502                         goto out;
2503                 cond_resched();
2504         }
2505         err = 0;
2506 out:
2507         kfree(vec.key);
2508         kfree(vec.iv);
2509         kfree(vec.ptext);
2510         kfree(vec.ctext);
2511         crypto_free_skcipher(generic_tfm);
2512         skcipher_request_free(generic_req);
2513         return err;
2514 }
2515 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
2516 static int test_skcipher_vs_generic_impl(const char *driver,
2517                                          const char *generic_driver,
2518                                          struct skcipher_request *req,
2519                                          struct cipher_test_sglists *tsgls)
2520 {
2521         return 0;
2522 }
2523 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
2524
2525 static int test_skcipher(const char *driver, int enc,
2526                          const struct cipher_test_suite *suite,
2527                          struct skcipher_request *req,
2528                          struct cipher_test_sglists *tsgls)
2529 {
2530         unsigned int i;
2531         int err;
2532
2533         for (i = 0; i < suite->count; i++) {
2534                 err = test_skcipher_vec(driver, enc, &suite->vecs[i], i, req,
2535                                         tsgls);
2536                 if (err)
2537                         return err;
2538         }
2539         return 0;
2540 }
2541
2542 static int alg_test_skcipher(const struct alg_test_desc *desc,
2543                              const char *driver, u32 type, u32 mask)
2544 {
2545         const struct cipher_test_suite *suite = &desc->suite.cipher;
2546         struct crypto_skcipher *tfm;
2547         struct skcipher_request *req = NULL;
2548         struct cipher_test_sglists *tsgls = NULL;
2549         int err;
2550
2551         if (suite->count <= 0) {
2552                 pr_err("alg: skcipher: empty test suite for %s\n", driver);
2553                 return -EINVAL;
2554         }
2555
2556         tfm = crypto_alloc_skcipher(driver, type, mask);
2557         if (IS_ERR(tfm)) {
2558                 pr_err("alg: skcipher: failed to allocate transform for %s: %ld\n",
2559                        driver, PTR_ERR(tfm));
2560                 return PTR_ERR(tfm);
2561         }
2562
2563         req = skcipher_request_alloc(tfm, GFP_KERNEL);
2564         if (!req) {
2565                 pr_err("alg: skcipher: failed to allocate request for %s\n",
2566                        driver);
2567                 err = -ENOMEM;
2568                 goto out;
2569         }
2570
2571         tsgls = alloc_cipher_test_sglists();
2572         if (!tsgls) {
2573                 pr_err("alg: skcipher: failed to allocate test buffers for %s\n",
2574                        driver);
2575                 err = -ENOMEM;
2576                 goto out;
2577         }
2578
2579         err = test_skcipher(driver, ENCRYPT, suite, req, tsgls);
2580         if (err)
2581                 goto out;
2582
2583         err = test_skcipher(driver, DECRYPT, suite, req, tsgls);
2584         if (err)
2585                 goto out;
2586
2587         err = test_skcipher_vs_generic_impl(driver, desc->generic_driver, req,
2588                                             tsgls);
2589 out:
2590         free_cipher_test_sglists(tsgls);
2591         skcipher_request_free(req);
2592         crypto_free_skcipher(tfm);
2593         return err;
2594 }
2595
2596 static int test_comp(struct crypto_comp *tfm,
2597                      const struct comp_testvec *ctemplate,
2598                      const struct comp_testvec *dtemplate,
2599                      int ctcount, int dtcount)
2600 {
2601         const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm));
2602         char *output, *decomp_output;
2603         unsigned int i;
2604         int ret;
2605
2606         output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
2607         if (!output)
2608                 return -ENOMEM;
2609
2610         decomp_output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
2611         if (!decomp_output) {
2612                 kfree(output);
2613                 return -ENOMEM;
2614         }
2615
2616         for (i = 0; i < ctcount; i++) {
2617                 int ilen;
2618                 unsigned int dlen = COMP_BUF_SIZE;
2619
2620                 memset(output, 0, COMP_BUF_SIZE);
2621                 memset(decomp_output, 0, COMP_BUF_SIZE);
2622
2623                 ilen = ctemplate[i].inlen;
2624                 ret = crypto_comp_compress(tfm, ctemplate[i].input,
2625                                            ilen, output, &dlen);
2626                 if (ret) {
2627                         printk(KERN_ERR "alg: comp: compression failed "
2628                                "on test %d for %s: ret=%d\n", i + 1, algo,
2629                                -ret);
2630                         goto out;
2631                 }
2632
2633                 ilen = dlen;
2634                 dlen = COMP_BUF_SIZE;
2635                 ret = crypto_comp_decompress(tfm, output,
2636                                              ilen, decomp_output, &dlen);
2637                 if (ret) {
2638                         pr_err("alg: comp: compression failed: decompress: on test %d for %s failed: ret=%d\n",
2639                                i + 1, algo, -ret);
2640                         goto out;
2641                 }
2642
2643                 if (dlen != ctemplate[i].inlen) {
2644                         printk(KERN_ERR "alg: comp: Compression test %d "
2645                                "failed for %s: output len = %d\n", i + 1, algo,
2646                                dlen);
2647                         ret = -EINVAL;
2648                         goto out;
2649                 }
2650
2651                 if (memcmp(decomp_output, ctemplate[i].input,
2652                            ctemplate[i].inlen)) {
2653                         pr_err("alg: comp: compression failed: output differs: on test %d for %s\n",
2654                                i + 1, algo);
2655                         hexdump(decomp_output, dlen);
2656                         ret = -EINVAL;
2657                         goto out;
2658                 }
2659         }
2660
2661         for (i = 0; i < dtcount; i++) {
2662                 int ilen;
2663                 unsigned int dlen = COMP_BUF_SIZE;
2664
2665                 memset(decomp_output, 0, COMP_BUF_SIZE);
2666
2667                 ilen = dtemplate[i].inlen;
2668                 ret = crypto_comp_decompress(tfm, dtemplate[i].input,
2669                                              ilen, decomp_output, &dlen);
2670                 if (ret) {
2671                         printk(KERN_ERR "alg: comp: decompression failed "
2672                                "on test %d for %s: ret=%d\n", i + 1, algo,
2673                                -ret);
2674                         goto out;
2675                 }
2676
2677                 if (dlen != dtemplate[i].outlen) {
2678                         printk(KERN_ERR "alg: comp: Decompression test %d "
2679                                "failed for %s: output len = %d\n", i + 1, algo,
2680                                dlen);
2681                         ret = -EINVAL;
2682                         goto out;
2683                 }
2684
2685                 if (memcmp(decomp_output, dtemplate[i].output, dlen)) {
2686                         printk(KERN_ERR "alg: comp: Decompression test %d "
2687                                "failed for %s\n", i + 1, algo);
2688                         hexdump(decomp_output, dlen);
2689                         ret = -EINVAL;
2690                         goto out;
2691                 }
2692         }
2693
2694         ret = 0;
2695
2696 out:
2697         kfree(decomp_output);
2698         kfree(output);
2699         return ret;
2700 }
2701
2702 static int test_acomp(struct crypto_acomp *tfm,
2703                               const struct comp_testvec *ctemplate,
2704                       const struct comp_testvec *dtemplate,
2705                       int ctcount, int dtcount)
2706 {
2707         const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));
2708         unsigned int i;
2709         char *output, *decomp_out;
2710         int ret;
2711         struct scatterlist src, dst;
2712         struct acomp_req *req;
2713         struct crypto_wait wait;
2714
2715         output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
2716         if (!output)
2717                 return -ENOMEM;
2718
2719         decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
2720         if (!decomp_out) {
2721                 kfree(output);
2722                 return -ENOMEM;
2723         }
2724
2725         for (i = 0; i < ctcount; i++) {
2726                 unsigned int dlen = COMP_BUF_SIZE;
2727                 int ilen = ctemplate[i].inlen;
2728                 void *input_vec;
2729
2730                 input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL);
2731                 if (!input_vec) {
2732                         ret = -ENOMEM;
2733                         goto out;
2734                 }
2735
2736                 memset(output, 0, dlen);
2737                 crypto_init_wait(&wait);
2738                 sg_init_one(&src, input_vec, ilen);
2739                 sg_init_one(&dst, output, dlen);
2740
2741                 req = acomp_request_alloc(tfm);
2742                 if (!req) {
2743                         pr_err("alg: acomp: request alloc failed for %s\n",
2744                                algo);
2745                         kfree(input_vec);
2746                         ret = -ENOMEM;
2747                         goto out;
2748                 }
2749
2750                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
2751                 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2752                                            crypto_req_done, &wait);
2753
2754                 ret = crypto_wait_req(crypto_acomp_compress(req), &wait);
2755                 if (ret) {
2756                         pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
2757                                i + 1, algo, -ret);
2758                         kfree(input_vec);
2759                         acomp_request_free(req);
2760                         goto out;
2761                 }
2762
2763                 ilen = req->dlen;
2764                 dlen = COMP_BUF_SIZE;
2765                 sg_init_one(&src, output, ilen);
2766                 sg_init_one(&dst, decomp_out, dlen);
2767                 crypto_init_wait(&wait);
2768                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
2769
2770                 ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
2771                 if (ret) {
2772                         pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
2773                                i + 1, algo, -ret);
2774                         kfree(input_vec);
2775                         acomp_request_free(req);
2776                         goto out;
2777                 }
2778
2779                 if (req->dlen != ctemplate[i].inlen) {
2780                         pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n",
2781                                i + 1, algo, req->dlen);
2782                         ret = -EINVAL;
2783                         kfree(input_vec);
2784                         acomp_request_free(req);
2785                         goto out;
2786                 }
2787
2788                 if (memcmp(input_vec, decomp_out, req->dlen)) {
2789                         pr_err("alg: acomp: Compression test %d failed for %s\n",
2790                                i + 1, algo);
2791                         hexdump(output, req->dlen);
2792                         ret = -EINVAL;
2793                         kfree(input_vec);
2794                         acomp_request_free(req);
2795                         goto out;
2796                 }
2797
2798                 kfree(input_vec);
2799                 acomp_request_free(req);
2800         }
2801
2802         for (i = 0; i < dtcount; i++) {
2803                 unsigned int dlen = COMP_BUF_SIZE;
2804                 int ilen = dtemplate[i].inlen;
2805                 void *input_vec;
2806
2807                 input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL);
2808                 if (!input_vec) {
2809                         ret = -ENOMEM;
2810                         goto out;
2811                 }
2812
2813                 memset(output, 0, dlen);
2814                 crypto_init_wait(&wait);
2815                 sg_init_one(&src, input_vec, ilen);
2816                 sg_init_one(&dst, output, dlen);
2817
2818                 req = acomp_request_alloc(tfm);
2819                 if (!req) {
2820                         pr_err("alg: acomp: request alloc failed for %s\n",
2821                                algo);
2822                         kfree(input_vec);
2823                         ret = -ENOMEM;
2824                         goto out;
2825                 }
2826
2827                 acomp_request_set_params(req, &src, &dst, ilen, dlen);
2828                 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2829                                            crypto_req_done, &wait);
2830
2831                 ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
2832                 if (ret) {
2833                         pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n",
2834                                i + 1, algo, -ret);
2835                         kfree(input_vec);
2836                         acomp_request_free(req);
2837                         goto out;
2838                 }
2839
2840                 if (req->dlen != dtemplate[i].outlen) {
2841                         pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n",
2842                                i + 1, algo, req->dlen);
2843                         ret = -EINVAL;
2844                         kfree(input_vec);
2845                         acomp_request_free(req);
2846                         goto out;
2847                 }
2848
2849                 if (memcmp(output, dtemplate[i].output, req->dlen)) {
2850                         pr_err("alg: acomp: Decompression test %d failed for %s\n",
2851                                i + 1, algo);
2852                         hexdump(output, req->dlen);
2853                         ret = -EINVAL;
2854                         kfree(input_vec);
2855                         acomp_request_free(req);
2856                         goto out;
2857                 }
2858
2859                 kfree(input_vec);
2860                 acomp_request_free(req);
2861         }
2862
2863         ret = 0;
2864
2865 out:
2866         kfree(decomp_out);
2867         kfree(output);
2868         return ret;
2869 }
2870
2871 static int test_cprng(struct crypto_rng *tfm,
2872                       const struct cprng_testvec *template,
2873                       unsigned int tcount)
2874 {
2875         const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm));
2876         int err = 0, i, j, seedsize;
2877         u8 *seed;
2878         char result[32];
2879
2880         seedsize = crypto_rng_seedsize(tfm);
2881
2882         seed = kmalloc(seedsize, GFP_KERNEL);
2883         if (!seed) {
2884                 printk(KERN_ERR "alg: cprng: Failed to allocate seed space "
2885                        "for %s\n", algo);
2886                 return -ENOMEM;
2887         }
2888
2889         for (i = 0; i < tcount; i++) {
2890                 memset(result, 0, 32);
2891
2892                 memcpy(seed, template[i].v, template[i].vlen);
2893                 memcpy(seed + template[i].vlen, template[i].key,
2894                        template[i].klen);
2895                 memcpy(seed + template[i].vlen + template[i].klen,
2896                        template[i].dt, template[i].dtlen);
2897
2898                 err = crypto_rng_reset(tfm, seed, seedsize);
2899                 if (err) {
2900                         printk(KERN_ERR "alg: cprng: Failed to reset rng "
2901                                "for %s\n", algo);
2902                         goto out;
2903                 }
2904
2905                 for (j = 0; j < template[i].loops; j++) {
2906                         err = crypto_rng_get_bytes(tfm, result,
2907                                                    template[i].rlen);
2908                         if (err < 0) {
2909                                 printk(KERN_ERR "alg: cprng: Failed to obtain "
2910                                        "the correct amount of random data for "
2911                                        "%s (requested %d)\n", algo,
2912                                        template[i].rlen);
2913                                 goto out;
2914                         }
2915                 }
2916
2917                 err = memcmp(result, template[i].result,
2918                              template[i].rlen);
2919                 if (err) {
2920                         printk(KERN_ERR "alg: cprng: Test %d failed for %s\n",
2921                                i, algo);
2922                         hexdump(result, template[i].rlen);
2923                         err = -EINVAL;
2924                         goto out;
2925                 }
2926         }
2927
2928 out:
2929         kfree(seed);
2930         return err;
2931 }
2932
2933 static int alg_test_cipher(const struct alg_test_desc *desc,
2934                            const char *driver, u32 type, u32 mask)
2935 {
2936         const struct cipher_test_suite *suite = &desc->suite.cipher;
2937         struct crypto_cipher *tfm;
2938         int err;
2939
2940         tfm = crypto_alloc_cipher(driver, type, mask);
2941         if (IS_ERR(tfm)) {
2942                 printk(KERN_ERR "alg: cipher: Failed to load transform for "
2943                        "%s: %ld\n", driver, PTR_ERR(tfm));
2944                 return PTR_ERR(tfm);
2945         }
2946
2947         err = test_cipher(tfm, ENCRYPT, suite->vecs, suite->count);
2948         if (!err)
2949                 err = test_cipher(tfm, DECRYPT, suite->vecs, suite->count);
2950
2951         crypto_free_cipher(tfm);
2952         return err;
2953 }
2954
2955 static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
2956                          u32 type, u32 mask)
2957 {
2958         struct crypto_comp *comp;
2959         struct crypto_acomp *acomp;
2960         int err;
2961         u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK;
2962
2963         if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) {
2964                 acomp = crypto_alloc_acomp(driver, type, mask);
2965                 if (IS_ERR(acomp)) {
2966                         pr_err("alg: acomp: Failed to load transform for %s: %ld\n",
2967                                driver, PTR_ERR(acomp));
2968                         return PTR_ERR(acomp);
2969                 }
2970                 err = test_acomp(acomp, desc->suite.comp.comp.vecs,
2971                                  desc->suite.comp.decomp.vecs,
2972                                  desc->suite.comp.comp.count,
2973                                  desc->suite.comp.decomp.count);
2974                 crypto_free_acomp(acomp);
2975         } else {
2976                 comp = crypto_alloc_comp(driver, type, mask);
2977                 if (IS_ERR(comp)) {
2978                         pr_err("alg: comp: Failed to load transform for %s: %ld\n",
2979                                driver, PTR_ERR(comp));
2980                         return PTR_ERR(comp);
2981                 }
2982
2983                 err = test_comp(comp, desc->suite.comp.comp.vecs,
2984                                 desc->suite.comp.decomp.vecs,
2985                                 desc->suite.comp.comp.count,
2986                                 desc->suite.comp.decomp.count);
2987
2988                 crypto_free_comp(comp);
2989         }
2990         return err;
2991 }
2992
2993 static int alg_test_crc32c(const struct alg_test_desc *desc,
2994                            const char *driver, u32 type, u32 mask)
2995 {
2996         struct crypto_shash *tfm;
2997         __le32 val;
2998         int err;
2999
3000         err = alg_test_hash(desc, driver, type, mask);
3001         if (err)
3002                 return err;
3003
3004         tfm = crypto_alloc_shash(driver, type, mask);
3005         if (IS_ERR(tfm)) {
3006                 if (PTR_ERR(tfm) == -ENOENT) {
3007                         /*
3008                          * This crc32c implementation is only available through
3009                          * ahash API, not the shash API, so the remaining part
3010                          * of the test is not applicable to it.
3011                          */
3012                         return 0;
3013                 }
3014                 printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: "
3015                        "%ld\n", driver, PTR_ERR(tfm));
3016                 return PTR_ERR(tfm);
3017         }
3018
3019         do {
3020                 SHASH_DESC_ON_STACK(shash, tfm);
3021                 u32 *ctx = (u32 *)shash_desc_ctx(shash);
3022
3023                 shash->tfm = tfm;
3024
3025                 *ctx = 420553207;
3026                 err = crypto_shash_final(shash, (u8 *)&val);
3027                 if (err) {
3028                         printk(KERN_ERR "alg: crc32c: Operation failed for "
3029                                "%s: %d\n", driver, err);
3030                         break;
3031                 }
3032
3033                 if (val != cpu_to_le32(~420553207)) {
3034                         pr_err("alg: crc32c: Test failed for %s: %u\n",
3035                                driver, le32_to_cpu(val));
3036                         err = -EINVAL;
3037                 }
3038         } while (0);
3039
3040         crypto_free_shash(tfm);
3041
3042         return err;
3043 }
3044
3045 static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver,
3046                           u32 type, u32 mask)
3047 {
3048         struct crypto_rng *rng;
3049         int err;
3050
3051         rng = crypto_alloc_rng(driver, type, mask);
3052         if (IS_ERR(rng)) {
3053                 printk(KERN_ERR "alg: cprng: Failed to load transform for %s: "
3054                        "%ld\n", driver, PTR_ERR(rng));
3055                 return PTR_ERR(rng);
3056         }
3057
3058         err = test_cprng(rng, desc->suite.cprng.vecs, desc->suite.cprng.count);
3059
3060         crypto_free_rng(rng);
3061
3062         return err;
3063 }
3064
3065
3066 static int drbg_cavs_test(const struct drbg_testvec *test, int pr,
3067                           const char *driver, u32 type, u32 mask)
3068 {
3069         int ret = -EAGAIN;
3070         struct crypto_rng *drng;
3071         struct drbg_test_data test_data;
3072         struct drbg_string addtl, pers, testentropy;
3073         unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL);
3074
3075         if (!buf)
3076                 return -ENOMEM;
3077
3078         drng = crypto_alloc_rng(driver, type, mask);
3079         if (IS_ERR(drng)) {
3080                 printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "
3081                        "%s\n", driver);
3082                 kzfree(buf);
3083                 return -ENOMEM;
3084         }
3085
3086         test_data.testentropy = &testentropy;
3087         drbg_string_fill(&testentropy, test->entropy, test->entropylen);
3088         drbg_string_fill(&pers, test->pers, test->perslen);
3089         ret = crypto_drbg_reset_test(drng, &pers, &test_data);
3090         if (ret) {
3091                 printk(KERN_ERR "alg: drbg: Failed to reset rng\n");
3092                 goto outbuf;
3093         }
3094
3095         drbg_string_fill(&addtl, test->addtla, test->addtllen);
3096         if (pr) {
3097                 drbg_string_fill(&testentropy, test->entpra, test->entprlen);
3098                 ret = crypto_drbg_get_bytes_addtl_test(drng,
3099                         buf, test->expectedlen, &addtl, &test_data);
3100         } else {
3101                 ret = crypto_drbg_get_bytes_addtl(drng,
3102                         buf, test->expectedlen, &addtl);
3103         }
3104         if (ret < 0) {
3105                 printk(KERN_ERR "alg: drbg: could not obtain random data for "
3106                        "driver %s\n", driver);
3107                 goto outbuf;
3108         }
3109
3110         drbg_string_fill(&addtl, test->addtlb, test->addtllen);
3111         if (pr) {
3112                 drbg_string_fill(&testentropy, test->entprb, test->entprlen);
3113                 ret = crypto_drbg_get_bytes_addtl_test(drng,
3114                         buf, test->expectedlen, &addtl, &test_data);
3115         } else {
3116                 ret = crypto_drbg_get_bytes_addtl(drng,
3117                         buf, test->expectedlen, &addtl);
3118         }
3119         if (ret < 0) {
3120                 printk(KERN_ERR "alg: drbg: could not obtain random data for "
3121                        "driver %s\n", driver);
3122                 goto outbuf;
3123         }
3124
3125         ret = memcmp(test->expected, buf, test->expectedlen);
3126
3127 outbuf:
3128         crypto_free_rng(drng);
3129         kzfree(buf);
3130         return ret;
3131 }
3132
3133
3134 static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver,
3135                          u32 type, u32 mask)
3136 {
3137         int err = 0;
3138         int pr = 0;
3139         int i = 0;
3140         const struct drbg_testvec *template = desc->suite.drbg.vecs;
3141         unsigned int tcount = desc->suite.drbg.count;
3142
3143         if (0 == memcmp(driver, "drbg_pr_", 8))
3144                 pr = 1;
3145
3146         for (i = 0; i < tcount; i++) {
3147                 err = drbg_cavs_test(&template[i], pr, driver, type, mask);
3148                 if (err) {
3149                         printk(KERN_ERR "alg: drbg: Test %d failed for %s\n",
3150                                i, driver);
3151                         err = -EINVAL;
3152                         break;
3153                 }
3154         }
3155         return err;
3156
3157 }
3158
3159 static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
3160                        const char *alg)
3161 {
3162         struct kpp_request *req;
3163         void *input_buf = NULL;
3164         void *output_buf = NULL;
3165         void *a_public = NULL;
3166         void *a_ss = NULL;
3167         void *shared_secret = NULL;
3168         struct crypto_wait wait;
3169         unsigned int out_len_max;
3170         int err = -ENOMEM;
3171         struct scatterlist src, dst;
3172
3173         req = kpp_request_alloc(tfm, GFP_KERNEL);
3174         if (!req)
3175                 return err;
3176
3177         crypto_init_wait(&wait);
3178
3179         err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size);
3180         if (err < 0)
3181                 goto free_req;
3182
3183         out_len_max = crypto_kpp_maxsize(tfm);
3184         output_buf = kzalloc(out_len_max, GFP_KERNEL);
3185         if (!output_buf) {
3186                 err = -ENOMEM;
3187                 goto free_req;
3188         }
3189
3190         /* Use appropriate parameter as base */
3191         kpp_request_set_input(req, NULL, 0);
3192         sg_init_one(&dst, output_buf, out_len_max);
3193         kpp_request_set_output(req, &dst, out_len_max);
3194         kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3195                                  crypto_req_done, &wait);
3196
3197         /* Compute party A's public key */
3198         err = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait);
3199         if (err) {
3200                 pr_err("alg: %s: Party A: generate public key test failed. err %d\n",
3201                        alg, err);
3202                 goto free_output;
3203         }
3204
3205         if (vec->genkey) {
3206                 /* Save party A's public key */
3207                 a_public = kmemdup(sg_virt(req->dst), out_len_max, GFP_KERNEL);
3208                 if (!a_public) {
3209                         err = -ENOMEM;
3210                         goto free_output;
3211                 }
3212         } else {
3213                 /* Verify calculated public key */
3214                 if (memcmp(vec->expected_a_public, sg_virt(req->dst),
3215                            vec->expected_a_public_size)) {
3216                         pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n",
3217                                alg);
3218                         err = -EINVAL;
3219                         goto free_output;
3220                 }
3221         }
3222
3223         /* Calculate shared secret key by using counter part (b) public key. */
3224         input_buf = kmemdup(vec->b_public, vec->b_public_size, GFP_KERNEL);
3225         if (!input_buf) {
3226                 err = -ENOMEM;
3227                 goto free_output;
3228         }
3229
3230         sg_init_one(&src, input_buf, vec->b_public_size);
3231         sg_init_one(&dst, output_buf, out_len_max);
3232         kpp_request_set_input(req, &src, vec->b_public_size);
3233         kpp_request_set_output(req, &dst, out_len_max);
3234         kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3235                                  crypto_req_done, &wait);
3236         err = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait);
3237         if (err) {
3238                 pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n",
3239                        alg, err);
3240                 goto free_all;
3241         }
3242
3243         if (vec->genkey) {
3244                 /* Save the shared secret obtained by party A */
3245                 a_ss = kmemdup(sg_virt(req->dst), vec->expected_ss_size, GFP_KERNEL);
3246                 if (!a_ss) {
3247                         err = -ENOMEM;
3248                         goto free_all;
3249                 }
3250
3251                 /*
3252                  * Calculate party B's shared secret by using party A's
3253                  * public key.
3254                  */
3255                 err = crypto_kpp_set_secret(tfm, vec->b_secret,
3256                                             vec->b_secret_size);
3257                 if (err < 0)
3258                         goto free_all;
3259
3260                 sg_init_one(&src, a_public, vec->expected_a_public_size);
3261                 sg_init_one(&dst, output_buf, out_len_max);
3262                 kpp_request_set_input(req, &src, vec->expected_a_public_size);
3263                 kpp_request_set_output(req, &dst, out_len_max);
3264                 kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3265                                          crypto_req_done, &wait);
3266                 err = crypto_wait_req(crypto_kpp_compute_shared_secret(req),
3267                                       &wait);
3268                 if (err) {
3269                         pr_err("alg: %s: Party B: compute shared secret failed. err %d\n",
3270                                alg, err);
3271                         goto free_all;
3272                 }
3273
3274                 shared_secret = a_ss;
3275         } else {
3276                 shared_secret = (void *)vec->expected_ss;
3277         }
3278
3279         /*
3280          * verify shared secret from which the user will derive
3281          * secret key by executing whatever hash it has chosen
3282          */
3283         if (memcmp(shared_secret, sg_virt(req->dst),
3284                    vec->expected_ss_size)) {
3285                 pr_err("alg: %s: compute shared secret test failed. Invalid output\n",
3286                        alg);
3287                 err = -EINVAL;
3288         }
3289
3290 free_all:
3291         kfree(a_ss);
3292         kfree(input_buf);
3293 free_output:
3294         kfree(a_public);
3295         kfree(output_buf);
3296 free_req:
3297         kpp_request_free(req);
3298         return err;
3299 }
3300
3301 static int test_kpp(struct crypto_kpp *tfm, const char *alg,
3302                     const struct kpp_testvec *vecs, unsigned int tcount)
3303 {
3304         int ret, i;
3305
3306         for (i = 0; i < tcount; i++) {
3307                 ret = do_test_kpp(tfm, vecs++, alg);
3308                 if (ret) {
3309                         pr_err("alg: %s: test failed on vector %d, err=%d\n",
3310                                alg, i + 1, ret);
3311                         return ret;
3312                 }
3313         }
3314         return 0;
3315 }
3316
3317 static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,
3318                         u32 type, u32 mask)
3319 {
3320         struct crypto_kpp *tfm;
3321         int err = 0;
3322
3323         tfm = crypto_alloc_kpp(driver, type, mask);
3324         if (IS_ERR(tfm)) {
3325                 pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",
3326                        driver, PTR_ERR(tfm));
3327                 return PTR_ERR(tfm);
3328         }
3329         if (desc->suite.kpp.vecs)
3330                 err = test_kpp(tfm, desc->alg, desc->suite.kpp.vecs,
3331                                desc->suite.kpp.count);
3332
3333         crypto_free_kpp(tfm);
3334         return err;
3335 }
3336
3337 static u8 *test_pack_u32(u8 *dst, u32 val)
3338 {
3339         memcpy(dst, &val, sizeof(val));
3340         return dst + sizeof(val);
3341 }
3342
3343 static int test_akcipher_one(struct crypto_akcipher *tfm,
3344                              const struct akcipher_testvec *vecs)
3345 {
3346         char *xbuf[XBUFSIZE];
3347         struct akcipher_request *req;
3348         void *outbuf_enc = NULL;
3349         void *outbuf_dec = NULL;
3350         struct crypto_wait wait;
3351         unsigned int out_len_max, out_len = 0;
3352         int err = -ENOMEM;
3353         struct scatterlist src, dst, src_tab[3];
3354         const char *m, *c;
3355         unsigned int m_size, c_size;
3356         const char *op;
3357         u8 *key, *ptr;
3358
3359         if (testmgr_alloc_buf(xbuf))
3360                 return err;
3361
3362         req = akcipher_request_alloc(tfm, GFP_KERNEL);
3363         if (!req)
3364                 goto free_xbuf;
3365
3366         crypto_init_wait(&wait);
3367
3368         key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len,
3369                       GFP_KERNEL);
3370         if (!key)
3371                 goto free_xbuf;
3372         memcpy(key, vecs->key, vecs->key_len);
3373         ptr = key + vecs->key_len;
3374         ptr = test_pack_u32(ptr, vecs->algo);
3375         ptr = test_pack_u32(ptr, vecs->param_len);
3376         memcpy(ptr, vecs->params, vecs->param_len);
3377
3378         if (vecs->public_key_vec)
3379                 err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len);
3380         else
3381                 err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len);
3382         if (err)
3383                 goto free_req;
3384
3385         /*
3386          * First run test which do not require a private key, such as
3387          * encrypt or verify.
3388          */
3389         err = -ENOMEM;
3390         out_len_max = crypto_akcipher_maxsize(tfm);
3391         outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
3392         if (!outbuf_enc)
3393                 goto free_req;
3394
3395         if (!vecs->siggen_sigver_test) {
3396                 m = vecs->m;
3397                 m_size = vecs->m_size;
3398                 c = vecs->c;
3399                 c_size = vecs->c_size;
3400                 op = "encrypt";
3401         } else {
3402                 /* Swap args so we could keep plaintext (digest)
3403                  * in vecs->m, and cooked signature in vecs->c.
3404                  */
3405                 m = vecs->c; /* signature */
3406                 m_size = vecs->c_size;
3407                 c = vecs->m; /* digest */
3408                 c_size = vecs->m_size;
3409                 op = "verify";
3410         }
3411
3412         if (WARN_ON(m_size > PAGE_SIZE))
3413                 goto free_all;
3414         memcpy(xbuf[0], m, m_size);
3415
3416         sg_init_table(src_tab, 3);
3417         sg_set_buf(&src_tab[0], xbuf[0], 8);
3418         sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8);
3419         if (vecs->siggen_sigver_test) {
3420                 if (WARN_ON(c_size > PAGE_SIZE))
3421                         goto free_all;
3422                 memcpy(xbuf[1], c, c_size);
3423                 sg_set_buf(&src_tab[2], xbuf[1], c_size);
3424                 akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size);
3425         } else {
3426                 sg_init_one(&dst, outbuf_enc, out_len_max);
3427                 akcipher_request_set_crypt(req, src_tab, &dst, m_size,
3428                                            out_len_max);
3429         }
3430         akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3431                                       crypto_req_done, &wait);
3432
3433         err = crypto_wait_req(vecs->siggen_sigver_test ?
3434                               /* Run asymmetric signature verification */
3435                               crypto_akcipher_verify(req) :
3436                               /* Run asymmetric encrypt */
3437                               crypto_akcipher_encrypt(req), &wait);
3438         if (err) {
3439                 pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
3440                 goto free_all;
3441         }
3442         if (!vecs->siggen_sigver_test) {
3443                 if (req->dst_len != c_size) {
3444                         pr_err("alg: akcipher: %s test failed. Invalid output len\n",
3445                                op);
3446                         err = -EINVAL;
3447                         goto free_all;
3448                 }
3449                 /* verify that encrypted message is equal to expected */
3450                 if (memcmp(c, outbuf_enc, c_size) != 0) {
3451                         pr_err("alg: akcipher: %s test failed. Invalid output\n",
3452                                op);
3453                         hexdump(outbuf_enc, c_size);
3454                         err = -EINVAL;
3455                         goto free_all;
3456                 }
3457         }
3458
3459         /*
3460          * Don't invoke (decrypt or sign) test which require a private key
3461          * for vectors with only a public key.
3462          */
3463         if (vecs->public_key_vec) {
3464                 err = 0;
3465                 goto free_all;
3466         }
3467         outbuf_dec = kzalloc(out_len_max, GFP_KERNEL);
3468         if (!outbuf_dec) {
3469                 err = -ENOMEM;
3470                 goto free_all;
3471         }
3472
3473         op = vecs->siggen_sigver_test ? "sign" : "decrypt";
3474         if (WARN_ON(c_size > PAGE_SIZE))
3475                 goto free_all;
3476         memcpy(xbuf[0], c, c_size);
3477
3478         sg_init_one(&src, xbuf[0], c_size);
3479         sg_init_one(&dst, outbuf_dec, out_len_max);
3480         crypto_init_wait(&wait);
3481         akcipher_request_set_crypt(req, &src, &dst, c_size, out_len_max);
3482
3483         err = crypto_wait_req(vecs->siggen_sigver_test ?
3484                               /* Run asymmetric signature generation */
3485                               crypto_akcipher_sign(req) :
3486                               /* Run asymmetric decrypt */
3487                               crypto_akcipher_decrypt(req), &wait);
3488         if (err) {
3489                 pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
3490                 goto free_all;
3491         }
3492         out_len = req->dst_len;
3493         if (out_len < m_size) {
3494                 pr_err("alg: akcipher: %s test failed. Invalid output len %u\n",
3495                        op, out_len);
3496                 err = -EINVAL;
3497                 goto free_all;
3498         }
3499         /* verify that decrypted message is equal to the original msg */
3500         if (memchr_inv(outbuf_dec, 0, out_len - m_size) ||
3501             memcmp(m, outbuf_dec + out_len - m_size, m_size)) {
3502                 pr_err("alg: akcipher: %s test failed. Invalid output\n", op);
3503                 hexdump(outbuf_dec, out_len);
3504                 err = -EINVAL;
3505         }
3506 free_all:
3507         kfree(outbuf_dec);
3508         kfree(outbuf_enc);
3509 free_req:
3510         akcipher_request_free(req);
3511         kfree(key);
3512 free_xbuf:
3513         testmgr_free_buf(xbuf);
3514         return err;
3515 }
3516
3517 static int test_akcipher(struct crypto_akcipher *tfm, const char *alg,
3518                          const struct akcipher_testvec *vecs,
3519                          unsigned int tcount)
3520 {
3521         const char *algo =
3522                 crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm));
3523         int ret, i;
3524
3525         for (i = 0; i < tcount; i++) {
3526                 ret = test_akcipher_one(tfm, vecs++);
3527                 if (!ret)
3528                         continue;
3529
3530                 pr_err("alg: akcipher: test %d failed for %s, err=%d\n",
3531                        i + 1, algo, ret);
3532                 return ret;
3533         }
3534         return 0;
3535 }
3536
3537 static int alg_test_akcipher(const struct alg_test_desc *desc,
3538                              const char *driver, u32 type, u32 mask)
3539 {
3540         struct crypto_akcipher *tfm;
3541         int err = 0;
3542
3543         tfm = crypto_alloc_akcipher(driver, type, mask);
3544         if (IS_ERR(tfm)) {
3545                 pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n",
3546                        driver, PTR_ERR(tfm));
3547                 return PTR_ERR(tfm);
3548         }
3549         if (desc->suite.akcipher.vecs)
3550                 err = test_akcipher(tfm, desc->alg, desc->suite.akcipher.vecs,
3551                                     desc->suite.akcipher.count);
3552
3553         crypto_free_akcipher(tfm);
3554         return err;
3555 }
3556
3557 static int alg_test_null(const struct alg_test_desc *desc,
3558                              const char *driver, u32 type, u32 mask)
3559 {
3560         return 0;
3561 }
3562
3563 #define __VECS(tv)      { .vecs = tv, .count = ARRAY_SIZE(tv) }
3564
3565 /* Please keep this list sorted by algorithm name. */
3566 static const struct alg_test_desc alg_test_descs[] = {
3567         {
3568                 .alg = "adiantum(xchacha12,aes)",
3569                 .generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)",
3570                 .test = alg_test_skcipher,
3571                 .suite = {
3572                         .cipher = __VECS(adiantum_xchacha12_aes_tv_template)
3573                 },
3574         }, {
3575                 .alg = "adiantum(xchacha20,aes)",
3576                 .generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)",
3577                 .test = alg_test_skcipher,
3578                 .suite = {
3579                         .cipher = __VECS(adiantum_xchacha20_aes_tv_template)
3580                 },
3581         }, {
3582                 .alg = "aegis128",
3583                 .test = alg_test_aead,
3584                 .suite = {
3585                         .aead = __VECS(aegis128_tv_template)
3586                 }
3587         }, {
3588                 .alg = "aegis128l",
3589                 .test = alg_test_aead,
3590                 .suite = {
3591                         .aead = __VECS(aegis128l_tv_template)
3592                 }
3593         }, {
3594                 .alg = "aegis256",
3595                 .test = alg_test_aead,
3596                 .suite = {
3597                         .aead = __VECS(aegis256_tv_template)
3598                 }
3599         }, {
3600                 .alg = "ansi_cprng",
3601                 .test = alg_test_cprng,
3602                 .suite = {
3603                         .cprng = __VECS(ansi_cprng_aes_tv_template)
3604                 }
3605         }, {
3606                 .alg = "authenc(hmac(md5),ecb(cipher_null))",
3607                 .test = alg_test_aead,
3608                 .suite = {
3609                         .aead = __VECS(hmac_md5_ecb_cipher_null_tv_template)
3610                 }
3611         }, {
3612                 .alg = "authenc(hmac(sha1),cbc(aes))",
3613                 .test = alg_test_aead,
3614                 .fips_allowed = 1,
3615                 .suite = {
3616                         .aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
3617                 }
3618         }, {
3619                 .alg = "authenc(hmac(sha1),cbc(des))",
3620                 .test = alg_test_aead,
3621                 .suite = {
3622                         .aead = __VECS(hmac_sha1_des_cbc_tv_temp)
3623                 }
3624         }, {
3625                 .alg = "authenc(hmac(sha1),cbc(des3_ede))",
3626                 .test = alg_test_aead,
3627                 .fips_allowed = 1,
3628                 .suite = {
3629                         .aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
3630                 }
3631         }, {
3632                 .alg = "authenc(hmac(sha1),ctr(aes))",
3633                 .test = alg_test_null,
3634                 .fips_allowed = 1,
3635         }, {
3636                 .alg = "authenc(hmac(sha1),ecb(cipher_null))",
3637                 .test = alg_test_aead,
3638                 .suite = {
3639                         .aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)
3640                 }
3641         }, {
3642                 .alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
3643                 .test = alg_test_null,
3644                 .fips_allowed = 1,
3645         }, {
3646                 .alg = "authenc(hmac(sha224),cbc(des))",
3647                 .test = alg_test_aead,
3648                 .suite = {
3649                         .aead = __VECS(hmac_sha224_des_cbc_tv_temp)
3650                 }
3651         }, {
3652                 .alg = "authenc(hmac(sha224),cbc(des3_ede))",
3653                 .test = alg_test_aead,
3654                 .fips_allowed = 1,
3655                 .suite = {
3656                         .aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
3657                 }
3658         }, {
3659                 .alg = "authenc(hmac(sha256),cbc(aes))",
3660                 .test = alg_test_aead,
3661                 .fips_allowed = 1,
3662                 .suite = {
3663                         .aead = __VECS(hmac_sha256_aes_cbc_tv_temp)
3664                 }
3665         }, {
3666                 .alg = "authenc(hmac(sha256),cbc(des))",
3667                 .test = alg_test_aead,
3668                 .suite = {
3669                         .aead = __VECS(hmac_sha256_des_cbc_tv_temp)
3670                 }
3671         }, {
3672                 .alg = "authenc(hmac(sha256),cbc(des3_ede))",
3673                 .test = alg_test_aead,
3674                 .fips_allowed = 1,
3675                 .suite = {
3676                         .aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
3677                 }
3678         }, {
3679                 .alg = "authenc(hmac(sha256),ctr(aes))",
3680                 .test = alg_test_null,
3681                 .fips_allowed = 1,
3682         }, {
3683                 .alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
3684                 .test = alg_test_null,
3685                 .fips_allowed = 1,
3686         }, {
3687                 .alg = "authenc(hmac(sha384),cbc(des))",
3688                 .test = alg_test_aead,
3689                 .suite = {
3690                         .aead = __VECS(hmac_sha384_des_cbc_tv_temp)
3691                 }
3692         }, {
3693                 .alg = "authenc(hmac(sha384),cbc(des3_ede))",
3694                 .test = alg_test_aead,
3695                 .fips_allowed = 1,
3696                 .suite = {
3697                         .aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
3698                 }
3699         }, {
3700                 .alg = "authenc(hmac(sha384),ctr(aes))",
3701                 .test = alg_test_null,
3702                 .fips_allowed = 1,
3703         }, {
3704                 .alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
3705                 .test = alg_test_null,
3706                 .fips_allowed = 1,
3707         }, {
3708                 .alg = "authenc(hmac(sha512),cbc(aes))",
3709                 .fips_allowed = 1,
3710                 .test = alg_test_aead,
3711                 .suite = {
3712                         .aead = __VECS(hmac_sha512_aes_cbc_tv_temp)
3713                 }
3714         }, {
3715                 .alg = "authenc(hmac(sha512),cbc(des))",
3716                 .test = alg_test_aead,
3717                 .suite = {
3718                         .aead = __VECS(hmac_sha512_des_cbc_tv_temp)
3719                 }
3720         }, {
3721                 .alg = "authenc(hmac(sha512),cbc(des3_ede))",
3722                 .test = alg_test_aead,
3723                 .fips_allowed = 1,
3724                 .suite = {
3725                         .aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
3726                 }
3727         }, {
3728                 .alg = "authenc(hmac(sha512),ctr(aes))",
3729                 .test = alg_test_null,
3730                 .fips_allowed = 1,
3731         }, {
3732                 .alg = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
3733                 .test = alg_test_null,
3734                 .fips_allowed = 1,
3735         }, {
3736                 .alg = "cbc(aes)",
3737                 .test = alg_test_skcipher,
3738                 .fips_allowed = 1,
3739                 .suite = {
3740                         .cipher = __VECS(aes_cbc_tv_template)
3741                 },
3742         }, {
3743                 .alg = "cbc(anubis)",
3744                 .test = alg_test_skcipher,
3745                 .suite = {
3746                         .cipher = __VECS(anubis_cbc_tv_template)
3747                 },
3748         }, {
3749                 .alg = "cbc(blowfish)",
3750                 .test = alg_test_skcipher,
3751                 .suite = {
3752                         .cipher = __VECS(bf_cbc_tv_template)
3753                 },
3754         }, {
3755                 .alg = "cbc(camellia)",
3756                 .test = alg_test_skcipher,
3757                 .suite = {
3758                         .cipher = __VECS(camellia_cbc_tv_template)
3759                 },
3760         }, {
3761                 .alg = "cbc(cast5)",
3762                 .test = alg_test_skcipher,
3763                 .suite = {
3764                         .cipher = __VECS(cast5_cbc_tv_template)
3765                 },
3766         }, {
3767                 .alg = "cbc(cast6)",
3768                 .test = alg_test_skcipher,
3769                 .suite = {
3770                         .cipher = __VECS(cast6_cbc_tv_template)
3771                 },
3772         }, {
3773                 .alg = "cbc(des)",
3774                 .test = alg_test_skcipher,
3775                 .suite = {
3776                         .cipher = __VECS(des_cbc_tv_template)
3777                 },
3778         }, {
3779                 .alg = "cbc(des3_ede)",
3780                 .test = alg_test_skcipher,
3781                 .fips_allowed = 1,
3782                 .suite = {
3783                         .cipher = __VECS(des3_ede_cbc_tv_template)
3784                 },
3785         }, {
3786                 /* Same as cbc(aes) except the key is stored in
3787                  * hardware secure memory which we reference by index
3788                  */
3789                 .alg = "cbc(paes)",
3790                 .test = alg_test_null,
3791                 .fips_allowed = 1,
3792         }, {
3793                 /* Same as cbc(sm4) except the key is stored in
3794                  * hardware secure memory which we reference by index
3795                  */
3796                 .alg = "cbc(psm4)",
3797                 .test = alg_test_null,
3798         }, {
3799                 .alg = "cbc(serpent)",
3800                 .test = alg_test_skcipher,
3801                 .suite = {
3802                         .cipher = __VECS(serpent_cbc_tv_template)
3803                 },
3804         }, {
3805                 .alg = "cbc(sm4)",
3806                 .test = alg_test_skcipher,
3807                 .suite = {
3808                         .cipher = __VECS(sm4_cbc_tv_template)
3809                 }
3810         }, {
3811                 .alg = "cbc(twofish)",
3812                 .test = alg_test_skcipher,
3813                 .suite = {
3814                         .cipher = __VECS(tf_cbc_tv_template)
3815                 },
3816         }, {
3817                 .alg = "cbcmac(aes)",
3818                 .fips_allowed = 1,
3819                 .test = alg_test_hash,
3820                 .suite = {
3821                         .hash = __VECS(aes_cbcmac_tv_template)
3822                 }
3823         }, {
3824                 .alg = "ccm(aes)",
3825                 .generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))",
3826                 .test = alg_test_aead,
3827                 .fips_allowed = 1,
3828                 .suite = {
3829                         .aead = __VECS(aes_ccm_tv_template)
3830                 }
3831         }, {
3832                 .alg = "cfb(aes)",
3833                 .test = alg_test_skcipher,
3834                 .fips_allowed = 1,
3835                 .suite = {
3836                         .cipher = __VECS(aes_cfb_tv_template)
3837                 },
3838         }, {
3839                 .alg = "chacha20",
3840                 .test = alg_test_skcipher,
3841                 .suite = {
3842                         .cipher = __VECS(chacha20_tv_template)
3843                 },
3844         }, {
3845                 .alg = "cmac(aes)",
3846                 .fips_allowed = 1,
3847                 .test = alg_test_hash,
3848                 .suite = {
3849                         .hash = __VECS(aes_cmac128_tv_template)
3850                 }
3851         }, {
3852                 .alg = "cmac(des3_ede)",
3853                 .fips_allowed = 1,
3854                 .test = alg_test_hash,
3855                 .suite = {
3856                         .hash = __VECS(des3_ede_cmac64_tv_template)
3857                 }
3858         }, {
3859                 .alg = "compress_null",
3860                 .test = alg_test_null,
3861         }, {
3862                 .alg = "crc32",
3863                 .test = alg_test_hash,
3864                 .fips_allowed = 1,
3865                 .suite = {
3866                         .hash = __VECS(crc32_tv_template)
3867                 }
3868         }, {
3869                 .alg = "crc32c",
3870                 .test = alg_test_crc32c,
3871                 .fips_allowed = 1,
3872                 .suite = {
3873                         .hash = __VECS(crc32c_tv_template)
3874                 }
3875         }, {
3876                 .alg = "crct10dif",
3877                 .test = alg_test_hash,
3878                 .fips_allowed = 1,
3879                 .suite = {
3880                         .hash = __VECS(crct10dif_tv_template)
3881                 }
3882         }, {
3883                 .alg = "ctr(aes)",
3884                 .test = alg_test_skcipher,
3885                 .fips_allowed = 1,
3886                 .suite = {
3887                         .cipher = __VECS(aes_ctr_tv_template)
3888                 }
3889         }, {
3890                 .alg = "ctr(blowfish)",
3891                 .test = alg_test_skcipher,
3892                 .suite = {
3893                         .cipher = __VECS(bf_ctr_tv_template)
3894                 }
3895         }, {
3896                 .alg = "ctr(camellia)",
3897                 .test = alg_test_skcipher,
3898                 .suite = {
3899                         .cipher = __VECS(camellia_ctr_tv_template)
3900                 }
3901         }, {
3902                 .alg = "ctr(cast5)",
3903                 .test = alg_test_skcipher,
3904                 .suite = {
3905                         .cipher = __VECS(cast5_ctr_tv_template)
3906                 }
3907         }, {
3908                 .alg = "ctr(cast6)",
3909                 .test = alg_test_skcipher,
3910                 .suite = {
3911                         .cipher = __VECS(cast6_ctr_tv_template)
3912                 }
3913         }, {
3914                 .alg = "ctr(des)",
3915                 .test = alg_test_skcipher,
3916                 .suite = {
3917                         .cipher = __VECS(des_ctr_tv_template)
3918                 }
3919         }, {
3920                 .alg = "ctr(des3_ede)",
3921                 .test = alg_test_skcipher,
3922                 .fips_allowed = 1,
3923                 .suite = {
3924                         .cipher = __VECS(des3_ede_ctr_tv_template)
3925                 }
3926         }, {
3927                 /* Same as ctr(aes) except the key is stored in
3928                  * hardware secure memory which we reference by index
3929                  */
3930                 .alg = "ctr(paes)",
3931                 .test = alg_test_null,
3932                 .fips_allowed = 1,
3933         }, {
3934
3935                 /* Same as ctr(sm4) except the key is stored in
3936                  * hardware secure memory which we reference by index
3937                  */
3938                 .alg = "ctr(psm4)",
3939                 .test = alg_test_null,
3940         }, {
3941                 .alg = "ctr(serpent)",
3942                 .test = alg_test_skcipher,
3943                 .suite = {
3944                         .cipher = __VECS(serpent_ctr_tv_template)
3945                 }
3946         }, {
3947                 .alg = "ctr(sm4)",
3948                 .test = alg_test_skcipher,
3949                 .suite = {
3950                         .cipher = __VECS(sm4_ctr_tv_template)
3951                 }
3952         }, {
3953                 .alg = "ctr(twofish)",
3954                 .test = alg_test_skcipher,
3955                 .suite = {
3956                         .cipher = __VECS(tf_ctr_tv_template)
3957                 }
3958         }, {
3959                 .alg = "cts(cbc(aes))",
3960                 .test = alg_test_skcipher,
3961                 .fips_allowed = 1,
3962                 .suite = {
3963                         .cipher = __VECS(cts_mode_tv_template)
3964                 }
3965         }, {
3966                 /* Same as cts(cbc((aes)) except the key is stored in
3967                  * hardware secure memory which we reference by index
3968                  */
3969                 .alg = "cts(cbc(paes))",
3970                 .test = alg_test_null,
3971                 .fips_allowed = 1,
3972         }, {
3973                 .alg = "deflate",
3974                 .test = alg_test_comp,
3975                 .fips_allowed = 1,
3976                 .suite = {
3977                         .comp = {
3978                                 .comp = __VECS(deflate_comp_tv_template),
3979                                 .decomp = __VECS(deflate_decomp_tv_template)
3980                         }
3981                 }
3982         }, {
3983                 .alg = "dh",
3984                 .test = alg_test_kpp,
3985                 .fips_allowed = 1,
3986                 .suite = {
3987                         .kpp = __VECS(dh_tv_template)
3988                 }
3989         }, {
3990                 .alg = "digest_null",
3991                 .test = alg_test_null,
3992         }, {
3993                 .alg = "drbg_nopr_ctr_aes128",
3994                 .test = alg_test_drbg,
3995                 .fips_allowed = 1,
3996                 .suite = {
3997                         .drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)
3998                 }
3999         }, {
4000                 .alg = "drbg_nopr_ctr_aes192",
4001                 .test = alg_test_drbg,
4002                 .fips_allowed = 1,
4003                 .suite = {
4004                         .drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)
4005                 }
4006         }, {
4007                 .alg = "drbg_nopr_ctr_aes256",
4008                 .test = alg_test_drbg,
4009                 .fips_allowed = 1,
4010                 .suite = {
4011                         .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
4012                 }
4013         }, {
4014                 /*
4015                  * There is no need to specifically test the DRBG with every
4016                  * backend cipher -- covered by drbg_nopr_hmac_sha256 test
4017                  */
4018                 .alg = "drbg_nopr_hmac_sha1",
4019                 .fips_allowed = 1,
4020                 .test = alg_test_null,
4021         }, {
4022                 .alg = "drbg_nopr_hmac_sha256",
4023                 .test = alg_test_drbg,
4024                 .fips_allowed = 1,
4025                 .suite = {
4026                         .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
4027                 }
4028         }, {
4029                 /* covered by drbg_nopr_hmac_sha256 test */
4030                 .alg = "drbg_nopr_hmac_sha384",
4031                 .fips_allowed = 1,
4032                 .test = alg_test_null,
4033         }, {
4034                 .alg = "drbg_nopr_hmac_sha512",
4035                 .test = alg_test_null,
4036                 .fips_allowed = 1,
4037         }, {
4038                 .alg = "drbg_nopr_sha1",
4039                 .fips_allowed = 1,
4040                 .test = alg_test_null,
4041         }, {
4042                 .alg = "drbg_nopr_sha256",
4043                 .test = alg_test_drbg,
4044                 .fips_allowed = 1,
4045                 .suite = {
4046                         .drbg = __VECS(drbg_nopr_sha256_tv_template)
4047                 }
4048         }, {
4049                 /* covered by drbg_nopr_sha256 test */
4050                 .alg = "drbg_nopr_sha384",
4051                 .fips_allowed = 1,
4052                 .test = alg_test_null,
4053         }, {
4054                 .alg = "drbg_nopr_sha512",
4055                 .fips_allowed = 1,
4056                 .test = alg_test_null,
4057         }, {
4058                 .alg = "drbg_pr_ctr_aes128",
4059                 .test = alg_test_drbg,
4060                 .fips_allowed = 1,
4061                 .suite = {
4062                         .drbg = __VECS(drbg_pr_ctr_aes128_tv_template)
4063                 }
4064         }, {
4065                 /* covered by drbg_pr_ctr_aes128 test */
4066                 .alg = "drbg_pr_ctr_aes192",
4067                 .fips_allowed = 1,
4068                 .test = alg_test_null,
4069         }, {
4070                 .alg = "drbg_pr_ctr_aes256",
4071                 .fips_allowed = 1,
4072                 .test = alg_test_null,
4073         }, {
4074                 .alg = "drbg_pr_hmac_sha1",
4075                 .fips_allowed = 1,
4076                 .test = alg_test_null,
4077         }, {
4078                 .alg = "drbg_pr_hmac_sha256",
4079                 .test = alg_test_drbg,
4080                 .fips_allowed = 1,
4081                 .suite = {
4082                         .drbg = __VECS(drbg_pr_hmac_sha256_tv_template)
4083                 }
4084         }, {
4085                 /* covered by drbg_pr_hmac_sha256 test */
4086                 .alg = "drbg_pr_hmac_sha384",
4087                 .fips_allowed = 1,
4088                 .test = alg_test_null,
4089         }, {
4090                 .alg = "drbg_pr_hmac_sha512",
4091                 .test = alg_test_null,
4092                 .fips_allowed = 1,
4093         }, {
4094                 .alg = "drbg_pr_sha1",
4095                 .fips_allowed = 1,
4096                 .test = alg_test_null,
4097         }, {
4098                 .alg = "drbg_pr_sha256",
4099                 .test = alg_test_drbg,
4100                 .fips_allowed = 1,
4101                 .suite = {
4102                         .drbg = __VECS(drbg_pr_sha256_tv_template)
4103                 }
4104         }, {
4105                 /* covered by drbg_pr_sha256 test */
4106                 .alg = "drbg_pr_sha384",
4107                 .fips_allowed = 1,
4108                 .test = alg_test_null,
4109         }, {
4110                 .alg = "drbg_pr_sha512",
4111                 .fips_allowed = 1,
4112                 .test = alg_test_null,
4113         }, {
4114                 .alg = "ecb(aes)",
4115                 .test = alg_test_skcipher,
4116                 .fips_allowed = 1,
4117                 .suite = {
4118                         .cipher = __VECS(aes_tv_template)
4119                 }
4120         }, {
4121                 .alg = "ecb(anubis)",
4122                 .test = alg_test_skcipher,
4123                 .suite = {
4124                         .cipher = __VECS(anubis_tv_template)
4125                 }
4126         }, {
4127                 .alg = "ecb(arc4)",
4128                 .test = alg_test_skcipher,
4129                 .suite = {
4130                         .cipher = __VECS(arc4_tv_template)
4131                 }
4132         }, {
4133                 .alg = "ecb(blowfish)",
4134                 .test = alg_test_skcipher,
4135                 .suite = {
4136                         .cipher = __VECS(bf_tv_template)
4137                 }
4138         }, {
4139                 .alg = "ecb(camellia)",
4140                 .test = alg_test_skcipher,
4141                 .suite = {
4142                         .cipher = __VECS(camellia_tv_template)
4143                 }
4144         }, {
4145                 .alg = "ecb(cast5)",
4146                 .test = alg_test_skcipher,
4147                 .suite = {
4148                         .cipher = __VECS(cast5_tv_template)
4149                 }
4150         }, {
4151                 .alg = "ecb(cast6)",
4152                 .test = alg_test_skcipher,
4153                 .suite = {
4154                         .cipher = __VECS(cast6_tv_template)
4155                 }
4156         }, {
4157                 .alg = "ecb(cipher_null)",
4158                 .test = alg_test_null,
4159                 .fips_allowed = 1,
4160         }, {
4161                 .alg = "ecb(des)",
4162                 .test = alg_test_skcipher,
4163                 .suite = {
4164                         .cipher = __VECS(des_tv_template)
4165                 }
4166         }, {
4167                 .alg = "ecb(des3_ede)",
4168                 .test = alg_test_skcipher,
4169                 .fips_allowed = 1,
4170                 .suite = {
4171                         .cipher = __VECS(des3_ede_tv_template)
4172                 }
4173         }, {
4174                 .alg = "ecb(fcrypt)",
4175                 .test = alg_test_skcipher,
4176                 .suite = {
4177                         .cipher = {
4178                                 .vecs = fcrypt_pcbc_tv_template,
4179                                 .count = 1
4180                         }
4181                 }
4182         }, {
4183                 .alg = "ecb(khazad)",
4184                 .test = alg_test_skcipher,
4185                 .suite = {
4186                         .cipher = __VECS(khazad_tv_template)
4187                 }
4188         }, {
4189                 /* Same as ecb(aes) except the key is stored in
4190                  * hardware secure memory which we reference by index
4191                  */
4192                 .alg = "ecb(paes)",
4193                 .test = alg_test_null,
4194                 .fips_allowed = 1,
4195         }, {
4196                 .alg = "ecb(seed)",
4197                 .test = alg_test_skcipher,
4198                 .suite = {
4199                         .cipher = __VECS(seed_tv_template)
4200                 }
4201         }, {
4202                 .alg = "ecb(serpent)",
4203                 .test = alg_test_skcipher,
4204                 .suite = {
4205                         .cipher = __VECS(serpent_tv_template)
4206                 }
4207         }, {
4208                 .alg = "ecb(sm4)",
4209                 .test = alg_test_skcipher,
4210                 .suite = {
4211                         .cipher = __VECS(sm4_tv_template)
4212                 }
4213         }, {
4214                 .alg = "ecb(tea)",
4215                 .test = alg_test_skcipher,
4216                 .suite = {
4217                         .cipher = __VECS(tea_tv_template)
4218                 }
4219         }, {
4220                 .alg = "ecb(tnepres)",
4221                 .test = alg_test_skcipher,
4222                 .suite = {
4223                         .cipher = __VECS(tnepres_tv_template)
4224                 }
4225         }, {
4226                 .alg = "ecb(twofish)",
4227                 .test = alg_test_skcipher,
4228                 .suite = {
4229                         .cipher = __VECS(tf_tv_template)
4230                 }
4231         }, {
4232                 .alg = "ecb(xeta)",
4233                 .test = alg_test_skcipher,
4234                 .suite = {
4235                         .cipher = __VECS(xeta_tv_template)
4236                 }
4237         }, {
4238                 .alg = "ecb(xtea)",
4239                 .test = alg_test_skcipher,
4240                 .suite = {
4241                         .cipher = __VECS(xtea_tv_template)
4242                 }
4243         }, {
4244                 .alg = "ecdh",
4245                 .test = alg_test_kpp,
4246                 .fips_allowed = 1,
4247                 .suite = {
4248                         .kpp = __VECS(ecdh_tv_template)
4249                 }
4250         }, {
4251                 .alg = "ecrdsa",
4252                 .test = alg_test_akcipher,
4253                 .suite = {
4254                         .akcipher = __VECS(ecrdsa_tv_template)
4255                 }
4256         }, {
4257                 .alg = "gcm(aes)",
4258                 .generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)",
4259                 .test = alg_test_aead,
4260                 .fips_allowed = 1,
4261                 .suite = {
4262                         .aead = __VECS(aes_gcm_tv_template)
4263                 }
4264         }, {
4265                 .alg = "ghash",
4266                 .test = alg_test_hash,
4267                 .fips_allowed = 1,
4268                 .suite = {
4269                         .hash = __VECS(ghash_tv_template)
4270                 }
4271         }, {
4272                 .alg = "hmac(md5)",
4273                 .test = alg_test_hash,
4274                 .suite = {
4275                         .hash = __VECS(hmac_md5_tv_template)
4276                 }
4277         }, {
4278                 .alg = "hmac(rmd128)",
4279                 .test = alg_test_hash,
4280                 .suite = {
4281                         .hash = __VECS(hmac_rmd128_tv_template)
4282                 }
4283         }, {
4284                 .alg = "hmac(rmd160)",
4285                 .test = alg_test_hash,
4286                 .suite = {
4287                         .hash = __VECS(hmac_rmd160_tv_template)
4288                 }
4289         }, {
4290                 .alg = "hmac(sha1)",
4291                 .test = alg_test_hash,
4292                 .fips_allowed = 1,
4293                 .suite = {
4294                         .hash = __VECS(hmac_sha1_tv_template)
4295                 }
4296         }, {
4297                 .alg = "hmac(sha224)",
4298                 .test = alg_test_hash,
4299                 .fips_allowed = 1,
4300                 .suite = {
4301                         .hash = __VECS(hmac_sha224_tv_template)
4302                 }
4303         }, {
4304                 .alg = "hmac(sha256)",
4305                 .test = alg_test_hash,
4306                 .fips_allowed = 1,
4307                 .suite = {
4308                         .hash = __VECS(hmac_sha256_tv_template)
4309                 }
4310         }, {
4311                 .alg = "hmac(sha3-224)",
4312                 .test = alg_test_hash,
4313                 .fips_allowed = 1,
4314                 .suite = {
4315                         .hash = __VECS(hmac_sha3_224_tv_template)
4316                 }
4317         }, {
4318                 .alg = "hmac(sha3-256)",
4319                 .test = alg_test_hash,
4320                 .fips_allowed = 1,
4321                 .suite = {
4322                         .hash = __VECS(hmac_sha3_256_tv_template)
4323                 }
4324         }, {
4325                 .alg = "hmac(sha3-384)",
4326                 .test = alg_test_hash,
4327                 .fips_allowed = 1,
4328                 .suite = {
4329                         .hash = __VECS(hmac_sha3_384_tv_template)
4330                 }
4331         }, {
4332                 .alg = "hmac(sha3-512)",
4333                 .test = alg_test_hash,
4334                 .fips_allowed = 1,
4335                 .suite = {
4336                         .hash = __VECS(hmac_sha3_512_tv_template)
4337                 }
4338         }, {
4339                 .alg = "hmac(sha384)",
4340                 .test = alg_test_hash,
4341                 .fips_allowed = 1,
4342                 .suite = {
4343                         .hash = __VECS(hmac_sha384_tv_template)
4344                 }
4345         }, {
4346                 .alg = "hmac(sha512)",
4347                 .test = alg_test_hash,
4348                 .fips_allowed = 1,
4349                 .suite = {
4350                         .hash = __VECS(hmac_sha512_tv_template)
4351                 }
4352         }, {
4353                 .alg = "hmac(streebog256)",
4354                 .test = alg_test_hash,
4355                 .suite = {
4356                         .hash = __VECS(hmac_streebog256_tv_template)
4357                 }
4358         }, {
4359                 .alg = "hmac(streebog512)",
4360                 .test = alg_test_hash,
4361                 .suite = {
4362                         .hash = __VECS(hmac_streebog512_tv_template)
4363                 }
4364         }, {
4365                 .alg = "jitterentropy_rng",
4366                 .fips_allowed = 1,
4367                 .test = alg_test_null,
4368         }, {
4369                 .alg = "kw(aes)",
4370                 .test = alg_test_skcipher,
4371                 .fips_allowed = 1,
4372                 .suite = {
4373                         .cipher = __VECS(aes_kw_tv_template)
4374                 }
4375         }, {
4376                 .alg = "lrw(aes)",
4377                 .generic_driver = "lrw(ecb(aes-generic))",
4378                 .test = alg_test_skcipher,
4379                 .suite = {
4380                         .cipher = __VECS(aes_lrw_tv_template)
4381                 }
4382         }, {
4383                 .alg = "lrw(camellia)",
4384                 .generic_driver = "lrw(ecb(camellia-generic))",
4385                 .test = alg_test_skcipher,
4386                 .suite = {
4387                         .cipher = __VECS(camellia_lrw_tv_template)
4388                 }
4389         }, {
4390                 .alg = "lrw(cast6)",
4391                 .generic_driver = "lrw(ecb(cast6-generic))",
4392                 .test = alg_test_skcipher,
4393                 .suite = {
4394                         .cipher = __VECS(cast6_lrw_tv_template)
4395                 }
4396         }, {
4397                 .alg = "lrw(serpent)",
4398                 .generic_driver = "lrw(ecb(serpent-generic))",
4399                 .test = alg_test_skcipher,
4400                 .suite = {
4401                         .cipher = __VECS(serpent_lrw_tv_template)
4402                 }
4403         }, {
4404                 .alg = "lrw(twofish)",
4405                 .generic_driver = "lrw(ecb(twofish-generic))",
4406                 .test = alg_test_skcipher,
4407                 .suite = {
4408                         .cipher = __VECS(tf_lrw_tv_template)
4409                 }
4410         }, {
4411                 .alg = "lz4",
4412                 .test = alg_test_comp,
4413                 .fips_allowed = 1,
4414                 .suite = {
4415                         .comp = {
4416                                 .comp = __VECS(lz4_comp_tv_template),
4417                                 .decomp = __VECS(lz4_decomp_tv_template)
4418                         }
4419                 }
4420         }, {
4421                 .alg = "lz4hc",
4422                 .test = alg_test_comp,
4423                 .fips_allowed = 1,
4424                 .suite = {
4425                         .comp = {
4426                                 .comp = __VECS(lz4hc_comp_tv_template),
4427                                 .decomp = __VECS(lz4hc_decomp_tv_template)
4428                         }
4429                 }
4430         }, {
4431                 .alg = "lzo",
4432                 .test = alg_test_comp,
4433                 .fips_allowed = 1,
4434                 .suite = {
4435                         .comp = {
4436                                 .comp = __VECS(lzo_comp_tv_template),
4437                                 .decomp = __VECS(lzo_decomp_tv_template)
4438                         }
4439                 }
4440         }, {
4441                 .alg = "md4",
4442                 .test = alg_test_hash,
4443                 .suite = {
4444                         .hash = __VECS(md4_tv_template)
4445                 }
4446         }, {
4447                 .alg = "md5",
4448                 .test = alg_test_hash,
4449                 .suite = {
4450                         .hash = __VECS(md5_tv_template)
4451                 }
4452         }, {
4453                 .alg = "michael_mic",
4454                 .test = alg_test_hash,
4455                 .suite = {
4456                         .hash = __VECS(michael_mic_tv_template)
4457                 }
4458         }, {
4459                 .alg = "morus1280",
4460                 .test = alg_test_aead,
4461                 .suite = {
4462                         .aead = __VECS(morus1280_tv_template)
4463                 }
4464         }, {
4465                 .alg = "morus640",
4466                 .test = alg_test_aead,
4467                 .suite = {
4468                         .aead = __VECS(morus640_tv_template)
4469                 }
4470         }, {
4471                 .alg = "nhpoly1305",
4472                 .test = alg_test_hash,
4473                 .suite = {
4474                         .hash = __VECS(nhpoly1305_tv_template)
4475                 }
4476         }, {
4477                 .alg = "ofb(aes)",
4478                 .test = alg_test_skcipher,
4479                 .fips_allowed = 1,
4480                 .suite = {
4481                         .cipher = __VECS(aes_ofb_tv_template)
4482                 }
4483         }, {
4484                 /* Same as ofb(aes) except the key is stored in
4485                  * hardware secure memory which we reference by index
4486                  */
4487                 .alg = "ofb(paes)",
4488                 .test = alg_test_null,
4489                 .fips_allowed = 1,
4490         }, {
4491                 .alg = "pcbc(fcrypt)",
4492                 .test = alg_test_skcipher,
4493                 .suite = {
4494                         .cipher = __VECS(fcrypt_pcbc_tv_template)
4495                 }
4496         }, {
4497                 .alg = "pkcs1pad(rsa,sha224)",
4498                 .test = alg_test_null,
4499                 .fips_allowed = 1,
4500         }, {
4501                 .alg = "pkcs1pad(rsa,sha256)",
4502                 .test = alg_test_akcipher,
4503                 .fips_allowed = 1,
4504                 .suite = {
4505                         .akcipher = __VECS(pkcs1pad_rsa_tv_template)
4506                 }
4507         }, {
4508                 .alg = "pkcs1pad(rsa,sha384)",
4509                 .test = alg_test_null,
4510                 .fips_allowed = 1,
4511         }, {
4512                 .alg = "pkcs1pad(rsa,sha512)",
4513                 .test = alg_test_null,
4514                 .fips_allowed = 1,
4515         }, {
4516                 .alg = "poly1305",
4517                 .test = alg_test_hash,
4518                 .suite = {
4519                         .hash = __VECS(poly1305_tv_template)
4520                 }
4521         }, {
4522                 .alg = "rfc3686(ctr(aes))",
4523                 .test = alg_test_skcipher,
4524                 .fips_allowed = 1,
4525                 .suite = {
4526                         .cipher = __VECS(aes_ctr_rfc3686_tv_template)
4527                 }
4528         }, {
4529                 .alg = "rfc4106(gcm(aes))",
4530                 .generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))",
4531                 .test = alg_test_aead,
4532                 .fips_allowed = 1,
4533                 .suite = {
4534                         .aead = __VECS(aes_gcm_rfc4106_tv_template)
4535                 }
4536         }, {
4537                 .alg = "rfc4309(ccm(aes))",
4538                 .generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))",
4539                 .test = alg_test_aead,
4540                 .fips_allowed = 1,
4541                 .suite = {
4542                         .aead = __VECS(aes_ccm_rfc4309_tv_template)
4543                 }
4544         }, {
4545                 .alg = "rfc4543(gcm(aes))",
4546                 .generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))",
4547                 .test = alg_test_aead,
4548                 .suite = {
4549                         .aead = __VECS(aes_gcm_rfc4543_tv_template)
4550                 }
4551         }, {
4552                 .alg = "rfc7539(chacha20,poly1305)",
4553                 .test = alg_test_aead,
4554                 .suite = {
4555                         .aead = __VECS(rfc7539_tv_template)
4556                 }
4557         }, {
4558                 .alg = "rfc7539esp(chacha20,poly1305)",
4559                 .test = alg_test_aead,
4560                 .suite = {
4561                         .aead = __VECS(rfc7539esp_tv_template)
4562                 }
4563         }, {
4564                 .alg = "rmd128",
4565                 .test = alg_test_hash,
4566                 .suite = {
4567                         .hash = __VECS(rmd128_tv_template)
4568                 }
4569         }, {
4570                 .alg = "rmd160",
4571                 .test = alg_test_hash,
4572                 .suite = {
4573                         .hash = __VECS(rmd160_tv_template)
4574                 }
4575         }, {
4576                 .alg = "rmd256",
4577                 .test = alg_test_hash,
4578                 .suite = {
4579                         .hash = __VECS(rmd256_tv_template)
4580                 }
4581         }, {
4582                 .alg = "rmd320",
4583                 .test = alg_test_hash,
4584                 .suite = {
4585                         .hash = __VECS(rmd320_tv_template)
4586                 }
4587         }, {
4588                 .alg = "rsa",
4589                 .test = alg_test_akcipher,
4590                 .fips_allowed = 1,
4591                 .suite = {
4592                         .akcipher = __VECS(rsa_tv_template)
4593                 }
4594         }, {
4595                 .alg = "salsa20",
4596                 .test = alg_test_skcipher,
4597                 .suite = {
4598                         .cipher = __VECS(salsa20_stream_tv_template)
4599                 }
4600         }, {
4601                 .alg = "sha1",
4602                 .test = alg_test_hash,
4603                 .fips_allowed = 1,
4604                 .suite = {
4605                         .hash = __VECS(sha1_tv_template)
4606                 }
4607         }, {
4608                 .alg = "sha224",
4609                 .test = alg_test_hash,
4610                 .fips_allowed = 1,
4611                 .suite = {
4612                         .hash = __VECS(sha224_tv_template)
4613                 }
4614         }, {
4615                 .alg = "sha256",
4616                 .test = alg_test_hash,
4617                 .fips_allowed = 1,
4618                 .suite = {
4619                         .hash = __VECS(sha256_tv_template)
4620                 }
4621         }, {
4622                 .alg = "sha3-224",
4623                 .test = alg_test_hash,
4624                 .fips_allowed = 1,
4625                 .suite = {
4626                         .hash = __VECS(sha3_224_tv_template)
4627                 }
4628         }, {
4629                 .alg = "sha3-256",
4630                 .test = alg_test_hash,
4631                 .fips_allowed = 1,
4632                 .suite = {
4633                         .hash = __VECS(sha3_256_tv_template)
4634                 }
4635         }, {
4636                 .alg = "sha3-384",
4637                 .test = alg_test_hash,
4638                 .fips_allowed = 1,
4639                 .suite = {
4640                         .hash = __VECS(sha3_384_tv_template)
4641                 }
4642         }, {
4643                 .alg = "sha3-512",
4644                 .test = alg_test_hash,
4645                 .fips_allowed = 1,
4646                 .suite = {
4647                         .hash = __VECS(sha3_512_tv_template)
4648                 }
4649         }, {
4650                 .alg = "sha384",
4651                 .test = alg_test_hash,
4652                 .fips_allowed = 1,
4653                 .suite = {
4654                         .hash = __VECS(sha384_tv_template)
4655                 }
4656         }, {
4657                 .alg = "sha512",
4658                 .test = alg_test_hash,
4659                 .fips_allowed = 1,
4660                 .suite = {
4661                         .hash = __VECS(sha512_tv_template)
4662                 }
4663         }, {
4664                 .alg = "sm3",
4665                 .test = alg_test_hash,
4666                 .suite = {
4667                         .hash = __VECS(sm3_tv_template)
4668                 }
4669         }, {
4670                 .alg = "streebog256",
4671                 .test = alg_test_hash,
4672                 .suite = {
4673                         .hash = __VECS(streebog256_tv_template)
4674                 }
4675         }, {
4676                 .alg = "streebog512",
4677                 .test = alg_test_hash,
4678                 .suite = {
4679                         .hash = __VECS(streebog512_tv_template)
4680                 }
4681         }, {
4682                 .alg = "tgr128",
4683                 .test = alg_test_hash,
4684                 .suite = {
4685                         .hash = __VECS(tgr128_tv_template)
4686                 }
4687         }, {
4688                 .alg = "tgr160",
4689                 .test = alg_test_hash,
4690                 .suite = {
4691                         .hash = __VECS(tgr160_tv_template)
4692                 }
4693         }, {
4694                 .alg = "tgr192",
4695                 .test = alg_test_hash,
4696                 .suite = {
4697                         .hash = __VECS(tgr192_tv_template)
4698                 }
4699         }, {
4700                 .alg = "vmac64(aes)",
4701                 .test = alg_test_hash,
4702                 .suite = {
4703                         .hash = __VECS(vmac64_aes_tv_template)
4704                 }
4705         }, {
4706                 .alg = "wp256",
4707                 .test = alg_test_hash,
4708                 .suite = {
4709                         .hash = __VECS(wp256_tv_template)
4710                 }
4711         }, {
4712                 .alg = "wp384",
4713                 .test = alg_test_hash,
4714                 .suite = {
4715                         .hash = __VECS(wp384_tv_template)
4716                 }
4717         }, {
4718                 .alg = "wp512",
4719                 .test = alg_test_hash,
4720                 .suite = {
4721                         .hash = __VECS(wp512_tv_template)
4722                 }
4723         }, {
4724                 .alg = "xcbc(aes)",
4725                 .test = alg_test_hash,
4726                 .suite = {
4727                         .hash = __VECS(aes_xcbc128_tv_template)
4728                 }
4729         }, {
4730                 .alg = "xchacha12",
4731                 .test = alg_test_skcipher,
4732                 .suite = {
4733                         .cipher = __VECS(xchacha12_tv_template)
4734                 },
4735         }, {
4736                 .alg = "xchacha20",
4737                 .test = alg_test_skcipher,
4738                 .suite = {
4739                         .cipher = __VECS(xchacha20_tv_template)
4740                 },
4741         }, {
4742                 .alg = "xts(aes)",
4743                 .generic_driver = "xts(ecb(aes-generic))",
4744                 .test = alg_test_skcipher,
4745                 .fips_allowed = 1,
4746                 .suite = {
4747                         .cipher = __VECS(aes_xts_tv_template)
4748                 }
4749         }, {
4750                 .alg = "xts(camellia)",
4751                 .generic_driver = "xts(ecb(camellia-generic))",
4752                 .test = alg_test_skcipher,
4753                 .suite = {
4754                         .cipher = __VECS(camellia_xts_tv_template)
4755                 }
4756         }, {
4757                 .alg = "xts(cast6)",
4758                 .generic_driver = "xts(ecb(cast6-generic))",
4759                 .test = alg_test_skcipher,
4760                 .suite = {
4761                         .cipher = __VECS(cast6_xts_tv_template)
4762                 }
4763         }, {
4764                 /* Same as xts(aes) except the key is stored in
4765                  * hardware secure memory which we reference by index
4766                  */
4767                 .alg = "xts(paes)",
4768                 .test = alg_test_null,
4769                 .fips_allowed = 1,
4770         }, {
4771                 .alg = "xts(serpent)",
4772                 .generic_driver = "xts(ecb(serpent-generic))",
4773                 .test = alg_test_skcipher,
4774                 .suite = {
4775                         .cipher = __VECS(serpent_xts_tv_template)
4776                 }
4777         }, {
4778                 .alg = "xts(twofish)",
4779                 .generic_driver = "xts(ecb(twofish-generic))",
4780                 .test = alg_test_skcipher,
4781                 .suite = {
4782                         .cipher = __VECS(tf_xts_tv_template)
4783                 }
4784         }, {
4785                 .alg = "xts4096(paes)",
4786                 .test = alg_test_null,
4787                 .fips_allowed = 1,
4788         }, {
4789                 .alg = "xts512(paes)",
4790                 .test = alg_test_null,
4791                 .fips_allowed = 1,
4792         }, {
4793                 .alg = "zlib-deflate",
4794                 .test = alg_test_comp,
4795                 .fips_allowed = 1,
4796                 .suite = {
4797                         .comp = {
4798                                 .comp = __VECS(zlib_deflate_comp_tv_template),
4799                                 .decomp = __VECS(zlib_deflate_decomp_tv_template)
4800                         }
4801                 }
4802         }, {
4803                 .alg = "zstd",
4804                 .test = alg_test_comp,
4805                 .fips_allowed = 1,
4806                 .suite = {
4807                         .comp = {
4808                                 .comp = __VECS(zstd_comp_tv_template),
4809                                 .decomp = __VECS(zstd_decomp_tv_template)
4810                         }
4811                 }
4812         }
4813 };
4814
4815 static void alg_check_test_descs_order(void)
4816 {
4817         int i;
4818
4819         for (i = 1; i < ARRAY_SIZE(alg_test_descs); i++) {
4820                 int diff = strcmp(alg_test_descs[i - 1].alg,
4821                                   alg_test_descs[i].alg);
4822
4823                 if (WARN_ON(diff > 0)) {
4824                         pr_warn("testmgr: alg_test_descs entries in wrong order: '%s' before '%s'\n",
4825                                 alg_test_descs[i - 1].alg,
4826                                 alg_test_descs[i].alg);
4827                 }
4828
4829                 if (WARN_ON(diff == 0)) {
4830                         pr_warn("testmgr: duplicate alg_test_descs entry: '%s'\n",
4831                                 alg_test_descs[i].alg);
4832                 }
4833         }
4834 }
4835
4836 static void alg_check_testvec_configs(void)
4837 {
4838         int i;
4839
4840         for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++)
4841                 WARN_ON(!valid_testvec_config(
4842                                 &default_cipher_testvec_configs[i]));
4843
4844         for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++)
4845                 WARN_ON(!valid_testvec_config(
4846                                 &default_hash_testvec_configs[i]));
4847 }
4848
4849 static void testmgr_onetime_init(void)
4850 {
4851         alg_check_test_descs_order();
4852         alg_check_testvec_configs();
4853
4854 #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
4855         pr_warn("alg: extra crypto tests enabled.  This is intended for developer use only.\n");
4856 #endif
4857 }
4858
4859 static int alg_find_test(const char *alg)
4860 {
4861         int start = 0;
4862         int end = ARRAY_SIZE(alg_test_descs);
4863
4864         while (start < end) {
4865                 int i = (start + end) / 2;
4866                 int diff = strcmp(alg_test_descs[i].alg, alg);
4867
4868                 if (diff > 0) {
4869                         end = i;
4870                         continue;
4871                 }
4872
4873                 if (diff < 0) {
4874                         start = i + 1;
4875                         continue;
4876                 }
4877
4878                 return i;
4879         }
4880
4881         return -1;
4882 }
4883
4884 int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
4885 {
4886         int i;
4887         int j;
4888         int rc;
4889
4890         if (!fips_enabled && notests) {
4891                 printk_once(KERN_INFO "alg: self-tests disabled\n");
4892                 return 0;
4893         }
4894
4895         DO_ONCE(testmgr_onetime_init);
4896
4897         if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) {
4898                 char nalg[CRYPTO_MAX_ALG_NAME];
4899
4900                 if (snprintf(nalg, sizeof(nalg), "ecb(%s)", alg) >=
4901                     sizeof(nalg))
4902                         return -ENAMETOOLONG;
4903
4904                 i = alg_find_test(nalg);
4905                 if (i < 0)
4906                         goto notest;
4907
4908                 if (fips_enabled && !alg_test_descs[i].fips_allowed)
4909                         goto non_fips_alg;
4910
4911                 rc = alg_test_cipher(alg_test_descs + i, driver, type, mask);
4912                 goto test_done;
4913         }
4914
4915         i = alg_find_test(alg);
4916         j = alg_find_test(driver);
4917         if (i < 0 && j < 0)
4918                 goto notest;
4919
4920         if (fips_enabled && ((i >= 0 && !alg_test_descs[i].fips_allowed) ||
4921                              (j >= 0 && !alg_test_descs[j].fips_allowed)))
4922                 goto non_fips_alg;
4923
4924         rc = 0;
4925         if (i >= 0)
4926                 rc |= alg_test_descs[i].test(alg_test_descs + i, driver,
4927                                              type, mask);
4928         if (j >= 0 && j != i)
4929                 rc |= alg_test_descs[j].test(alg_test_descs + j, driver,
4930                                              type, mask);
4931
4932 test_done:
4933         if (rc && (fips_enabled || panic_on_fail))
4934                 panic("alg: self-tests for %s (%s) failed in %s mode!\n",
4935                       driver, alg, fips_enabled ? "fips" : "panic_on_fail");
4936
4937         if (fips_enabled && !rc)
4938                 pr_info("alg: self-tests for %s (%s) passed\n", driver, alg);
4939
4940         return rc;
4941
4942 notest:
4943         printk(KERN_INFO "alg: No test for %s (%s)\n", alg, driver);
4944         return 0;
4945 non_fips_alg:
4946         return -EINVAL;
4947 }
4948
4949 #endif /* CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */
4950
4951 EXPORT_SYMBOL_GPL(alg_test);