tools/testing/selftests/vm/mremap_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright 2020 Google LLC
   4  */
   5 #define _GNU_SOURCE
   6
   7 #include <errno.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <sys/mman.h>
  11 #include <time.h>
  12
  13 #include "../kselftest.h"
  14
  15 #define EXPECT_SUCCESS 0
  16 #define EXPECT_FAILURE 1
  17 #define NON_OVERLAPPING 0
  18 #define OVERLAPPING 1
  19 #define NS_PER_SEC 1000000000ULL
  20 #define VALIDATION_DEFAULT_THRESHOLD 4  /* 4MB */
  21 #define VALIDATION_NO_THRESHOLD 0       /* Verify the entire region */
  22
  23 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
  24 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
  25
  26 struct config {
  27         unsigned long long src_alignment;
  28         unsigned long long dest_alignment;
  29         unsigned long long region_size;
  30         int overlapping;
  31 };
  32
  33 struct test {
  34         const char *name;
  35         struct config config;
  36         int expect_failure;
  37 };
  38
  39 enum {
  40         _1KB = 1ULL << 10,      /* 1KB -> not page aligned */
  41         _4KB = 4ULL << 10,
  42         _8KB = 8ULL << 10,
  43         _1MB = 1ULL << 20,
  44         _2MB = 2ULL << 20,
  45         _4MB = 4ULL << 20,
  46         _1GB = 1ULL << 30,
  47         _2GB = 2ULL << 30,
  48         PTE = _4KB,
  49         PMD = _2MB,
  50         PUD = _1GB,
  51 };
  52
  53 #define MAKE_TEST(source_align, destination_align, size,        \
  54                   overlaps, should_fail, test_name)             \
  55 {                                                               \
  56         .name = test_name,                                      \
  57         .config = {                                             \
  58                 .src_alignment = source_align,                  \
  59                 .dest_alignment = destination_align,            \
  60                 .region_size = size,                            \
  61                 .overlapping = overlaps,                        \
  62         },                                                      \
  63         .expect_failure = should_fail                           \
  64 }
  65
  66 /*
  67  * Returns the start address of the mapping on success, else returns
  68  * NULL on failure.
  69  */
  70 static void *get_source_mapping(struct config c)
  71 {
  72         unsigned long long addr = 0ULL;
  73         void *src_addr = NULL;
  74 retry:
  75         addr += c.src_alignment;
  76         src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
  77                         MAP_FIXED | MAP_ANONYMOUS | MAP_SHARED, -1, 0);
  78         if (src_addr == MAP_FAILED) {
  79                 if (errno == EPERM)
  80                         goto retry;
  81                 goto error;
  82         }
  83         /*
  84          * Check that the address is aligned to the specified alignment.
  85          * Addresses which have alignments that are multiples of that
  86          * specified are not considered valid. For instance, 1GB address is
  87          * 2MB-aligned, however it will not be considered valid for a
  88          * requested alignment of 2MB. This is done to reduce coincidental
  89          * alignment in the tests.
  90          */
  91         if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
  92                         !((unsigned long long) src_addr & c.src_alignment))
  93                 goto retry;
  94
  95         if (!src_addr)
  96                 goto error;
  97
  98         return src_addr;
  99 error:
 100         ksft_print_msg("Failed to map source region: %s\n",
 101                         strerror(errno));
 102         return NULL;
 103 }
 104
 105 /* Returns the time taken for the remap on success else returns -1. */
 106 static long long remap_region(struct config c, unsigned int threshold_mb,
 107                               char pattern_seed)
 108 {
 109         void *addr, *src_addr, *dest_addr;
 110         unsigned long long i;
 111         struct timespec t_start = {0, 0}, t_end = {0, 0};
 112         long long  start_ns, end_ns, align_mask, ret, offset;
 113         unsigned long long threshold;
 114
 115         if (threshold_mb == VALIDATION_NO_THRESHOLD)
 116                 threshold = c.region_size;
 117         else
 118                 threshold = MIN(threshold_mb * _1MB, c.region_size);
 119
 120         src_addr = get_source_mapping(c);
 121         if (!src_addr) {
 122                 ret = -1;
 123                 goto out;
 124         }
 125
 126         /* Set byte pattern */
 127         srand(pattern_seed);
 128         for (i = 0; i < threshold; i++)
 129                 memset((char *) src_addr + i, (char) rand(), 1);
 130
 131         /* Mask to zero out lower bits of address for alignment */
 132         align_mask = ~(c.dest_alignment - 1);
 133         /* Offset of destination address from the end of the source region */
 134         offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
 135         addr = (void *) (((unsigned long long) src_addr + c.region_size
 136                           + offset) & align_mask);
 137
 138         /* See comment in get_source_mapping() */
 139         if (!((unsigned long long) addr & c.dest_alignment))
 140                 addr = (void *) ((unsigned long long) addr | c.dest_alignment);
 141
 142         clock_gettime(CLOCK_MONOTONIC, &t_start);
 143         dest_addr = mremap(src_addr, c.region_size, c.region_size,
 144                         MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
 145         clock_gettime(CLOCK_MONOTONIC, &t_end);
 146
 147         if (dest_addr == MAP_FAILED) {
 148                 ksft_print_msg("mremap failed: %s\n", strerror(errno));
 149                 ret = -1;
 150                 goto clean_up_src;
 151         }
 152
 153         /* Verify byte pattern after remapping */
 154         srand(pattern_seed);
 155         for (i = 0; i < threshold; i++) {
 156                 char c = (char) rand();
 157
 158                 if (((char *) dest_addr)[i] != c) {
 159                         ksft_print_msg("Data after remap doesn't match at offset %d\n",
 160                                        i);
 161                         ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
 162                                         ((char *) dest_addr)[i] & 0xff);
 163                         ret = -1;
 164                         goto clean_up_dest;
 165                 }
 166         }
 167
 168         start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
 169         end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
 170         ret = end_ns - start_ns;
 171
 172 /*
 173  * Since the destination address is specified using MREMAP_FIXED, subsequent
 174  * mremap will unmap any previous mapping at the address range specified by
 175  * dest_addr and region_size. This significantly affects the remap time of
 176  * subsequent tests. So we clean up mappings after each test.
 177  */
 178 clean_up_dest:
 179         munmap(dest_addr, c.region_size);
 180 clean_up_src:
 181         munmap(src_addr, c.region_size);
 182 out:
 183         return ret;
 184 }
 185
 186 static void run_mremap_test_case(struct test test_case, int *failures,
 187                                  unsigned int threshold_mb,
 188                                  unsigned int pattern_seed)
 189 {
 190         long long remap_time = remap_region(test_case.config, threshold_mb,
 191                                             pattern_seed);
 192
 193         if (remap_time < 0) {
 194                 if (test_case.expect_failure)
 195                         ksft_test_result_pass("%s\n\tExpected mremap failure\n",
 196                                               test_case.name);
 197                 else {
 198                         ksft_test_result_fail("%s\n", test_case.name);
 199                         *failures += 1;
 200                 }
 201         } else {
 202                 /*
 203                  * Comparing mremap time is only applicable if entire region
 204                  * was faulted in.
 205                  */
 206                 if (threshold_mb == VALIDATION_NO_THRESHOLD ||
 207                     test_case.config.region_size <= threshold_mb * _1MB)
 208                         ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
 209                                               test_case.name, remap_time);
 210                 else
 211                         ksft_test_result_pass("%s\n", test_case.name);
 212         }
 213 }
 214
 215 static void usage(const char *cmd)
 216 {
 217         fprintf(stderr,
 218                 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
 219                 "-t\t only validate threshold_mb of the remapped region\n"
 220                 "  \t if 0 is supplied no threshold is used; all tests\n"
 221                 "  \t are run and remapped regions validated fully.\n"
 222                 "  \t The default threshold used is 4MB.\n"
 223                 "-p\t provide a seed to generate the random pattern for\n"
 224                 "  \t validating the remapped region.\n", cmd);
 225 }
 226
 227 static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
 228                       unsigned int *pattern_seed)
 229 {
 230         const char *optstr = "t:p:";
 231         int opt;
 232
 233         while ((opt = getopt(argc, argv, optstr)) != -1) {
 234                 switch (opt) {
 235                 case 't':
 236                         *threshold_mb = atoi(optarg);
 237                         break;
 238                 case 'p':
 239                         *pattern_seed = atoi(optarg);
 240                         break;
 241                 default:
 242                         usage(argv[0]);
 243                         return -1;
 244                 }
 245         }
 246
 247         if (optind < argc) {
 248                 usage(argv[0]);
 249                 return -1;
 250         }
 251
 252         return 0;
 253 }
 254
 255 int main(int argc, char **argv)
 256 {
 257         int failures = 0;
 258         int i, run_perf_tests;
 259         unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
 260         unsigned int pattern_seed;
 261         time_t t;
 262
 263         pattern_seed = (unsigned int) time(&t);
 264
 265         if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
 266                 exit(EXIT_FAILURE);
 267
 268         ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
 269                        threshold_mb, pattern_seed);
 270
 271         struct test test_cases[] = {
 272                 /* Expected mremap failures */
 273                 MAKE_TEST(_4KB, _4KB, _4KB, OVERLAPPING, EXPECT_FAILURE,
 274                   "mremap - Source and Destination Regions Overlapping"),
 275                 MAKE_TEST(_4KB, _1KB, _4KB, NON_OVERLAPPING, EXPECT_FAILURE,
 276                   "mremap - Destination Address Misaligned (1KB-aligned)"),
 277                 MAKE_TEST(_1KB, _4KB, _4KB, NON_OVERLAPPING, EXPECT_FAILURE,
 278                   "mremap - Source Address Misaligned (1KB-aligned)"),
 279
 280                 /* Src addr PTE aligned */
 281                 MAKE_TEST(PTE, PTE, _8KB, NON_OVERLAPPING, EXPECT_SUCCESS,
 282                   "8KB mremap - Source PTE-aligned, Destination PTE-aligned"),
 283
 284                 /* Src addr 1MB aligned */
 285                 MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 286                   "2MB mremap - Source 1MB-aligned, Destination PTE-aligned"),
 287                 MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 288                   "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned"),
 289
 290                 /* Src addr PMD aligned */
 291                 MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 292                   "4MB mremap - Source PMD-aligned, Destination PTE-aligned"),
 293                 MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 294                   "4MB mremap - Source PMD-aligned, Destination 1MB-aligned"),
 295                 MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 296                   "4MB mremap - Source PMD-aligned, Destination PMD-aligned"),
 297
 298                 /* Src addr PUD aligned */
 299                 MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 300                   "2GB mremap - Source PUD-aligned, Destination PTE-aligned"),
 301                 MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 302                   "2GB mremap - Source PUD-aligned, Destination 1MB-aligned"),
 303                 MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 304                   "2GB mremap - Source PUD-aligned, Destination PMD-aligned"),
 305                 MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 306                   "2GB mremap - Source PUD-aligned, Destination PUD-aligned"),
 307         };
 308
 309         struct test perf_test_cases[] = {
 310                 /*
 311                  * mremap 1GB region - Page table level aligned time
 312                  * comparison.
 313                  */
 314                 MAKE_TEST(PTE, PTE, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 315                   "1GB mremap - Source PTE-aligned, Destination PTE-aligned"),
 316                 MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 317                   "1GB mremap - Source PMD-aligned, Destination PMD-aligned"),
 318                 MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 319                   "1GB mremap - Source PUD-aligned, Destination PUD-aligned"),
 320         };
 321
 322         run_perf_tests =  (threshold_mb == VALIDATION_NO_THRESHOLD) ||
 323                                 (threshold_mb * _1MB >= _1GB);
 324
 325         ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
 326                       ARRAY_SIZE(perf_test_cases) : 0));
 327
 328         for (i = 0; i < ARRAY_SIZE(test_cases); i++)
 329                 run_mremap_test_case(test_cases[i], &failures, threshold_mb,
 330                                      pattern_seed);
 331
 332         if (run_perf_tests) {
 333                 ksft_print_msg("\n%s\n",
 334                  "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
 335                 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
 336                         run_mremap_test_case(perf_test_cases[i], &failures,
 337                                              threshold_mb, pattern_seed);
 338         }
 339
 340         if (failures > 0)
 341                 ksft_exit_fail();
 342         else
 343                 ksft_exit_pass();
 344 }