tools/testing/selftests/vm/mremap_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright 2020 Google LLC
   4  */
   5 #define _GNU_SOURCE
   6
   7 #include <errno.h>
   8 #include <stdlib.h>
   9 #include <stdio.h>
  10 #include <string.h>
  11 #include <sys/mman.h>
  12 #include <time.h>
  13
  14 #include "../kselftest.h"
  15
  16 #define EXPECT_SUCCESS 0
  17 #define EXPECT_FAILURE 1
  18 #define NON_OVERLAPPING 0
  19 #define OVERLAPPING 1
  20 #define NS_PER_SEC 1000000000ULL
  21 #define VALIDATION_DEFAULT_THRESHOLD 4  /* 4MB */
  22 #define VALIDATION_NO_THRESHOLD 0       /* Verify the entire region */
  23
  24 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
  25
  26 struct config {
  27         unsigned long long src_alignment;
  28         unsigned long long dest_alignment;
  29         unsigned long long region_size;
  30         int overlapping;
  31 };
  32
  33 struct test {
  34         const char *name;
  35         struct config config;
  36         int expect_failure;
  37 };
  38
  39 enum {
  40         _1KB = 1ULL << 10,      /* 1KB -> not page aligned */
  41         _4KB = 4ULL << 10,
  42         _8KB = 8ULL << 10,
  43         _1MB = 1ULL << 20,
  44         _2MB = 2ULL << 20,
  45         _4MB = 4ULL << 20,
  46         _1GB = 1ULL << 30,
  47         _2GB = 2ULL << 30,
  48         PMD = _2MB,
  49         PUD = _1GB,
  50 };
  51
  52 #define PTE page_size
  53
  54 #define MAKE_TEST(source_align, destination_align, size,        \
  55                   overlaps, should_fail, test_name)             \
  56 (struct test){                                                  \
  57         .name = test_name,                                      \
  58         .config = {                                             \
  59                 .src_alignment = source_align,                  \
  60                 .dest_alignment = destination_align,            \
  61                 .region_size = size,                            \
  62                 .overlapping = overlaps,                        \
  63         },                                                      \
  64         .expect_failure = should_fail                           \
  65 }
  66
  67 /* Returns mmap_min_addr sysctl tunable from procfs */
  68 static unsigned long long get_mmap_min_addr(void)
  69 {
  70         FILE *fp;
  71         int n_matched;
  72         static unsigned long long addr;
  73
  74         if (addr)
  75                 return addr;
  76
  77         fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
  78         if (fp == NULL) {
  79                 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
  80                         strerror(errno));
  81                 exit(KSFT_SKIP);
  82         }
  83
  84         n_matched = fscanf(fp, "%llu", &addr);
  85         if (n_matched != 1) {
  86                 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
  87                         strerror(errno));
  88                 fclose(fp);
  89                 exit(KSFT_SKIP);
  90         }
  91
  92         fclose(fp);
  93         return addr;
  94 }
  95
  96 /*
  97  * Returns the start address of the mapping on success, else returns
  98  * NULL on failure.
  99  */
 100 static void *get_source_mapping(struct config c)
 101 {
 102         unsigned long long addr = 0ULL;
 103         void *src_addr = NULL;
 104         unsigned long long mmap_min_addr;
 105
 106         mmap_min_addr = get_mmap_min_addr();
 107
 108 retry:
 109         addr += c.src_alignment;
 110         if (addr < mmap_min_addr)
 111                 goto retry;
 112
 113         src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
 114                         MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
 115                         -1, 0);
 116         if (src_addr == MAP_FAILED) {
 117                 if (errno == EPERM || errno == EEXIST)
 118                         goto retry;
 119                 goto error;
 120         }
 121         /*
 122          * Check that the address is aligned to the specified alignment.
 123          * Addresses which have alignments that are multiples of that
 124          * specified are not considered valid. For instance, 1GB address is
 125          * 2MB-aligned, however it will not be considered valid for a
 126          * requested alignment of 2MB. This is done to reduce coincidental
 127          * alignment in the tests.
 128          */
 129         if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
 130                         !((unsigned long long) src_addr & c.src_alignment)) {
 131                 munmap(src_addr, c.region_size);
 132                 goto retry;
 133         }
 134
 135         if (!src_addr)
 136                 goto error;
 137
 138         return src_addr;
 139 error:
 140         ksft_print_msg("Failed to map source region: %s\n",
 141                         strerror(errno));
 142         return NULL;
 143 }
 144
 145 /* Returns the time taken for the remap on success else returns -1. */
 146 static long long remap_region(struct config c, unsigned int threshold_mb,
 147                               char pattern_seed)
 148 {
 149         void *addr, *src_addr, *dest_addr;
 150         unsigned long long i;
 151         struct timespec t_start = {0, 0}, t_end = {0, 0};
 152         long long  start_ns, end_ns, align_mask, ret, offset;
 153         unsigned long long threshold;
 154
 155         if (threshold_mb == VALIDATION_NO_THRESHOLD)
 156                 threshold = c.region_size;
 157         else
 158                 threshold = MIN(threshold_mb * _1MB, c.region_size);
 159
 160         src_addr = get_source_mapping(c);
 161         if (!src_addr) {
 162                 ret = -1;
 163                 goto out;
 164         }
 165
 166         /* Set byte pattern */
 167         srand(pattern_seed);
 168         for (i = 0; i < threshold; i++)
 169                 memset((char *) src_addr + i, (char) rand(), 1);
 170
 171         /* Mask to zero out lower bits of address for alignment */
 172         align_mask = ~(c.dest_alignment - 1);
 173         /* Offset of destination address from the end of the source region */
 174         offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
 175         addr = (void *) (((unsigned long long) src_addr + c.region_size
 176                           + offset) & align_mask);
 177
 178         /* See comment in get_source_mapping() */
 179         if (!((unsigned long long) addr & c.dest_alignment))
 180                 addr = (void *) ((unsigned long long) addr | c.dest_alignment);
 181
 182         clock_gettime(CLOCK_MONOTONIC, &t_start);
 183         dest_addr = mremap(src_addr, c.region_size, c.region_size,
 184                         MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
 185         clock_gettime(CLOCK_MONOTONIC, &t_end);
 186
 187         if (dest_addr == MAP_FAILED) {
 188                 ksft_print_msg("mremap failed: %s\n", strerror(errno));
 189                 ret = -1;
 190                 goto clean_up_src;
 191         }
 192
 193         /* Verify byte pattern after remapping */
 194         srand(pattern_seed);
 195         for (i = 0; i < threshold; i++) {
 196                 char c = (char) rand();
 197
 198                 if (((char *) dest_addr)[i] != c) {
 199                         ksft_print_msg("Data after remap doesn't match at offset %d\n",
 200                                        i);
 201                         ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
 202                                         ((char *) dest_addr)[i] & 0xff);
 203                         ret = -1;
 204                         goto clean_up_dest;
 205                 }
 206         }
 207
 208         start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
 209         end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
 210         ret = end_ns - start_ns;
 211
 212 /*
 213  * Since the destination address is specified using MREMAP_FIXED, subsequent
 214  * mremap will unmap any previous mapping at the address range specified by
 215  * dest_addr and region_size. This significantly affects the remap time of
 216  * subsequent tests. So we clean up mappings after each test.
 217  */
 218 clean_up_dest:
 219         munmap(dest_addr, c.region_size);
 220 clean_up_src:
 221         munmap(src_addr, c.region_size);
 222 out:
 223         return ret;
 224 }
 225
 226 static void run_mremap_test_case(struct test test_case, int *failures,
 227                                  unsigned int threshold_mb,
 228                                  unsigned int pattern_seed)
 229 {
 230         long long remap_time = remap_region(test_case.config, threshold_mb,
 231                                             pattern_seed);
 232
 233         if (remap_time < 0) {
 234                 if (test_case.expect_failure)
 235                         ksft_test_result_pass("%s\n\tExpected mremap failure\n",
 236                                               test_case.name);
 237                 else {
 238                         ksft_test_result_fail("%s\n", test_case.name);
 239                         *failures += 1;
 240                 }
 241         } else {
 242                 /*
 243                  * Comparing mremap time is only applicable if entire region
 244                  * was faulted in.
 245                  */
 246                 if (threshold_mb == VALIDATION_NO_THRESHOLD ||
 247                     test_case.config.region_size <= threshold_mb * _1MB)
 248                         ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
 249                                               test_case.name, remap_time);
 250                 else
 251                         ksft_test_result_pass("%s\n", test_case.name);
 252         }
 253 }
 254
 255 static void usage(const char *cmd)
 256 {
 257         fprintf(stderr,
 258                 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
 259                 "-t\t only validate threshold_mb of the remapped region\n"
 260                 "  \t if 0 is supplied no threshold is used; all tests\n"
 261                 "  \t are run and remapped regions validated fully.\n"
 262                 "  \t The default threshold used is 4MB.\n"
 263                 "-p\t provide a seed to generate the random pattern for\n"
 264                 "  \t validating the remapped region.\n", cmd);
 265 }
 266
 267 static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
 268                       unsigned int *pattern_seed)
 269 {
 270         const char *optstr = "t:p:";
 271         int opt;
 272
 273         while ((opt = getopt(argc, argv, optstr)) != -1) {
 274                 switch (opt) {
 275                 case 't':
 276                         *threshold_mb = atoi(optarg);
 277                         break;
 278                 case 'p':
 279                         *pattern_seed = atoi(optarg);
 280                         break;
 281                 default:
 282                         usage(argv[0]);
 283                         return -1;
 284                 }
 285         }
 286
 287         if (optind < argc) {
 288                 usage(argv[0]);
 289                 return -1;
 290         }
 291
 292         return 0;
 293 }
 294
 295 #define MAX_TEST 13
 296 #define MAX_PERF_TEST 3
 297 int main(int argc, char **argv)
 298 {
 299         int failures = 0;
 300         int i, run_perf_tests;
 301         unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
 302         unsigned int pattern_seed;
 303         struct test test_cases[MAX_TEST];
 304         struct test perf_test_cases[MAX_PERF_TEST];
 305         int page_size;
 306         time_t t;
 307
 308         pattern_seed = (unsigned int) time(&t);
 309
 310         if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
 311                 exit(EXIT_FAILURE);
 312
 313         ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
 314                        threshold_mb, pattern_seed);
 315
 316         page_size = sysconf(_SC_PAGESIZE);
 317
 318         /* Expected mremap failures */
 319         test_cases[0] = MAKE_TEST(page_size, page_size, page_size,
 320                                   OVERLAPPING, EXPECT_FAILURE,
 321                                   "mremap - Source and Destination Regions Overlapping");
 322
 323         test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
 324                                   NON_OVERLAPPING, EXPECT_FAILURE,
 325                                   "mremap - Destination Address Misaligned (1KB-aligned)");
 326         test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
 327                                   NON_OVERLAPPING, EXPECT_FAILURE,
 328                                   "mremap - Source Address Misaligned (1KB-aligned)");
 329
 330         /* Src addr PTE aligned */
 331         test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
 332                                   NON_OVERLAPPING, EXPECT_SUCCESS,
 333                                   "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
 334
 335         /* Src addr 1MB aligned */
 336         test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 337                                   "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
 338         test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 339                                   "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
 340
 341         /* Src addr PMD aligned */
 342         test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 343                                   "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
 344         test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 345                                   "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
 346         test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 347                                   "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
 348
 349         /* Src addr PUD aligned */
 350         test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 351                                   "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
 352         test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 353                                    "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
 354         test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 355                                    "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
 356         test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 357                                    "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
 358
 359         perf_test_cases[0] =  MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 360                                         "1GB mremap - Source PTE-aligned, Destination PTE-aligned");
 361         /*
 362          * mremap 1GB region - Page table level aligned time
 363          * comparison.
 364          */
 365         perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 366                                        "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
 367         perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 368                                        "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
 369
 370         run_perf_tests =  (threshold_mb == VALIDATION_NO_THRESHOLD) ||
 371                                 (threshold_mb * _1MB >= _1GB);
 372
 373         ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
 374                       ARRAY_SIZE(perf_test_cases) : 0));
 375
 376         for (i = 0; i < ARRAY_SIZE(test_cases); i++)
 377                 run_mremap_test_case(test_cases[i], &failures, threshold_mb,
 378                                      pattern_seed);
 379
 380         if (run_perf_tests) {
 381                 ksft_print_msg("\n%s\n",
 382                  "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
 383                 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
 384                         run_mremap_test_case(perf_test_cases[i], &failures,
 385                                              threshold_mb, pattern_seed);
 386         }
 387
 388         if (failures > 0)
 389                 ksft_exit_fail();
 390         else
 391                 ksft_exit_pass();
 392 }