tools/testing/selftests/vm/userfaultfd.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Stress userfaultfd syscall.
   4  *
   5  *  Copyright (C) 2015  Red Hat, Inc.
   6  *
   7  * This test allocates two virtual areas and bounces the physical
   8  * memory across the two virtual areas (from area_src to area_dst)
   9  * using userfaultfd.
  10  *
  11  * There are three threads running per CPU:
  12  *
  13  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
  14  *    page of the area_dst (while the physical page may still be in
  15  *    area_src), and increments a per-page counter in the same page,
  16  *    and checks its value against a verification region.
  17  *
  18  * 2) another per-CPU thread handles the userfaults generated by
  19  *    thread 1 above. userfaultfd blocking reads or poll() modes are
  20  *    exercised interleaved.
  21  *
  22  * 3) one last per-CPU thread transfers the memory in the background
  23  *    at maximum bandwidth (if not already transferred by thread
  24  *    2). Each cpu thread takes cares of transferring a portion of the
  25  *    area.
  26  *
  27  * When all threads of type 3 completed the transfer, one bounce is
  28  * complete. area_src and area_dst are then swapped. All threads are
  29  * respawned and so the bounce is immediately restarted in the
  30  * opposite direction.
  31  *
  32  * per-CPU threads 1 by triggering userfaults inside
  33  * pthread_mutex_lock will also verify the atomicity of the memory
  34  * transfer (UFFDIO_COPY).
  35  */
  36
  37 #define _GNU_SOURCE
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <unistd.h>
  41 #include <stdlib.h>
  42 #include <sys/types.h>
  43 #include <sys/stat.h>
  44 #include <fcntl.h>
  45 #include <time.h>
  46 #include <signal.h>
  47 #include <poll.h>
  48 #include <string.h>
  49 #include <sys/mman.h>
  50 #include <sys/syscall.h>
  51 #include <sys/ioctl.h>
  52 #include <sys/wait.h>
  53 #include <pthread.h>
  54 #include <linux/userfaultfd.h>
  55 #include <setjmp.h>
  56 #include <stdbool.h>
  57 #include <assert.h>
  58 #include <inttypes.h>
  59 #include <stdint.h>
  60
  61 #include "../kselftest.h"
  62
  63 #ifdef __NR_userfaultfd
  64
  65 static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
  66
  67 #define BOUNCE_RANDOM           (1<<0)
  68 #define BOUNCE_RACINGFAULTS     (1<<1)
  69 #define BOUNCE_VERIFY           (1<<2)
  70 #define BOUNCE_POLL             (1<<3)
  71 static int bounces;
  72
  73 #define TEST_ANON       1
  74 #define TEST_HUGETLB    2
  75 #define TEST_SHMEM      3
  76 static int test_type;
  77
  78 /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
  79 #define ALARM_INTERVAL_SECS 10
  80 static volatile bool test_uffdio_copy_eexist = true;
  81 static volatile bool test_uffdio_zeropage_eexist = true;
  82 /* Whether to test uffd write-protection */
  83 static bool test_uffdio_wp = false;
  84 /* Whether to test uffd minor faults */
  85 static bool test_uffdio_minor = false;
  86
  87 static bool map_shared;
  88 static int shm_fd;
  89 static int huge_fd;
  90 static char *huge_fd_off0;
  91 static unsigned long long *count_verify;
  92 static int uffd = -1;
  93 static int uffd_flags, finished, *pipefd;
  94 static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;
  95 static char *zeropage;
  96 pthread_attr_t attr;
  97
  98 /* Userfaultfd test statistics */
  99 struct uffd_stats {
 100         int cpu;
 101         unsigned long missing_faults;
 102         unsigned long wp_faults;
 103         unsigned long minor_faults;
 104 };
 105
 106 /* pthread_mutex_t starts at page offset 0 */
 107 #define area_mutex(___area, ___nr)                                      \
 108         ((pthread_mutex_t *) ((___area) + (___nr)*page_size))
 109 /*
 110  * count is placed in the page after pthread_mutex_t naturally aligned
 111  * to avoid non alignment faults on non-x86 archs.
 112  */
 113 #define area_count(___area, ___nr)                                      \
 114         ((volatile unsigned long long *) ((unsigned long)               \
 115                                  ((___area) + (___nr)*page_size +       \
 116                                   sizeof(pthread_mutex_t) +             \
 117                                   sizeof(unsigned long long) - 1) &     \
 118                                  ~(unsigned long)(sizeof(unsigned long long) \
 119                                                   -  1)))
 120
 121 const char *examples =
 122     "# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
 123     "./userfaultfd anon 100 99999\n\n"
 124     "# Run share memory test on 1GiB region with 99 bounces:\n"
 125     "./userfaultfd shmem 1000 99\n\n"
 126     "# Run hugetlb memory test on 256MiB region with 50 bounces (using /dev/hugepages/hugefile):\n"
 127     "./userfaultfd hugetlb 256 50 /dev/hugepages/hugefile\n\n"
 128     "# Run the same hugetlb test but using shmem:\n"
 129     "./userfaultfd hugetlb_shared 256 50 /dev/hugepages/hugefile\n\n"
 130     "# 10MiB-~6GiB 999 bounces anonymous test, "
 131     "continue forever unless an error triggers\n"
 132     "while ./userfaultfd anon $[RANDOM % 6000 + 10] 999; do true; done\n\n";
 133
 134 static void usage(void)
 135 {
 136         fprintf(stderr, "\nUsage: ./userfaultfd <test type> <MiB> <bounces> "
 137                 "[hugetlbfs_file]\n\n");
 138         fprintf(stderr, "Supported <test type>: anon, hugetlb, "
 139                 "hugetlb_shared, shmem\n\n");
 140         fprintf(stderr, "Examples:\n\n");
 141         fprintf(stderr, "%s", examples);
 142         exit(1);
 143 }
 144
 145 #define _err(fmt, ...)                                          \
 146         do {                                                    \
 147                 int ret = errno;                                \
 148                 fprintf(stderr, "ERROR: " fmt, ##__VA_ARGS__);  \
 149                 fprintf(stderr, " (errno=%d, line=%d)\n",       \
 150                         ret, __LINE__);                         \
 151         } while (0)
 152
 153 #define err(fmt, ...)                           \
 154         do {                                    \
 155                 _err(fmt, ##__VA_ARGS__);       \
 156                 exit(1);                        \
 157         } while (0)
 158
 159 static void uffd_stats_reset(struct uffd_stats *uffd_stats,
 160                              unsigned long n_cpus)
 161 {
 162         int i;
 163
 164         for (i = 0; i < n_cpus; i++) {
 165                 uffd_stats[i].cpu = i;
 166                 uffd_stats[i].missing_faults = 0;
 167                 uffd_stats[i].wp_faults = 0;
 168                 uffd_stats[i].minor_faults = 0;
 169         }
 170 }
 171
 172 static void uffd_stats_report(struct uffd_stats *stats, int n_cpus)
 173 {
 174         int i;
 175         unsigned long long miss_total = 0, wp_total = 0, minor_total = 0;
 176
 177         for (i = 0; i < n_cpus; i++) {
 178                 miss_total += stats[i].missing_faults;
 179                 wp_total += stats[i].wp_faults;
 180                 minor_total += stats[i].minor_faults;
 181         }
 182
 183         printf("userfaults: ");
 184         if (miss_total) {
 185                 printf("%llu missing (", miss_total);
 186                 for (i = 0; i < n_cpus; i++)
 187                         printf("%lu+", stats[i].missing_faults);
 188                 printf("\b) ");
 189         }
 190         if (wp_total) {
 191                 printf("%llu wp (", wp_total);
 192                 for (i = 0; i < n_cpus; i++)
 193                         printf("%lu+", stats[i].wp_faults);
 194                 printf("\b) ");
 195         }
 196         if (minor_total) {
 197                 printf("%llu minor (", minor_total);
 198                 for (i = 0; i < n_cpus; i++)
 199                         printf("%lu+", stats[i].minor_faults);
 200                 printf("\b)");
 201         }
 202         printf("\n");
 203 }
 204
 205 static void anon_release_pages(char *rel_area)
 206 {
 207         if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED))
 208                 err("madvise(MADV_DONTNEED) failed");
 209 }
 210
 211 static void anon_allocate_area(void **alloc_area)
 212 {
 213         if (posix_memalign(alloc_area, page_size, nr_pages * page_size))
 214                 err("posix_memalign() failed");
 215 }
 216
 217 static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
 218 {
 219 }
 220
 221 static void hugetlb_release_pages(char *rel_area)
 222 {
 223         if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
 224                       rel_area == huge_fd_off0 ? 0 : nr_pages * page_size,
 225                       nr_pages * page_size))
 226                 err("fallocate() failed");
 227 }
 228
 229 static void hugetlb_allocate_area(void **alloc_area)
 230 {
 231         void *area_alias = NULL;
 232         char **alloc_area_alias;
 233
 234         *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
 235                            (map_shared ? MAP_SHARED : MAP_PRIVATE) |
 236                            MAP_HUGETLB,
 237                            huge_fd, *alloc_area == area_src ? 0 :
 238                            nr_pages * page_size);
 239         if (*alloc_area == MAP_FAILED)
 240                 err("mmap of hugetlbfs file failed");
 241
 242         if (map_shared) {
 243                 area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
 244                                   MAP_SHARED | MAP_HUGETLB,
 245                                   huge_fd, *alloc_area == area_src ? 0 :
 246                                   nr_pages * page_size);
 247                 if (area_alias == MAP_FAILED)
 248                         err("mmap of hugetlb file alias failed");
 249         }
 250
 251         if (*alloc_area == area_src) {
 252                 huge_fd_off0 = *alloc_area;
 253                 alloc_area_alias = &area_src_alias;
 254         } else {
 255                 alloc_area_alias = &area_dst_alias;
 256         }
 257         if (area_alias)
 258                 *alloc_area_alias = area_alias;
 259 }
 260
 261 static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset)
 262 {
 263         if (!map_shared)
 264                 return;
 265         /*
 266          * We can't zap just the pagetable with hugetlbfs because
 267          * MADV_DONTEED won't work. So exercise -EEXIST on a alias
 268          * mapping where the pagetables are not established initially,
 269          * this way we'll exercise the -EEXEC at the fs level.
 270          */
 271         *start = (unsigned long) area_dst_alias + offset;
 272 }
 273
 274 static void shmem_release_pages(char *rel_area)
 275 {
 276         if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE))
 277                 err("madvise(MADV_REMOVE) failed");
 278 }
 279
 280 static void shmem_allocate_area(void **alloc_area)
 281 {
 282         void *area_alias = NULL;
 283         bool is_src = alloc_area == (void **)&area_src;
 284         unsigned long offset = is_src ? 0 : nr_pages * page_size;
 285
 286         *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
 287                            MAP_SHARED, shm_fd, offset);
 288         if (*alloc_area == MAP_FAILED)
 289                 err("mmap of memfd failed");
 290
 291         area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
 292                           MAP_SHARED, shm_fd, offset);
 293         if (area_alias == MAP_FAILED)
 294                 err("mmap of memfd alias failed");
 295
 296         if (is_src)
 297                 area_src_alias = area_alias;
 298         else
 299                 area_dst_alias = area_alias;
 300 }
 301
 302 static void shmem_alias_mapping(__u64 *start, size_t len, unsigned long offset)
 303 {
 304         *start = (unsigned long)area_dst_alias + offset;
 305 }
 306
 307 struct uffd_test_ops {
 308         unsigned long expected_ioctls;
 309         void (*allocate_area)(void **alloc_area);
 310         void (*release_pages)(char *rel_area);
 311         void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
 312 };
 313
 314 #define SHMEM_EXPECTED_IOCTLS           ((1 << _UFFDIO_WAKE) | \
 315                                          (1 << _UFFDIO_COPY) | \
 316                                          (1 << _UFFDIO_ZEROPAGE))
 317
 318 #define ANON_EXPECTED_IOCTLS            ((1 << _UFFDIO_WAKE) | \
 319                                          (1 << _UFFDIO_COPY) | \
 320                                          (1 << _UFFDIO_ZEROPAGE) | \
 321                                          (1 << _UFFDIO_WRITEPROTECT))
 322
 323 static struct uffd_test_ops anon_uffd_test_ops = {
 324         .expected_ioctls = ANON_EXPECTED_IOCTLS,
 325         .allocate_area  = anon_allocate_area,
 326         .release_pages  = anon_release_pages,
 327         .alias_mapping = noop_alias_mapping,
 328 };
 329
 330 static struct uffd_test_ops shmem_uffd_test_ops = {
 331         .expected_ioctls = SHMEM_EXPECTED_IOCTLS,
 332         .allocate_area  = shmem_allocate_area,
 333         .release_pages  = shmem_release_pages,
 334         .alias_mapping = shmem_alias_mapping,
 335 };
 336
 337 static struct uffd_test_ops hugetlb_uffd_test_ops = {
 338         .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC & ~(1 << _UFFDIO_CONTINUE),
 339         .allocate_area  = hugetlb_allocate_area,
 340         .release_pages  = hugetlb_release_pages,
 341         .alias_mapping = hugetlb_alias_mapping,
 342 };
 343
 344 static struct uffd_test_ops *uffd_test_ops;
 345
 346 static void userfaultfd_open(uint64_t *features)
 347 {
 348         struct uffdio_api uffdio_api;
 349
 350         uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
 351         if (uffd < 0)
 352                 err("userfaultfd syscall not available in this kernel");
 353         uffd_flags = fcntl(uffd, F_GETFD, NULL);
 354
 355         uffdio_api.api = UFFD_API;
 356         uffdio_api.features = *features;
 357         if (ioctl(uffd, UFFDIO_API, &uffdio_api))
 358                 err("UFFDIO_API failed.\nPlease make sure to "
 359                     "run with either root or ptrace capability.");
 360         if (uffdio_api.api != UFFD_API)
 361                 err("UFFDIO_API error: %" PRIu64, (uint64_t)uffdio_api.api);
 362
 363         *features = uffdio_api.features;
 364 }
 365
 366 static inline void munmap_area(void **area)
 367 {
 368         if (*area)
 369                 if (munmap(*area, nr_pages * page_size))
 370                         err("munmap");
 371
 372         *area = NULL;
 373 }
 374
 375 static void uffd_test_ctx_clear(void)
 376 {
 377         size_t i;
 378
 379         if (pipefd) {
 380                 for (i = 0; i < nr_cpus * 2; ++i) {
 381                         if (close(pipefd[i]))
 382                                 err("close pipefd");
 383                 }
 384                 free(pipefd);
 385                 pipefd = NULL;
 386         }
 387
 388         if (count_verify) {
 389                 free(count_verify);
 390                 count_verify = NULL;
 391         }
 392
 393         if (uffd != -1) {
 394                 if (close(uffd))
 395                         err("close uffd");
 396                 uffd = -1;
 397         }
 398
 399         huge_fd_off0 = NULL;
 400         munmap_area((void **)&area_src);
 401         munmap_area((void **)&area_src_alias);
 402         munmap_area((void **)&area_dst);
 403         munmap_area((void **)&area_dst_alias);
 404 }
 405
 406 static void uffd_test_ctx_init_ext(uint64_t *features)
 407 {
 408         unsigned long nr, cpu;
 409
 410         uffd_test_ctx_clear();
 411
 412         uffd_test_ops->allocate_area((void **)&area_src);
 413         uffd_test_ops->allocate_area((void **)&area_dst);
 414
 415         uffd_test_ops->release_pages(area_src);
 416         uffd_test_ops->release_pages(area_dst);
 417
 418         userfaultfd_open(features);
 419
 420         count_verify = malloc(nr_pages * sizeof(unsigned long long));
 421         if (!count_verify)
 422                 err("count_verify");
 423
 424         for (nr = 0; nr < nr_pages; nr++) {
 425                 *area_mutex(area_src, nr) =
 426                         (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
 427                 count_verify[nr] = *area_count(area_src, nr) = 1;
 428                 /*
 429                  * In the transition between 255 to 256, powerpc will
 430                  * read out of order in my_bcmp and see both bytes as
 431                  * zero, so leave a placeholder below always non-zero
 432                  * after the count, to avoid my_bcmp to trigger false
 433                  * positives.
 434                  */
 435                 *(area_count(area_src, nr) + 1) = 1;
 436         }
 437
 438         pipefd = malloc(sizeof(int) * nr_cpus * 2);
 439         if (!pipefd)
 440                 err("pipefd");
 441         for (cpu = 0; cpu < nr_cpus; cpu++)
 442                 if (pipe2(&pipefd[cpu * 2], O_CLOEXEC | O_NONBLOCK))
 443                         err("pipe");
 444 }
 445
 446 static inline void uffd_test_ctx_init(uint64_t features)
 447 {
 448         uffd_test_ctx_init_ext(&features);
 449 }
 450
 451 static int my_bcmp(char *str1, char *str2, size_t n)
 452 {
 453         unsigned long i;
 454         for (i = 0; i < n; i++)
 455                 if (str1[i] != str2[i])
 456                         return 1;
 457         return 0;
 458 }
 459
 460 static void wp_range(int ufd, __u64 start, __u64 len, bool wp)
 461 {
 462         struct uffdio_writeprotect prms;
 463
 464         /* Write protection page faults */
 465         prms.range.start = start;
 466         prms.range.len = len;
 467         /* Undo write-protect, do wakeup after that */
 468         prms.mode = wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0;
 469
 470         if (ioctl(ufd, UFFDIO_WRITEPROTECT, &prms))
 471                 err("clear WP failed: address=0x%"PRIx64, (uint64_t)start);
 472 }
 473
 474 static void continue_range(int ufd, __u64 start, __u64 len)
 475 {
 476         struct uffdio_continue req;
 477
 478         req.range.start = start;
 479         req.range.len = len;
 480         req.mode = 0;
 481
 482         if (ioctl(ufd, UFFDIO_CONTINUE, &req))
 483                 err("UFFDIO_CONTINUE failed for address 0x%" PRIx64,
 484                     (uint64_t)start);
 485 }
 486
 487 static void *locking_thread(void *arg)
 488 {
 489         unsigned long cpu = (unsigned long) arg;
 490         struct random_data rand;
 491         unsigned long page_nr = *(&(page_nr)); /* uninitialized warning */
 492         int32_t rand_nr;
 493         unsigned long long count;
 494         char randstate[64];
 495         unsigned int seed;
 496
 497         if (bounces & BOUNCE_RANDOM) {
 498                 seed = (unsigned int) time(NULL) - bounces;
 499                 if (!(bounces & BOUNCE_RACINGFAULTS))
 500                         seed += cpu;
 501                 bzero(&rand, sizeof(rand));
 502                 bzero(&randstate, sizeof(randstate));
 503                 if (initstate_r(seed, randstate, sizeof(randstate), &rand))
 504                         err("initstate_r failed");
 505         } else {
 506                 page_nr = -bounces;
 507                 if (!(bounces & BOUNCE_RACINGFAULTS))
 508                         page_nr += cpu * nr_pages_per_cpu;
 509         }
 510
 511         while (!finished) {
 512                 if (bounces & BOUNCE_RANDOM) {
 513                         if (random_r(&rand, &rand_nr))
 514                                 err("random_r failed");
 515                         page_nr = rand_nr;
 516                         if (sizeof(page_nr) > sizeof(rand_nr)) {
 517                                 if (random_r(&rand, &rand_nr))
 518                                         err("random_r failed");
 519                                 page_nr |= (((unsigned long) rand_nr) << 16) <<
 520                                            16;
 521                         }
 522                 } else
 523                         page_nr += 1;
 524                 page_nr %= nr_pages;
 525                 pthread_mutex_lock(area_mutex(area_dst, page_nr));
 526                 count = *area_count(area_dst, page_nr);
 527                 if (count != count_verify[page_nr])
 528                         err("page_nr %lu memory corruption %llu %llu",
 529                             page_nr, count, count_verify[page_nr]);
 530                 count++;
 531                 *area_count(area_dst, page_nr) = count_verify[page_nr] = count;
 532                 pthread_mutex_unlock(area_mutex(area_dst, page_nr));
 533         }
 534
 535         return NULL;
 536 }
 537
 538 static void retry_copy_page(int ufd, struct uffdio_copy *uffdio_copy,
 539                             unsigned long offset)
 540 {
 541         uffd_test_ops->alias_mapping(&uffdio_copy->dst,
 542                                      uffdio_copy->len,
 543                                      offset);
 544         if (ioctl(ufd, UFFDIO_COPY, uffdio_copy)) {
 545                 /* real retval in ufdio_copy.copy */
 546                 if (uffdio_copy->copy != -EEXIST)
 547                         err("UFFDIO_COPY retry error: %"PRId64,
 548                             (int64_t)uffdio_copy->copy);
 549         } else {
 550                 err("UFFDIO_COPY retry unexpected: %"PRId64,
 551                     (int64_t)uffdio_copy->copy);
 552         }
 553 }
 554
 555 static int __copy_page(int ufd, unsigned long offset, bool retry)
 556 {
 557         struct uffdio_copy uffdio_copy;
 558
 559         if (offset >= nr_pages * page_size)
 560                 err("unexpected offset %lu\n", offset);
 561         uffdio_copy.dst = (unsigned long) area_dst + offset;
 562         uffdio_copy.src = (unsigned long) area_src + offset;
 563         uffdio_copy.len = page_size;
 564         if (test_uffdio_wp)
 565                 uffdio_copy.mode = UFFDIO_COPY_MODE_WP;
 566         else
 567                 uffdio_copy.mode = 0;
 568         uffdio_copy.copy = 0;
 569         if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) {
 570                 /* real retval in ufdio_copy.copy */
 571                 if (uffdio_copy.copy != -EEXIST)
 572                         err("UFFDIO_COPY error: %"PRId64,
 573                             (int64_t)uffdio_copy.copy);
 574         } else if (uffdio_copy.copy != page_size) {
 575                 err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy);
 576         } else {
 577                 if (test_uffdio_copy_eexist && retry) {
 578                         test_uffdio_copy_eexist = false;
 579                         retry_copy_page(ufd, &uffdio_copy, offset);
 580                 }
 581                 return 1;
 582         }
 583         return 0;
 584 }
 585
 586 static int copy_page_retry(int ufd, unsigned long offset)
 587 {
 588         return __copy_page(ufd, offset, true);
 589 }
 590
 591 static int copy_page(int ufd, unsigned long offset)
 592 {
 593         return __copy_page(ufd, offset, false);
 594 }
 595
 596 static int uffd_read_msg(int ufd, struct uffd_msg *msg)
 597 {
 598         int ret = read(uffd, msg, sizeof(*msg));
 599
 600         if (ret != sizeof(*msg)) {
 601                 if (ret < 0) {
 602                         if (errno == EAGAIN)
 603                                 return 1;
 604                         err("blocking read error");
 605                 } else {
 606                         err("short read");
 607                 }
 608         }
 609
 610         return 0;
 611 }
 612
 613 static void uffd_handle_page_fault(struct uffd_msg *msg,
 614                                    struct uffd_stats *stats)
 615 {
 616         unsigned long offset;
 617
 618         if (msg->event != UFFD_EVENT_PAGEFAULT)
 619                 err("unexpected msg event %u", msg->event);
 620
 621         if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {
 622                 /* Write protect page faults */
 623                 wp_range(uffd, msg->arg.pagefault.address, page_size, false);
 624                 stats->wp_faults++;
 625         } else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {
 626                 uint8_t *area;
 627                 int b;
 628
 629                 /*
 630                  * Minor page faults
 631                  *
 632                  * To prove we can modify the original range for testing
 633                  * purposes, we're going to bit flip this range before
 634                  * continuing.
 635                  *
 636                  * Note that this requires all minor page fault tests operate on
 637                  * area_dst (non-UFFD-registered) and area_dst_alias
 638                  * (UFFD-registered).
 639                  */
 640
 641                 area = (uint8_t *)(area_dst +
 642                                    ((char *)msg->arg.pagefault.address -
 643                                     area_dst_alias));
 644                 for (b = 0; b < page_size; ++b)
 645                         area[b] = ~area[b];
 646                 continue_range(uffd, msg->arg.pagefault.address, page_size);
 647                 stats->minor_faults++;
 648         } else {
 649                 /* Missing page faults */
 650                 if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)
 651                         err("unexpected write fault");
 652
 653                 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
 654                 offset &= ~(page_size-1);
 655
 656                 if (copy_page(uffd, offset))
 657                         stats->missing_faults++;
 658         }
 659 }
 660
 661 static void *uffd_poll_thread(void *arg)
 662 {
 663         struct uffd_stats *stats = (struct uffd_stats *)arg;
 664         unsigned long cpu = stats->cpu;
 665         struct pollfd pollfd[2];
 666         struct uffd_msg msg;
 667         struct uffdio_register uffd_reg;
 668         int ret;
 669         char tmp_chr;
 670
 671         pollfd[0].fd = uffd;
 672         pollfd[0].events = POLLIN;
 673         pollfd[1].fd = pipefd[cpu*2];
 674         pollfd[1].events = POLLIN;
 675
 676         for (;;) {
 677                 ret = poll(pollfd, 2, -1);
 678                 if (ret <= 0)
 679                         err("poll error: %d", ret);
 680                 if (pollfd[1].revents & POLLIN) {
 681                         if (read(pollfd[1].fd, &tmp_chr, 1) != 1)
 682                                 err("read pipefd error");
 683                         break;
 684                 }
 685                 if (!(pollfd[0].revents & POLLIN))
 686                         err("pollfd[0].revents %d", pollfd[0].revents);
 687                 if (uffd_read_msg(uffd, &msg))
 688                         continue;
 689                 switch (msg.event) {
 690                 default:
 691                         err("unexpected msg event %u\n", msg.event);
 692                         break;
 693                 case UFFD_EVENT_PAGEFAULT:
 694                         uffd_handle_page_fault(&msg, stats);
 695                         break;
 696                 case UFFD_EVENT_FORK:
 697                         close(uffd);
 698                         uffd = msg.arg.fork.ufd;
 699                         pollfd[0].fd = uffd;
 700                         break;
 701                 case UFFD_EVENT_REMOVE:
 702                         uffd_reg.range.start = msg.arg.remove.start;
 703                         uffd_reg.range.len = msg.arg.remove.end -
 704                                 msg.arg.remove.start;
 705                         if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))
 706                                 err("remove failure");
 707                         break;
 708                 case UFFD_EVENT_REMAP:
 709                         area_dst = (char *)(unsigned long)msg.arg.remap.to;
 710                         break;
 711                 }
 712         }
 713
 714         return NULL;
 715 }
 716
 717 pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER;
 718
 719 static void *uffd_read_thread(void *arg)
 720 {
 721         struct uffd_stats *stats = (struct uffd_stats *)arg;
 722         struct uffd_msg msg;
 723
 724         pthread_mutex_unlock(&uffd_read_mutex);
 725         /* from here cancellation is ok */
 726
 727         for (;;) {
 728                 if (uffd_read_msg(uffd, &msg))
 729                         continue;
 730                 uffd_handle_page_fault(&msg, stats);
 731         }
 732
 733         return NULL;
 734 }
 735
 736 static void *background_thread(void *arg)
 737 {
 738         unsigned long cpu = (unsigned long) arg;
 739         unsigned long page_nr, start_nr, mid_nr, end_nr;
 740
 741         start_nr = cpu * nr_pages_per_cpu;
 742         end_nr = (cpu+1) * nr_pages_per_cpu;
 743         mid_nr = (start_nr + end_nr) / 2;
 744
 745         /* Copy the first half of the pages */
 746         for (page_nr = start_nr; page_nr < mid_nr; page_nr++)
 747                 copy_page_retry(uffd, page_nr * page_size);
 748
 749         /*
 750          * If we need to test uffd-wp, set it up now.  Then we'll have
 751          * at least the first half of the pages mapped already which
 752          * can be write-protected for testing
 753          */
 754         if (test_uffdio_wp)
 755                 wp_range(uffd, (unsigned long)area_dst + start_nr * page_size,
 756                         nr_pages_per_cpu * page_size, true);
 757
 758         /*
 759          * Continue the 2nd half of the page copying, handling write
 760          * protection faults if any
 761          */
 762         for (page_nr = mid_nr; page_nr < end_nr; page_nr++)
 763                 copy_page_retry(uffd, page_nr * page_size);
 764
 765         return NULL;
 766 }
 767
 768 static int stress(struct uffd_stats *uffd_stats)
 769 {
 770         unsigned long cpu;
 771         pthread_t locking_threads[nr_cpus];
 772         pthread_t uffd_threads[nr_cpus];
 773         pthread_t background_threads[nr_cpus];
 774
 775         finished = 0;
 776         for (cpu = 0; cpu < nr_cpus; cpu++) {
 777                 if (pthread_create(&locking_threads[cpu], &attr,
 778                                    locking_thread, (void *)cpu))
 779                         return 1;
 780                 if (bounces & BOUNCE_POLL) {
 781                         if (pthread_create(&uffd_threads[cpu], &attr,
 782                                            uffd_poll_thread,
 783                                            (void *)&uffd_stats[cpu]))
 784                                 return 1;
 785                 } else {
 786                         if (pthread_create(&uffd_threads[cpu], &attr,
 787                                            uffd_read_thread,
 788                                            (void *)&uffd_stats[cpu]))
 789                                 return 1;
 790                         pthread_mutex_lock(&uffd_read_mutex);
 791                 }
 792                 if (pthread_create(&background_threads[cpu], &attr,
 793                                    background_thread, (void *)cpu))
 794                         return 1;
 795         }
 796         for (cpu = 0; cpu < nr_cpus; cpu++)
 797                 if (pthread_join(background_threads[cpu], NULL))
 798                         return 1;
 799
 800         /*
 801          * Be strict and immediately zap area_src, the whole area has
 802          * been transferred already by the background treads. The
 803          * area_src could then be faulted in in a racy way by still
 804          * running uffdio_threads reading zeropages after we zapped
 805          * area_src (but they're guaranteed to get -EEXIST from
 806          * UFFDIO_COPY without writing zero pages into area_dst
 807          * because the background threads already completed).
 808          */
 809         uffd_test_ops->release_pages(area_src);
 810
 811         finished = 1;
 812         for (cpu = 0; cpu < nr_cpus; cpu++)
 813                 if (pthread_join(locking_threads[cpu], NULL))
 814                         return 1;
 815
 816         for (cpu = 0; cpu < nr_cpus; cpu++) {
 817                 char c;
 818                 if (bounces & BOUNCE_POLL) {
 819                         if (write(pipefd[cpu*2+1], &c, 1) != 1)
 820                                 err("pipefd write error");
 821                         if (pthread_join(uffd_threads[cpu],
 822                                          (void *)&uffd_stats[cpu]))
 823                                 return 1;
 824                 } else {
 825                         if (pthread_cancel(uffd_threads[cpu]))
 826                                 return 1;
 827                         if (pthread_join(uffd_threads[cpu], NULL))
 828                                 return 1;
 829                 }
 830         }
 831
 832         return 0;
 833 }
 834
 835 sigjmp_buf jbuf, *sigbuf;
 836
 837 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
 838 {
 839         if (sig == SIGBUS) {
 840                 if (sigbuf)
 841                         siglongjmp(*sigbuf, 1);
 842                 abort();
 843         }
 844 }
 845
 846 /*
 847  * For non-cooperative userfaultfd test we fork() a process that will
 848  * generate pagefaults, will mremap the area monitored by the
 849  * userfaultfd and at last this process will release the monitored
 850  * area.
 851  * For the anonymous and shared memory the area is divided into two
 852  * parts, the first part is accessed before mremap, and the second
 853  * part is accessed after mremap. Since hugetlbfs does not support
 854  * mremap, the entire monitored area is accessed in a single pass for
 855  * HUGETLB_TEST.
 856  * The release of the pages currently generates event for shmem and
 857  * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
 858  * for hugetlb.
 859  * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
 860  * monitored area, generate pagefaults and test that signal is delivered.
 861  * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
 862  * test robustness use case - we release monitored area, fork a process
 863  * that will generate pagefaults and verify signal is generated.
 864  * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
 865  * feature. Using monitor thread, verify no userfault events are generated.
 866  */
 867 static int faulting_process(int signal_test)
 868 {
 869         unsigned long nr;
 870         unsigned long long count;
 871         unsigned long split_nr_pages;
 872         unsigned long lastnr;
 873         struct sigaction act;
 874         unsigned long signalled = 0;
 875
 876         if (test_type != TEST_HUGETLB)
 877                 split_nr_pages = (nr_pages + 1) / 2;
 878         else
 879                 split_nr_pages = nr_pages;
 880
 881         if (signal_test) {
 882                 sigbuf = &jbuf;
 883                 memset(&act, 0, sizeof(act));
 884                 act.sa_sigaction = sighndl;
 885                 act.sa_flags = SA_SIGINFO;
 886                 if (sigaction(SIGBUS, &act, 0))
 887                         err("sigaction");
 888                 lastnr = (unsigned long)-1;
 889         }
 890
 891         for (nr = 0; nr < split_nr_pages; nr++) {
 892                 int steps = 1;
 893                 unsigned long offset = nr * page_size;
 894
 895                 if (signal_test) {
 896                         if (sigsetjmp(*sigbuf, 1) != 0) {
 897                                 if (steps == 1 && nr == lastnr)
 898                                         err("Signal repeated");
 899
 900                                 lastnr = nr;
 901                                 if (signal_test == 1) {
 902                                         if (steps == 1) {
 903                                                 /* This is a MISSING request */
 904                                                 steps++;
 905                                                 if (copy_page(uffd, offset))
 906                                                         signalled++;
 907                                         } else {
 908                                                 /* This is a WP request */
 909                                                 assert(steps == 2);
 910                                                 wp_range(uffd,
 911                                                          (__u64)area_dst +
 912                                                          offset,
 913                                                          page_size, false);
 914                                         }
 915                                 } else {
 916                                         signalled++;
 917                                         continue;
 918                                 }
 919                         }
 920                 }
 921
 922                 count = *area_count(area_dst, nr);
 923                 if (count != count_verify[nr])
 924                         err("nr %lu memory corruption %llu %llu\n",
 925                             nr, count, count_verify[nr]);
 926                 /*
 927                  * Trigger write protection if there is by writing
 928                  * the same value back.
 929                  */
 930                 *area_count(area_dst, nr) = count;
 931         }
 932
 933         if (signal_test)
 934                 return signalled != split_nr_pages;
 935
 936         if (test_type == TEST_HUGETLB)
 937                 return 0;
 938
 939         area_dst = mremap(area_dst, nr_pages * page_size,  nr_pages * page_size,
 940                           MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
 941         if (area_dst == MAP_FAILED)
 942                 err("mremap");
 943         /* Reset area_src since we just clobbered it */
 944         area_src = NULL;
 945
 946         for (; nr < nr_pages; nr++) {
 947                 count = *area_count(area_dst, nr);
 948                 if (count != count_verify[nr]) {
 949                         err("nr %lu memory corruption %llu %llu\n",
 950                             nr, count, count_verify[nr]);
 951                 }
 952                 /*
 953                  * Trigger write protection if there is by writing
 954                  * the same value back.
 955                  */
 956                 *area_count(area_dst, nr) = count;
 957         }
 958
 959         uffd_test_ops->release_pages(area_dst);
 960
 961         for (nr = 0; nr < nr_pages; nr++)
 962                 if (my_bcmp(area_dst + nr * page_size, zeropage, page_size))
 963                         err("nr %lu is not zero", nr);
 964
 965         return 0;
 966 }
 967
 968 static void retry_uffdio_zeropage(int ufd,
 969                                   struct uffdio_zeropage *uffdio_zeropage,
 970                                   unsigned long offset)
 971 {
 972         uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
 973                                      uffdio_zeropage->range.len,
 974                                      offset);
 975         if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
 976                 if (uffdio_zeropage->zeropage != -EEXIST)
 977                         err("UFFDIO_ZEROPAGE error: %"PRId64,
 978                             (int64_t)uffdio_zeropage->zeropage);
 979         } else {
 980                 err("UFFDIO_ZEROPAGE error: %"PRId64,
 981                     (int64_t)uffdio_zeropage->zeropage);
 982         }
 983 }
 984
 985 static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry)
 986 {
 987         struct uffdio_zeropage uffdio_zeropage;
 988         int ret;
 989         unsigned long has_zeropage;
 990         __s64 res;
 991
 992         has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);
 993
 994         if (offset >= nr_pages * page_size)
 995                 err("unexpected offset %lu", offset);
 996         uffdio_zeropage.range.start = (unsigned long) area_dst + offset;
 997         uffdio_zeropage.range.len = page_size;
 998         uffdio_zeropage.mode = 0;
 999         ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
1000         res = uffdio_zeropage.zeropage;
1001         if (ret) {
1002                 /* real retval in ufdio_zeropage.zeropage */
1003                 if (has_zeropage)
1004                         err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
1005                 else if (res != -EINVAL)
1006                         err("UFFDIO_ZEROPAGE not -EINVAL");
1007         } else if (has_zeropage) {
1008                 if (res != page_size) {
1009                         err("UFFDIO_ZEROPAGE unexpected size");
1010                 } else {
1011                         if (test_uffdio_zeropage_eexist && retry) {
1012                                 test_uffdio_zeropage_eexist = false;
1013                                 retry_uffdio_zeropage(ufd, &uffdio_zeropage,
1014                                                       offset);
1015                         }
1016                         return 1;
1017                 }
1018         } else
1019                 err("UFFDIO_ZEROPAGE succeeded");
1020
1021         return 0;
1022 }
1023
1024 static int uffdio_zeropage(int ufd, unsigned long offset)
1025 {
1026         return __uffdio_zeropage(ufd, offset, false);
1027 }
1028
1029 /* exercise UFFDIO_ZEROPAGE */
1030 static int userfaultfd_zeropage_test(void)
1031 {
1032         struct uffdio_register uffdio_register;
1033         unsigned long expected_ioctls;
1034
1035         printf("testing UFFDIO_ZEROPAGE: ");
1036         fflush(stdout);
1037
1038         uffd_test_ctx_init(0);
1039
1040         uffdio_register.range.start = (unsigned long) area_dst;
1041         uffdio_register.range.len = nr_pages * page_size;
1042         uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1043         if (test_uffdio_wp)
1044                 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1045         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1046                 err("register failure");
1047
1048         expected_ioctls = uffd_test_ops->expected_ioctls;
1049         if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls)
1050                 err("unexpected missing ioctl for anon memory");
1051
1052         if (uffdio_zeropage(uffd, 0))
1053                 if (my_bcmp(area_dst, zeropage, page_size))
1054                         err("zeropage is not zero");
1055
1056         printf("done.\n");
1057         return 0;
1058 }
1059
1060 static int userfaultfd_events_test(void)
1061 {
1062         struct uffdio_register uffdio_register;
1063         unsigned long expected_ioctls;
1064         pthread_t uffd_mon;
1065         int err, features;
1066         pid_t pid;
1067         char c;
1068         struct uffd_stats stats = { 0 };
1069
1070         printf("testing events (fork, remap, remove): ");
1071         fflush(stdout);
1072
1073         features = UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP |
1074                 UFFD_FEATURE_EVENT_REMOVE;
1075         uffd_test_ctx_init(features);
1076
1077         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1078
1079         uffdio_register.range.start = (unsigned long) area_dst;
1080         uffdio_register.range.len = nr_pages * page_size;
1081         uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1082         if (test_uffdio_wp)
1083                 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1084         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1085                 err("register failure");
1086
1087         expected_ioctls = uffd_test_ops->expected_ioctls;
1088         if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls)
1089                 err("unexpected missing ioctl for anon memory");
1090
1091         if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
1092                 err("uffd_poll_thread create");
1093
1094         pid = fork();
1095         if (pid < 0)
1096                 err("fork");
1097
1098         if (!pid)
1099                 exit(faulting_process(0));
1100
1101         waitpid(pid, &err, 0);
1102         if (err)
1103                 err("faulting process failed");
1104         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1105                 err("pipe write");
1106         if (pthread_join(uffd_mon, NULL))
1107                 return 1;
1108
1109         uffd_stats_report(&stats, 1);
1110
1111         return stats.missing_faults != nr_pages;
1112 }
1113
1114 static int userfaultfd_sig_test(void)
1115 {
1116         struct uffdio_register uffdio_register;
1117         unsigned long expected_ioctls;
1118         unsigned long userfaults;
1119         pthread_t uffd_mon;
1120         int err, features;
1121         pid_t pid;
1122         char c;
1123         struct uffd_stats stats = { 0 };
1124
1125         printf("testing signal delivery: ");
1126         fflush(stdout);
1127
1128         features = UFFD_FEATURE_EVENT_FORK|UFFD_FEATURE_SIGBUS;
1129         uffd_test_ctx_init(features);
1130
1131         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1132
1133         uffdio_register.range.start = (unsigned long) area_dst;
1134         uffdio_register.range.len = nr_pages * page_size;
1135         uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1136         if (test_uffdio_wp)
1137                 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1138         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1139                 err("register failure");
1140
1141         expected_ioctls = uffd_test_ops->expected_ioctls;
1142         if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls)
1143                 err("unexpected missing ioctl for anon memory");
1144
1145         if (faulting_process(1))
1146                 err("faulting process failed");
1147
1148         uffd_test_ops->release_pages(area_dst);
1149
1150         if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
1151                 err("uffd_poll_thread create");
1152
1153         pid = fork();
1154         if (pid < 0)
1155                 err("fork");
1156
1157         if (!pid)
1158                 exit(faulting_process(2));
1159
1160         waitpid(pid, &err, 0);
1161         if (err)
1162                 err("faulting process failed");
1163         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1164                 err("pipe write");
1165         if (pthread_join(uffd_mon, (void **)&userfaults))
1166                 return 1;
1167
1168         printf("done.\n");
1169         if (userfaults)
1170                 err("Signal test failed, userfaults: %ld", userfaults);
1171
1172         return userfaults != 0;
1173 }
1174
1175 static int userfaultfd_minor_test(void)
1176 {
1177         struct uffdio_register uffdio_register;
1178         unsigned long expected_ioctls;
1179         unsigned long p;
1180         pthread_t uffd_mon;
1181         uint8_t expected_byte;
1182         void *expected_page;
1183         char c;
1184         struct uffd_stats stats = { 0 };
1185         uint64_t features = UFFD_FEATURE_MINOR_HUGETLBFS;
1186
1187         if (!test_uffdio_minor)
1188                 return 0;
1189
1190         printf("testing minor faults: ");
1191         fflush(stdout);
1192
1193         uffd_test_ctx_init_ext(&features);
1194         /* If kernel reports the feature isn't supported, skip the test. */
1195         if (!(features & UFFD_FEATURE_MINOR_HUGETLBFS)) {
1196                 printf("skipping test due to lack of feature support\n");
1197                 fflush(stdout);
1198                 return 0;
1199         }
1200
1201         uffdio_register.range.start = (unsigned long)area_dst_alias;
1202         uffdio_register.range.len = nr_pages * page_size;
1203         uffdio_register.mode = UFFDIO_REGISTER_MODE_MINOR;
1204         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1205                 err("register failure");
1206
1207         expected_ioctls = uffd_test_ops->expected_ioctls;
1208         expected_ioctls |= 1 << _UFFDIO_CONTINUE;
1209         if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls)
1210                 err("unexpected missing ioctl(s)");
1211
1212         /*
1213          * After registering with UFFD, populate the non-UFFD-registered side of
1214          * the shared mapping. This should *not* trigger any UFFD minor faults.
1215          */
1216         for (p = 0; p < nr_pages; ++p) {
1217                 memset(area_dst + (p * page_size), p % ((uint8_t)-1),
1218                        page_size);
1219         }
1220
1221         if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
1222                 err("uffd_poll_thread create");
1223
1224         /*
1225          * Read each of the pages back using the UFFD-registered mapping. We
1226          * expect that the first time we touch a page, it will result in a minor
1227          * fault. uffd_poll_thread will resolve the fault by bit-flipping the
1228          * page's contents, and then issuing a CONTINUE ioctl.
1229          */
1230
1231         if (posix_memalign(&expected_page, page_size, page_size))
1232                 err("out of memory");
1233
1234         for (p = 0; p < nr_pages; ++p) {
1235                 expected_byte = ~((uint8_t)(p % ((uint8_t)-1)));
1236                 memset(expected_page, expected_byte, page_size);
1237                 if (my_bcmp(expected_page, area_dst_alias + (p * page_size),
1238                             page_size))
1239                         err("unexpected page contents after minor fault");
1240         }
1241
1242         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1243                 err("pipe write");
1244         if (pthread_join(uffd_mon, NULL))
1245                 return 1;
1246
1247         uffd_stats_report(&stats, 1);
1248
1249         return stats.missing_faults != 0 || stats.minor_faults != nr_pages;
1250 }
1251
1252 #define BIT_ULL(nr)                   (1ULL << (nr))
1253 #define PM_SOFT_DIRTY                 BIT_ULL(55)
1254 #define PM_MMAP_EXCLUSIVE             BIT_ULL(56)
1255 #define PM_UFFD_WP                    BIT_ULL(57)
1256 #define PM_FILE                       BIT_ULL(61)
1257 #define PM_SWAP                       BIT_ULL(62)
1258 #define PM_PRESENT                    BIT_ULL(63)
1259
1260 static int pagemap_open(void)
1261 {
1262         int fd = open("/proc/self/pagemap", O_RDONLY);
1263
1264         if (fd < 0)
1265                 err("open pagemap");
1266
1267         return fd;
1268 }
1269
1270 static uint64_t pagemap_read_vaddr(int fd, void *vaddr)
1271 {
1272         uint64_t value;
1273         int ret;
1274
1275         ret = pread(fd, &value, sizeof(uint64_t),
1276                     ((uint64_t)vaddr >> 12) * sizeof(uint64_t));
1277         if (ret != sizeof(uint64_t))
1278                 err("pread() on pagemap failed");
1279
1280         return value;
1281 }
1282
1283 /* This macro let __LINE__ works in err() */
1284 #define  pagemap_check_wp(value, wp) do {                               \
1285                 if (!!(value & PM_UFFD_WP) != wp)                       \
1286                         err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
1287         } while (0)
1288
1289 static int pagemap_test_fork(bool present)
1290 {
1291         pid_t child = fork();
1292         uint64_t value;
1293         int fd, result;
1294
1295         if (!child) {
1296                 /* Open the pagemap fd of the child itself */
1297                 fd = pagemap_open();
1298                 value = pagemap_read_vaddr(fd, area_dst);
1299                 /*
1300                  * After fork() uffd-wp bit should be gone as long as we're
1301                  * without UFFD_FEATURE_EVENT_FORK
1302                  */
1303                 pagemap_check_wp(value, false);
1304                 /* Succeed */
1305                 exit(0);
1306         }
1307         waitpid(child, &result, 0);
1308         return result;
1309 }
1310
1311 static void userfaultfd_pagemap_test(unsigned int test_pgsize)
1312 {
1313         struct uffdio_register uffdio_register;
1314         int pagemap_fd;
1315         uint64_t value;
1316
1317         /* Pagemap tests uffd-wp only */
1318         if (!test_uffdio_wp)
1319                 return;
1320
1321         /* Not enough memory to test this page size */
1322         if (test_pgsize > nr_pages * page_size)
1323                 return;
1324
1325         printf("testing uffd-wp with pagemap (pgsize=%u): ", test_pgsize);
1326         /* Flush so it doesn't flush twice in parent/child later */
1327         fflush(stdout);
1328
1329         uffd_test_ctx_init(0);
1330
1331         if (test_pgsize > page_size) {
1332                 /* This is a thp test */
1333                 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
1334                         err("madvise(MADV_HUGEPAGE) failed");
1335         } else if (test_pgsize == page_size) {
1336                 /* This is normal page test; force no thp */
1337                 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
1338                         err("madvise(MADV_NOHUGEPAGE) failed");
1339         }
1340
1341         uffdio_register.range.start = (unsigned long) area_dst;
1342         uffdio_register.range.len = nr_pages * page_size;
1343         uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
1344         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1345                 err("register failed");
1346
1347         pagemap_fd = pagemap_open();
1348
1349         /* Touch the page */
1350         *area_dst = 1;
1351         wp_range(uffd, (uint64_t)area_dst, test_pgsize, true);
1352         value = pagemap_read_vaddr(pagemap_fd, area_dst);
1353         pagemap_check_wp(value, true);
1354         /* Make sure uffd-wp bit dropped when fork */
1355         if (pagemap_test_fork(true))
1356                 err("Detected stall uffd-wp bit in child");
1357
1358         /* Exclusive required or PAGEOUT won't work */
1359         if (!(value & PM_MMAP_EXCLUSIVE))
1360                 err("multiple mapping detected: 0x%"PRIx64, value);
1361
1362         if (madvise(area_dst, test_pgsize, MADV_PAGEOUT))
1363                 err("madvise(MADV_PAGEOUT) failed");
1364
1365         /* Uffd-wp should persist even swapped out */
1366         value = pagemap_read_vaddr(pagemap_fd, area_dst);
1367         pagemap_check_wp(value, true);
1368         /* Make sure uffd-wp bit dropped when fork */
1369         if (pagemap_test_fork(false))
1370                 err("Detected stall uffd-wp bit in child");
1371
1372         /* Unprotect; this tests swap pte modifications */
1373         wp_range(uffd, (uint64_t)area_dst, page_size, false);
1374         value = pagemap_read_vaddr(pagemap_fd, area_dst);
1375         pagemap_check_wp(value, false);
1376
1377         /* Fault in the page from disk */
1378         *area_dst = 2;
1379         value = pagemap_read_vaddr(pagemap_fd, area_dst);
1380         pagemap_check_wp(value, false);
1381
1382         close(pagemap_fd);
1383         printf("done\n");
1384 }
1385
1386 static int userfaultfd_stress(void)
1387 {
1388         void *area;
1389         char *tmp_area;
1390         unsigned long nr;
1391         struct uffdio_register uffdio_register;
1392         struct uffd_stats uffd_stats[nr_cpus];
1393
1394         uffd_test_ctx_init(0);
1395
1396         if (posix_memalign(&area, page_size, page_size))
1397                 err("out of memory");
1398         zeropage = area;
1399         bzero(zeropage, page_size);
1400
1401         pthread_mutex_lock(&uffd_read_mutex);
1402
1403         pthread_attr_init(&attr);
1404         pthread_attr_setstacksize(&attr, 16*1024*1024);
1405
1406         while (bounces--) {
1407                 unsigned long expected_ioctls;
1408
1409                 printf("bounces: %d, mode:", bounces);
1410                 if (bounces & BOUNCE_RANDOM)
1411                         printf(" rnd");
1412                 if (bounces & BOUNCE_RACINGFAULTS)
1413                         printf(" racing");
1414                 if (bounces & BOUNCE_VERIFY)
1415                         printf(" ver");
1416                 if (bounces & BOUNCE_POLL)
1417                         printf(" poll");
1418                 else
1419                         printf(" read");
1420                 printf(", ");
1421                 fflush(stdout);
1422
1423                 if (bounces & BOUNCE_POLL)
1424                         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1425                 else
1426                         fcntl(uffd, F_SETFL, uffd_flags & ~O_NONBLOCK);
1427
1428                 /* register */
1429                 uffdio_register.range.start = (unsigned long) area_dst;
1430                 uffdio_register.range.len = nr_pages * page_size;
1431                 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1432                 if (test_uffdio_wp)
1433                         uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1434                 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1435                         err("register failure");
1436                 expected_ioctls = uffd_test_ops->expected_ioctls;
1437                 if ((uffdio_register.ioctls & expected_ioctls) !=
1438                     expected_ioctls)
1439                         err("unexpected missing ioctl for anon memory");
1440
1441                 if (area_dst_alias) {
1442                         uffdio_register.range.start = (unsigned long)
1443                                 area_dst_alias;
1444                         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1445                                 err("register failure alias");
1446                 }
1447
1448                 /*
1449                  * The madvise done previously isn't enough: some
1450                  * uffd_thread could have read userfaults (one of
1451                  * those already resolved by the background thread)
1452                  * and it may be in the process of calling
1453                  * UFFDIO_COPY. UFFDIO_COPY will read the zapped
1454                  * area_src and it would map a zero page in it (of
1455                  * course such a UFFDIO_COPY is perfectly safe as it'd
1456                  * return -EEXIST). The problem comes at the next
1457                  * bounce though: that racing UFFDIO_COPY would
1458                  * generate zeropages in the area_src, so invalidating
1459                  * the previous MADV_DONTNEED. Without this additional
1460                  * MADV_DONTNEED those zeropages leftovers in the
1461                  * area_src would lead to -EEXIST failure during the
1462                  * next bounce, effectively leaving a zeropage in the
1463                  * area_dst.
1464                  *
1465                  * Try to comment this out madvise to see the memory
1466                  * corruption being caught pretty quick.
1467                  *
1468                  * khugepaged is also inhibited to collapse THP after
1469                  * MADV_DONTNEED only after the UFFDIO_REGISTER, so it's
1470                  * required to MADV_DONTNEED here.
1471                  */
1472                 uffd_test_ops->release_pages(area_dst);
1473
1474                 uffd_stats_reset(uffd_stats, nr_cpus);
1475
1476                 /* bounce pass */
1477                 if (stress(uffd_stats))
1478                         return 1;
1479
1480                 /* Clear all the write protections if there is any */
1481                 if (test_uffdio_wp)
1482                         wp_range(uffd, (unsigned long)area_dst,
1483                                  nr_pages * page_size, false);
1484
1485                 /* unregister */
1486                 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range))
1487                         err("unregister failure");
1488                 if (area_dst_alias) {
1489                         uffdio_register.range.start = (unsigned long) area_dst;
1490                         if (ioctl(uffd, UFFDIO_UNREGISTER,
1491                                   &uffdio_register.range))
1492                                 err("unregister failure alias");
1493                 }
1494
1495                 /* verification */
1496                 if (bounces & BOUNCE_VERIFY)
1497                         for (nr = 0; nr < nr_pages; nr++)
1498                                 if (*area_count(area_dst, nr) != count_verify[nr])
1499                                         err("error area_count %llu %llu %lu\n",
1500                                             *area_count(area_src, nr),
1501                                             count_verify[nr], nr);
1502
1503                 /* prepare next bounce */
1504                 tmp_area = area_src;
1505                 area_src = area_dst;
1506                 area_dst = tmp_area;
1507
1508                 tmp_area = area_src_alias;
1509                 area_src_alias = area_dst_alias;
1510                 area_dst_alias = tmp_area;
1511
1512                 uffd_stats_report(uffd_stats, nr_cpus);
1513         }
1514
1515         if (test_type == TEST_ANON) {
1516                 /*
1517                  * shmem/hugetlb won't be able to run since they have different
1518                  * behavior on fork() (file-backed memory normally drops ptes
1519                  * directly when fork), meanwhile the pagemap test will verify
1520                  * pgtable entry of fork()ed child.
1521                  */
1522                 userfaultfd_pagemap_test(page_size);
1523                 /*
1524                  * Hard-code for x86_64 for now for 2M THP, as x86_64 is
1525                  * currently the only one that supports uffd-wp
1526                  */
1527                 userfaultfd_pagemap_test(page_size * 512);
1528         }
1529
1530         return userfaultfd_zeropage_test() || userfaultfd_sig_test()
1531                 || userfaultfd_events_test() || userfaultfd_minor_test();
1532 }
1533
1534 /*
1535  * Copied from mlock2-tests.c
1536  */
1537 unsigned long default_huge_page_size(void)
1538 {
1539         unsigned long hps = 0;
1540         char *line = NULL;
1541         size_t linelen = 0;
1542         FILE *f = fopen("/proc/meminfo", "r");
1543
1544         if (!f)
1545                 return 0;
1546         while (getline(&line, &linelen, f) > 0) {
1547                 if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
1548                         hps <<= 10;
1549                         break;
1550                 }
1551         }
1552
1553         free(line);
1554         fclose(f);
1555         return hps;
1556 }
1557
1558 static void set_test_type(const char *type)
1559 {
1560         if (!strcmp(type, "anon")) {
1561                 test_type = TEST_ANON;
1562                 uffd_test_ops = &anon_uffd_test_ops;
1563                 /* Only enable write-protect test for anonymous test */
1564                 test_uffdio_wp = true;
1565         } else if (!strcmp(type, "hugetlb")) {
1566                 test_type = TEST_HUGETLB;
1567                 uffd_test_ops = &hugetlb_uffd_test_ops;
1568         } else if (!strcmp(type, "hugetlb_shared")) {
1569                 map_shared = true;
1570                 test_type = TEST_HUGETLB;
1571                 uffd_test_ops = &hugetlb_uffd_test_ops;
1572                 /* Minor faults require shared hugetlb; only enable here. */
1573                 test_uffdio_minor = true;
1574         } else if (!strcmp(type, "shmem")) {
1575                 map_shared = true;
1576                 test_type = TEST_SHMEM;
1577                 uffd_test_ops = &shmem_uffd_test_ops;
1578         } else {
1579                 err("Unknown test type: %s", type);
1580         }
1581
1582         if (test_type == TEST_HUGETLB)
1583                 page_size = default_huge_page_size();
1584         else
1585                 page_size = sysconf(_SC_PAGE_SIZE);
1586
1587         if (!page_size)
1588                 err("Unable to determine page size");
1589         if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2
1590             > page_size)
1591                 err("Impossible to run this test");
1592 }
1593
1594 static void sigalrm(int sig)
1595 {
1596         if (sig != SIGALRM)
1597                 abort();
1598         test_uffdio_copy_eexist = true;
1599         test_uffdio_zeropage_eexist = true;
1600         alarm(ALARM_INTERVAL_SECS);
1601 }
1602
1603 int main(int argc, char **argv)
1604 {
1605         if (argc < 4)
1606                 usage();
1607
1608         if (signal(SIGALRM, sigalrm) == SIG_ERR)
1609                 err("failed to arm SIGALRM");
1610         alarm(ALARM_INTERVAL_SECS);
1611
1612         set_test_type(argv[1]);
1613
1614         nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
1615         nr_pages_per_cpu = atol(argv[2]) * 1024*1024 / page_size /
1616                 nr_cpus;
1617         if (!nr_pages_per_cpu) {
1618                 _err("invalid MiB");
1619                 usage();
1620         }
1621
1622         bounces = atoi(argv[3]);
1623         if (bounces <= 0) {
1624                 _err("invalid bounces");
1625                 usage();
1626         }
1627         nr_pages = nr_pages_per_cpu * nr_cpus;
1628
1629         if (test_type == TEST_HUGETLB) {
1630                 if (argc < 5)
1631                         usage();
1632                 huge_fd = open(argv[4], O_CREAT | O_RDWR, 0755);
1633                 if (huge_fd < 0)
1634                         err("Open of %s failed", argv[4]);
1635                 if (ftruncate(huge_fd, 0))
1636                         err("ftruncate %s to size 0 failed", argv[4]);
1637         } else if (test_type == TEST_SHMEM) {
1638                 shm_fd = memfd_create(argv[0], 0);
1639                 if (shm_fd < 0)
1640                         err("memfd_create");
1641                 if (ftruncate(shm_fd, nr_pages * page_size * 2))
1642                         err("ftruncate");
1643                 if (fallocate(shm_fd,
1644                               FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0,
1645                               nr_pages * page_size * 2))
1646                         err("fallocate");
1647         }
1648         printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n",
1649                nr_pages, nr_pages_per_cpu);
1650         return userfaultfd_stress();
1651 }
1652
1653 #else /* __NR_userfaultfd */
1654
1655 #warning "missing __NR_userfaultfd definition"
1656
1657 int main(void)
1658 {
1659         printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");
1660         return KSFT_SKIP;
1661 }
1662
1663 #endif /* __NR_userfaultfd */