tools/testing/selftests/vm/transhuge-stress.c

   1 /*
   2  * Stress test for transparent huge pages, memory compaction and migration.
   3  *
   4  * Authors: Konstantin Khlebnikov <koct9i@gmail.com>
   5  *
   6  * This is free and unencumbered software released into the public domain.
   7  */
   8
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11 #include <stdint.h>
  12 #include <err.h>
  13 #include <time.h>
  14 #include <unistd.h>
  15 #include <fcntl.h>
  16 #include <string.h>
  17 #include <sys/mman.h>
  18
  19 #define PAGE_SHIFT 12
  20 #define HPAGE_SHIFT 21
  21
  22 #define PAGE_SIZE (1 << PAGE_SHIFT)
  23 #define HPAGE_SIZE (1 << HPAGE_SHIFT)
  24
  25 #define PAGEMAP_PRESENT(ent)    (((ent) & (1ull << 63)) != 0)
  26 #define PAGEMAP_PFN(ent)        ((ent) & ((1ull << 55) - 1))
  27
  28 int pagemap_fd;
  29 int backing_fd = -1;
  30 int mmap_flags = MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE;
  31 #define PROT_RW (PROT_READ | PROT_WRITE)
  32
  33 int64_t allocate_transhuge(void *ptr)
  34 {
  35         uint64_t ent[2];
  36
  37         /* drop pmd */
  38         if (mmap(ptr, HPAGE_SIZE, PROT_RW, MAP_FIXED | mmap_flags,
  39                  backing_fd, 0) != ptr)
  40                 errx(2, "mmap transhuge");
  41
  42         if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
  43                 err(2, "MADV_HUGEPAGE");
  44
  45         /* allocate transparent huge page */
  46         *(volatile void **)ptr = ptr;
  47
  48         if (pread(pagemap_fd, ent, sizeof(ent),
  49                         (uintptr_t)ptr >> (PAGE_SHIFT - 3)) != sizeof(ent))
  50                 err(2, "read pagemap");
  51
  52         if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
  53             PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
  54             !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - PAGE_SHIFT)) - 1)))
  55                 return PAGEMAP_PFN(ent[0]);
  56
  57         return -1;
  58 }
  59
  60 int main(int argc, char **argv)
  61 {
  62         size_t ram, len;
  63         void *ptr, *p;
  64         struct timespec a, b;
  65         int i = 0;
  66         char *name = NULL;
  67         double s;
  68         uint8_t *map;
  69         size_t map_len;
  70
  71         ram = sysconf(_SC_PHYS_PAGES);
  72         if (ram > SIZE_MAX / sysconf(_SC_PAGESIZE) / 4)
  73                 ram = SIZE_MAX / 4;
  74         else
  75                 ram *= sysconf(_SC_PAGESIZE);
  76         len = ram;
  77
  78         while (++i < argc) {
  79                 if (!strcmp(argv[i], "-h"))
  80                         errx(1, "usage: %s [size in MiB]", argv[0]);
  81                 else if (!strcmp(argv[i], "-f"))
  82                         name = argv[++i];
  83                 else
  84                         len = atoll(argv[i]) << 20;
  85         }
  86
  87         if (name) {
  88                 backing_fd = open(name, O_RDWR);
  89                 if (backing_fd == -1)
  90                         errx(2, "open %s", name);
  91                 mmap_flags = MAP_SHARED;
  92         }
  93
  94         warnx("allocate %zd transhuge pages, using %zd MiB virtual memory"
  95               " and %zd MiB of ram", len >> HPAGE_SHIFT, len >> 20,
  96               ram >> (20 + HPAGE_SHIFT - PAGE_SHIFT - 1));
  97
  98         pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
  99         if (pagemap_fd < 0)
 100                 err(2, "open pagemap");
 101
 102         len -= len % HPAGE_SIZE;
 103         ptr = mmap(NULL, len + HPAGE_SIZE, PROT_RW, mmap_flags, backing_fd, 0);
 104         if (ptr == MAP_FAILED)
 105                 err(2, "initial mmap");
 106         ptr += HPAGE_SIZE - (uintptr_t)ptr % HPAGE_SIZE;
 107
 108         if (madvise(ptr, len, MADV_HUGEPAGE))
 109                 err(2, "MADV_HUGEPAGE");
 110
 111         map_len = ram >> (HPAGE_SHIFT - 1);
 112         map = malloc(map_len);
 113         if (!map)
 114                 errx(2, "map malloc");
 115
 116         while (1) {
 117                 int nr_succeed = 0, nr_failed = 0, nr_pages = 0;
 118
 119                 memset(map, 0, map_len);
 120
 121                 clock_gettime(CLOCK_MONOTONIC, &a);
 122                 for (p = ptr; p < ptr + len; p += HPAGE_SIZE) {
 123                         int64_t pfn;
 124
 125                         pfn = allocate_transhuge(p);
 126
 127                         if (pfn < 0) {
 128                                 nr_failed++;
 129                         } else {
 130                                 size_t idx = pfn >> (HPAGE_SHIFT - PAGE_SHIFT);
 131
 132                                 nr_succeed++;
 133                                 if (idx >= map_len) {
 134                                         map = realloc(map, idx + 1);
 135                                         if (!map)
 136                                                 errx(2, "map realloc");
 137                                         memset(map + map_len, 0, idx + 1 - map_len);
 138                                         map_len = idx + 1;
 139                                 }
 140                                 if (!map[idx])
 141                                         nr_pages++;
 142                                 map[idx] = 1;
 143                         }
 144
 145                         /* split transhuge page, keep last page */
 146                         if (madvise(p, HPAGE_SIZE - PAGE_SIZE, MADV_DONTNEED))
 147                                 err(2, "MADV_DONTNEED");
 148                 }
 149                 clock_gettime(CLOCK_MONOTONIC, &b);
 150                 s = b.tv_sec - a.tv_sec + (b.tv_nsec - a.tv_nsec) / 1000000000.;
 151
 152                 warnx("%.3f s/loop, %.3f ms/page, %10.3f MiB/s\t"
 153                       "%4d succeed, %4d failed, %4d different pages",
 154                       s, s * 1000 / (len >> HPAGE_SHIFT), len / s / (1 << 20),
 155                       nr_succeed, nr_failed, nr_pages);
 156         }
 157 }