tools/testing/selftests/vm/ksm_tests.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <sys/mman.h>
   4 #include <stdbool.h>
   5 #include <time.h>
   6 #include <string.h>
   7 #include <numa.h>
   8
   9 #include "../kselftest.h"
  10 #include "../../../../include/vdso/time64.h"
  11
  12 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
  13 #define KSM_FP(s) (KSM_SYSFS_PATH s)
  14 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120
  15 #define KSM_PAGE_COUNT_DEFAULT 10l
  16 #define KSM_PROT_STR_DEFAULT "rw"
  17 #define KSM_USE_ZERO_PAGES_DEFAULT false
  18 #define KSM_MERGE_ACROSS_NODES_DEFAULT true
  19 #define MB (1ul << 20)
  20
  21 struct ksm_sysfs {
  22         unsigned long max_page_sharing;
  23         unsigned long merge_across_nodes;
  24         unsigned long pages_to_scan;
  25         unsigned long run;
  26         unsigned long sleep_millisecs;
  27         unsigned long stable_node_chains_prune_millisecs;
  28         unsigned long use_zero_pages;
  29 };
  30
  31 enum ksm_test_name {
  32         CHECK_KSM_MERGE,
  33         CHECK_KSM_UNMERGE,
  34         CHECK_KSM_ZERO_PAGE_MERGE,
  35         CHECK_KSM_NUMA_MERGE,
  36         KSM_MERGE_TIME
  37 };
  38
  39 static int ksm_write_sysfs(const char *file_path, unsigned long val)
  40 {
  41         FILE *f = fopen(file_path, "w");
  42
  43         if (!f) {
  44                 fprintf(stderr, "f %s\n", file_path);
  45                 perror("fopen");
  46                 return 1;
  47         }
  48         if (fprintf(f, "%lu", val) < 0) {
  49                 perror("fprintf");
  50                 return 1;
  51         }
  52         fclose(f);
  53
  54         return 0;
  55 }
  56
  57 static int ksm_read_sysfs(const char *file_path, unsigned long *val)
  58 {
  59         FILE *f = fopen(file_path, "r");
  60
  61         if (!f) {
  62                 fprintf(stderr, "f %s\n", file_path);
  63                 perror("fopen");
  64                 return 1;
  65         }
  66         if (fscanf(f, "%lu", val) != 1) {
  67                 perror("fscanf");
  68                 return 1;
  69         }
  70         fclose(f);
  71
  72         return 0;
  73 }
  74
  75 static int str_to_prot(char *prot_str)
  76 {
  77         int prot = 0;
  78
  79         if ((strchr(prot_str, 'r')) != NULL)
  80                 prot |= PROT_READ;
  81         if ((strchr(prot_str, 'w')) != NULL)
  82                 prot |= PROT_WRITE;
  83         if ((strchr(prot_str, 'x')) != NULL)
  84                 prot |= PROT_EXEC;
  85
  86         return prot;
  87 }
  88
  89 static void print_help(void)
  90 {
  91         printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
  92                "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
  93
  94         printf("Supported <test type>:\n"
  95                " -M (page merging)\n"
  96                " -Z (zero pages merging)\n"
  97                " -N (merging of pages in different NUMA nodes)\n"
  98                " -U (page unmerging)\n"
  99                " -P evaluate merging time and speed.\n"
 100                "    For this test, the size of duplicated memory area (in MiB)\n"
 101                "    must be provided using -s option\n\n");
 102
 103         printf(" -a: specify the access protections of pages.\n"
 104                "     <prot> must be of the form [rwx].\n"
 105                "     Default: %s\n", KSM_PROT_STR_DEFAULT);
 106         printf(" -p: specify the number of pages to test.\n"
 107                "     Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
 108         printf(" -l: limit the maximum running time (in seconds) for a test.\n"
 109                "     Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
 110         printf(" -z: change use_zero_pages tunable\n"
 111                "     Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
 112         printf(" -m: change merge_across_nodes tunable\n"
 113                "     Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
 114         printf(" -s: the size of duplicated memory area (in MiB)\n");
 115
 116         exit(0);
 117 }
 118
 119 static void  *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
 120 {
 121         void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
 122
 123         if (!map_ptr) {
 124                 perror("mmap");
 125                 return NULL;
 126         }
 127         memset(map_ptr, data, map_size);
 128         if (mprotect(map_ptr, map_size, prot)) {
 129                 perror("mprotect");
 130                 munmap(map_ptr, map_size);
 131                 return NULL;
 132         }
 133
 134         return map_ptr;
 135 }
 136
 137 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
 138 {
 139         struct timespec cur_time;
 140         unsigned long cur_scan, init_scan;
 141
 142         if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
 143                 return 1;
 144         cur_scan = init_scan;
 145
 146         while (cur_scan < init_scan + scan_count) {
 147                 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
 148                         return 1;
 149                 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
 150                         perror("clock_gettime");
 151                         return 1;
 152                 }
 153                 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
 154                         printf("Scan time limit exceeded\n");
 155                         return 1;
 156                 }
 157         }
 158
 159         return 0;
 160 }
 161
 162 static int ksm_merge_pages(void *addr, size_t size, struct timespec start_time, int timeout)
 163 {
 164         if (madvise(addr, size, MADV_MERGEABLE)) {
 165                 perror("madvise");
 166                 return 1;
 167         }
 168         if (ksm_write_sysfs(KSM_FP("run"), 1))
 169                 return 1;
 170
 171         /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
 172         if (ksm_do_scan(2, start_time, timeout))
 173                 return 1;
 174
 175         return 0;
 176 }
 177
 178 static bool assert_ksm_pages_count(long dupl_page_count)
 179 {
 180         unsigned long max_page_sharing, pages_sharing, pages_shared;
 181
 182         if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
 183             ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
 184             ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
 185                 return false;
 186
 187         /*
 188          * Since there must be at least 2 pages for merging and 1 page can be
 189          * shared with the limited number of pages (max_page_sharing), sometimes
 190          * there are 'leftover' pages that cannot be merged. For example, if there
 191          * are 11 pages and max_page_sharing = 10, then only 10 pages will be
 192          * merged and the 11th page won't be affected. As a result, when the number
 193          * of duplicate pages is divided by max_page_sharing and the remainder is 1,
 194          * pages_shared and pages_sharing values will be equal between dupl_page_count
 195          * and dupl_page_count - 1.
 196          */
 197         if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
 198                 if (pages_shared == dupl_page_count / max_page_sharing &&
 199                     pages_sharing == pages_shared * (max_page_sharing - 1))
 200                         return true;
 201         } else {
 202                 if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
 203                     pages_sharing == dupl_page_count - pages_shared)
 204                         return true;
 205         }
 206
 207         return false;
 208 }
 209
 210 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
 211 {
 212         if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
 213             ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
 214             ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
 215             ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
 216             ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
 217             ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
 218                            &ksm_sysfs->stable_node_chains_prune_millisecs) ||
 219             ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
 220                 return 1;
 221
 222         return 0;
 223 }
 224
 225 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
 226 {
 227         if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
 228             ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
 229             ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
 230             ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
 231             ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
 232             ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
 233                             ksm_sysfs->stable_node_chains_prune_millisecs) ||
 234             ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
 235                 return 1;
 236
 237         return 0;
 238 }
 239
 240 static int check_ksm_merge(int mapping, int prot, long page_count, int timeout, size_t page_size)
 241 {
 242         void *map_ptr;
 243         struct timespec start_time;
 244
 245         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 246                 perror("clock_gettime");
 247                 return KSFT_FAIL;
 248         }
 249
 250         /* fill pages with the same data and merge them */
 251         map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
 252         if (!map_ptr)
 253                 return KSFT_FAIL;
 254
 255         if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
 256                 goto err_out;
 257
 258         /* verify that the right number of pages are merged */
 259         if (assert_ksm_pages_count(page_count)) {
 260                 printf("OK\n");
 261                 munmap(map_ptr, page_size * page_count);
 262                 return KSFT_PASS;
 263         }
 264
 265 err_out:
 266         printf("Not OK\n");
 267         munmap(map_ptr, page_size * page_count);
 268         return KSFT_FAIL;
 269 }
 270
 271 static int check_ksm_unmerge(int mapping, int prot, int timeout, size_t page_size)
 272 {
 273         void *map_ptr;
 274         struct timespec start_time;
 275         int page_count = 2;
 276
 277         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 278                 perror("clock_gettime");
 279                 return KSFT_FAIL;
 280         }
 281
 282         /* fill pages with the same data and merge them */
 283         map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
 284         if (!map_ptr)
 285                 return KSFT_FAIL;
 286
 287         if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
 288                 goto err_out;
 289
 290         /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
 291         memset(map_ptr, '-', 1);
 292         memset(map_ptr + page_size, '+', 1);
 293
 294         /* get at least 1 scan, so KSM can detect that the pages were modified */
 295         if (ksm_do_scan(1, start_time, timeout))
 296                 goto err_out;
 297
 298         /* check that unmerging was successful and 0 pages are currently merged */
 299         if (assert_ksm_pages_count(0)) {
 300                 printf("OK\n");
 301                 munmap(map_ptr, page_size * page_count);
 302                 return KSFT_PASS;
 303         }
 304
 305 err_out:
 306         printf("Not OK\n");
 307         munmap(map_ptr, page_size * page_count);
 308         return KSFT_FAIL;
 309 }
 310
 311 static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int timeout,
 312                                      bool use_zero_pages, size_t page_size)
 313 {
 314         void *map_ptr;
 315         struct timespec start_time;
 316
 317         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 318                 perror("clock_gettime");
 319                 return KSFT_FAIL;
 320         }
 321
 322         if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
 323                 return KSFT_FAIL;
 324
 325         /* fill pages with zero and try to merge them */
 326         map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
 327         if (!map_ptr)
 328                 return KSFT_FAIL;
 329
 330         if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
 331                 goto err_out;
 332
 333        /*
 334         * verify that the right number of pages are merged:
 335         * 1) if use_zero_pages is set to 1, empty pages are merged
 336         *    with the kernel zero page instead of with each other;
 337         * 2) if use_zero_pages is set to 0, empty pages are not treated specially
 338         *    and merged as usual.
 339         */
 340         if (use_zero_pages && !assert_ksm_pages_count(0))
 341                 goto err_out;
 342         else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
 343                 goto err_out;
 344
 345         printf("OK\n");
 346         munmap(map_ptr, page_size * page_count);
 347         return KSFT_PASS;
 348
 349 err_out:
 350         printf("Not OK\n");
 351         munmap(map_ptr, page_size * page_count);
 352         return KSFT_FAIL;
 353 }
 354
 355 static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_across_nodes,
 356                                 size_t page_size)
 357 {
 358         void *numa1_map_ptr, *numa2_map_ptr;
 359         struct timespec start_time;
 360         int page_count = 2;
 361
 362         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 363                 perror("clock_gettime");
 364                 return KSFT_FAIL;
 365         }
 366
 367         if (numa_available() < 0) {
 368                 perror("NUMA support not enabled");
 369                 return KSFT_SKIP;
 370         }
 371         if (numa_max_node() < 1) {
 372                 printf("At least 2 NUMA nodes must be available\n");
 373                 return KSFT_SKIP;
 374         }
 375         if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
 376                 return KSFT_FAIL;
 377
 378         /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
 379         numa1_map_ptr = numa_alloc_onnode(page_size, 0);
 380         numa2_map_ptr = numa_alloc_onnode(page_size, 1);
 381         if (!numa1_map_ptr || !numa2_map_ptr) {
 382                 perror("numa_alloc_onnode");
 383                 return KSFT_FAIL;
 384         }
 385
 386         memset(numa1_map_ptr, '*', page_size);
 387         memset(numa2_map_ptr, '*', page_size);
 388
 389         /* try to merge the pages */
 390         if (ksm_merge_pages(numa1_map_ptr, page_size, start_time, timeout) ||
 391             ksm_merge_pages(numa2_map_ptr, page_size, start_time, timeout))
 392                 goto err_out;
 393
 394        /*
 395         * verify that the right number of pages are merged:
 396         * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
 397         * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
 398         *    only 1 unique page in each node and they can't be shared.
 399         */
 400         if (merge_across_nodes && !assert_ksm_pages_count(page_count))
 401                 goto err_out;
 402         else if (!merge_across_nodes && !assert_ksm_pages_count(0))
 403                 goto err_out;
 404
 405         numa_free(numa1_map_ptr, page_size);
 406         numa_free(numa2_map_ptr, page_size);
 407         printf("OK\n");
 408         return KSFT_PASS;
 409
 410 err_out:
 411         numa_free(numa1_map_ptr, page_size);
 412         numa_free(numa2_map_ptr, page_size);
 413         printf("Not OK\n");
 414         return KSFT_FAIL;
 415 }
 416
 417 static int ksm_merge_time(int mapping, int prot, int timeout, size_t map_size)
 418 {
 419         void *map_ptr;
 420         struct timespec start_time, end_time;
 421         unsigned long scan_time_ns;
 422
 423         map_size *= MB;
 424
 425         map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
 426         if (!map_ptr)
 427                 return KSFT_FAIL;
 428
 429         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 430                 perror("clock_gettime");
 431                 goto err_out;
 432         }
 433         if (ksm_merge_pages(map_ptr, map_size, start_time, timeout))
 434                 goto err_out;
 435         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 436                 perror("clock_gettime");
 437                 goto err_out;
 438         }
 439
 440         scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 441                        (end_time.tv_nsec - start_time.tv_nsec);
 442
 443         printf("Total size:    %lu MiB\n", map_size / MB);
 444         printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
 445                scan_time_ns % NSEC_PER_SEC);
 446         printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
 447                                                ((double)scan_time_ns / NSEC_PER_SEC));
 448
 449         munmap(map_ptr, map_size);
 450         return KSFT_PASS;
 451
 452 err_out:
 453         printf("Not OK\n");
 454         munmap(map_ptr, map_size);
 455         return KSFT_FAIL;
 456 }
 457
 458 int main(int argc, char *argv[])
 459 {
 460         int ret, opt;
 461         int prot = 0;
 462         int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
 463         long page_count = KSM_PAGE_COUNT_DEFAULT;
 464         size_t page_size = sysconf(_SC_PAGESIZE);
 465         struct ksm_sysfs ksm_sysfs_old;
 466         int test_name = CHECK_KSM_MERGE;
 467         bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
 468         bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
 469         long size_MB = 0;
 470
 471         while ((opt = getopt(argc, argv, "ha:p:l:z:m:s:MUZNP")) != -1) {
 472                 switch (opt) {
 473                 case 'a':
 474                         prot = str_to_prot(optarg);
 475                         break;
 476                 case 'p':
 477                         page_count = atol(optarg);
 478                         if (page_count <= 0) {
 479                                 printf("The number of pages must be greater than 0\n");
 480                                 return KSFT_FAIL;
 481                         }
 482                         break;
 483                 case 'l':
 484                         ksm_scan_limit_sec = atoi(optarg);
 485                         if (ksm_scan_limit_sec <= 0) {
 486                                 printf("Timeout value must be greater than 0\n");
 487                                 return KSFT_FAIL;
 488                         }
 489                         break;
 490                 case 'h':
 491                         print_help();
 492                         break;
 493                 case 'z':
 494                         if (strcmp(optarg, "0") == 0)
 495                                 use_zero_pages = 0;
 496                         else
 497                                 use_zero_pages = 1;
 498                         break;
 499                 case 'm':
 500                         if (strcmp(optarg, "0") == 0)
 501                                 merge_across_nodes = 0;
 502                         else
 503                                 merge_across_nodes = 1;
 504                         break;
 505                 case 's':
 506                         size_MB = atoi(optarg);
 507                         if (size_MB <= 0) {
 508                                 printf("Size must be greater than 0\n");
 509                                 return KSFT_FAIL;
 510                         }
 511                 case 'M':
 512                         break;
 513                 case 'U':
 514                         test_name = CHECK_KSM_UNMERGE;
 515                         break;
 516                 case 'Z':
 517                         test_name = CHECK_KSM_ZERO_PAGE_MERGE;
 518                         break;
 519                 case 'N':
 520                         test_name = CHECK_KSM_NUMA_MERGE;
 521                         break;
 522                 case 'P':
 523                         test_name = KSM_MERGE_TIME;
 524                         break;
 525                 default:
 526                         return KSFT_FAIL;
 527                 }
 528         }
 529
 530         if (prot == 0)
 531                 prot = str_to_prot(KSM_PROT_STR_DEFAULT);
 532
 533         if (access(KSM_SYSFS_PATH, F_OK)) {
 534                 printf("Config KSM not enabled\n");
 535                 return KSFT_SKIP;
 536         }
 537
 538         if (ksm_save_def(&ksm_sysfs_old)) {
 539                 printf("Cannot save default tunables\n");
 540                 return KSFT_FAIL;
 541         }
 542
 543         if (ksm_write_sysfs(KSM_FP("run"), 2) ||
 544             ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
 545             ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
 546             ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
 547                 return KSFT_FAIL;
 548
 549         switch (test_name) {
 550         case CHECK_KSM_MERGE:
 551                 ret = check_ksm_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
 552                                       ksm_scan_limit_sec, page_size);
 553                 break;
 554         case CHECK_KSM_UNMERGE:
 555                 ret = check_ksm_unmerge(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
 556                                         page_size);
 557                 break;
 558         case CHECK_KSM_ZERO_PAGE_MERGE:
 559                 ret = check_ksm_zero_page_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
 560                                                 ksm_scan_limit_sec, use_zero_pages, page_size);
 561                 break;
 562         case CHECK_KSM_NUMA_MERGE:
 563                 ret = check_ksm_numa_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
 564                                            merge_across_nodes, page_size);
 565                 break;
 566         case KSM_MERGE_TIME:
 567                 if (size_MB == 0) {
 568                         printf("Option '-s' is required.\n");
 569                         return KSFT_FAIL;
 570                 }
 571                 ret = ksm_merge_time(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
 572                                      size_MB);
 573                 break;
 574         }
 575
 576         if (ksm_restore(&ksm_sysfs_old)) {
 577                 printf("Cannot restore default tunables\n");
 578                 return KSFT_FAIL;
 579         }
 580
 581         return ret;
 582 }