tools/testing/selftests/rseq/param_test.c

   1 // SPDX-License-Identifier: LGPL-2.1
   2 #define _GNU_SOURCE
   3 #include <assert.h>
   4 #include <pthread.h>
   5 #include <sched.h>
   6 #include <stdint.h>
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <syscall.h>
  11 #include <unistd.h>
  12 #include <poll.h>
  13 #include <sys/types.h>
  14 #include <signal.h>
  15 #include <errno.h>
  16 #include <stddef.h>
  17
  18 static inline pid_t gettid(void)
  19 {
  20         return syscall(__NR_gettid);
  21 }
  22
  23 #define NR_INJECT       9
  24 static int loop_cnt[NR_INJECT + 1];
  25
  26 static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));
  27 static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));
  28 static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));
  29 static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));
  30 static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));
  31 static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used));
  32
  33 static int opt_modulo, verbose;
  34
  35 static int opt_yield, opt_signal, opt_sleep,
  36                 opt_disable_rseq, opt_threads = 200,
  37                 opt_disable_mod = 0, opt_test = 's', opt_mb = 0;
  38
  39 #ifndef RSEQ_SKIP_FASTPATH
  40 static long long opt_reps = 5000;
  41 #else
  42 static long long opt_reps = 100;
  43 #endif
  44
  45 static __thread __attribute__((tls_model("initial-exec")))
  46 unsigned int signals_delivered;
  47
  48 #ifndef BENCHMARK
  49
  50 static __thread __attribute__((tls_model("initial-exec"), unused))
  51 unsigned int yield_mod_cnt, nr_abort;
  52
  53 #define printf_verbose(fmt, ...)                        \
  54         do {                                            \
  55                 if (verbose)                            \
  56                         printf(fmt, ## __VA_ARGS__);    \
  57         } while (0)
  58
  59 #if defined(__x86_64__) || defined(__i386__)
  60
  61 #define INJECT_ASM_REG  "eax"
  62
  63 #define RSEQ_INJECT_CLOBBER \
  64         , INJECT_ASM_REG
  65
  66 #ifdef __i386__
  67
  68 #define RSEQ_INJECT_ASM(n) \
  69         "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
  70         "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
  71         "jz 333f\n\t" \
  72         "222:\n\t" \
  73         "dec %%" INJECT_ASM_REG "\n\t" \
  74         "jnz 222b\n\t" \
  75         "333:\n\t"
  76
  77 #elif defined(__x86_64__)
  78
  79 #define RSEQ_INJECT_ASM(n) \
  80         "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG "\n\t" \
  81         "mov (%%" INJECT_ASM_REG "), %%" INJECT_ASM_REG "\n\t" \
  82         "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
  83         "jz 333f\n\t" \
  84         "222:\n\t" \
  85         "dec %%" INJECT_ASM_REG "\n\t" \
  86         "jnz 222b\n\t" \
  87         "333:\n\t"
  88
  89 #else
  90 #error "Unsupported architecture"
  91 #endif
  92
  93 #elif defined(__ARMEL__)
  94
  95 #define RSEQ_INJECT_INPUT \
  96         , [loop_cnt_1]"m"(loop_cnt[1]) \
  97         , [loop_cnt_2]"m"(loop_cnt[2]) \
  98         , [loop_cnt_3]"m"(loop_cnt[3]) \
  99         , [loop_cnt_4]"m"(loop_cnt[4]) \
 100         , [loop_cnt_5]"m"(loop_cnt[5]) \
 101         , [loop_cnt_6]"m"(loop_cnt[6])
 102
 103 #define INJECT_ASM_REG  "r4"
 104
 105 #define RSEQ_INJECT_CLOBBER \
 106         , INJECT_ASM_REG
 107
 108 #define RSEQ_INJECT_ASM(n) \
 109         "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
 110         "cmp " INJECT_ASM_REG ", #0\n\t" \
 111         "beq 333f\n\t" \
 112         "222:\n\t" \
 113         "subs " INJECT_ASM_REG ", #1\n\t" \
 114         "bne 222b\n\t" \
 115         "333:\n\t"
 116
 117 #elif __PPC__
 118
 119 #define RSEQ_INJECT_INPUT \
 120         , [loop_cnt_1]"m"(loop_cnt[1]) \
 121         , [loop_cnt_2]"m"(loop_cnt[2]) \
 122         , [loop_cnt_3]"m"(loop_cnt[3]) \
 123         , [loop_cnt_4]"m"(loop_cnt[4]) \
 124         , [loop_cnt_5]"m"(loop_cnt[5]) \
 125         , [loop_cnt_6]"m"(loop_cnt[6])
 126
 127 #define INJECT_ASM_REG  "r18"
 128
 129 #define RSEQ_INJECT_CLOBBER \
 130         , INJECT_ASM_REG
 131
 132 #define RSEQ_INJECT_ASM(n) \
 133         "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
 134         "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
 135         "beq 333f\n\t" \
 136         "222:\n\t" \
 137         "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
 138         "bne 222b\n\t" \
 139         "333:\n\t"
 140
 141 #elif defined(__mips__)
 142
 143 #define RSEQ_INJECT_INPUT \
 144         , [loop_cnt_1]"m"(loop_cnt[1]) \
 145         , [loop_cnt_2]"m"(loop_cnt[2]) \
 146         , [loop_cnt_3]"m"(loop_cnt[3]) \
 147         , [loop_cnt_4]"m"(loop_cnt[4]) \
 148         , [loop_cnt_5]"m"(loop_cnt[5]) \
 149         , [loop_cnt_6]"m"(loop_cnt[6])
 150
 151 #define INJECT_ASM_REG  "$5"
 152
 153 #define RSEQ_INJECT_CLOBBER \
 154         , INJECT_ASM_REG
 155
 156 #define RSEQ_INJECT_ASM(n) \
 157         "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
 158         "beqz " INJECT_ASM_REG ", 333f\n\t" \
 159         "222:\n\t" \
 160         "addiu " INJECT_ASM_REG ", -1\n\t" \
 161         "bnez " INJECT_ASM_REG ", 222b\n\t" \
 162         "333:\n\t"
 163
 164 #else
 165 #error unsupported target
 166 #endif
 167
 168 #define RSEQ_INJECT_FAILED \
 169         nr_abort++;
 170
 171 #define RSEQ_INJECT_C(n) \
 172 { \
 173         int loc_i, loc_nr_loops = loop_cnt[n]; \
 174         \
 175         for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \
 176                 rseq_barrier(); \
 177         } \
 178         if (loc_nr_loops == -1 && opt_modulo) { \
 179                 if (yield_mod_cnt == opt_modulo - 1) { \
 180                         if (opt_sleep > 0) \
 181                                 poll(NULL, 0, opt_sleep); \
 182                         if (opt_yield) \
 183                                 sched_yield(); \
 184                         if (opt_signal) \
 185                                 raise(SIGUSR1); \
 186                         yield_mod_cnt = 0; \
 187                 } else { \
 188                         yield_mod_cnt++; \
 189                 } \
 190         } \
 191 }
 192
 193 #else
 194
 195 #define printf_verbose(fmt, ...)
 196
 197 #endif /* BENCHMARK */
 198
 199 #include "rseq.h"
 200
 201 struct percpu_lock_entry {
 202         intptr_t v;
 203 } __attribute__((aligned(128)));
 204
 205 struct percpu_lock {
 206         struct percpu_lock_entry c[CPU_SETSIZE];
 207 };
 208
 209 struct test_data_entry {
 210         intptr_t count;
 211 } __attribute__((aligned(128)));
 212
 213 struct spinlock_test_data {
 214         struct percpu_lock lock;
 215         struct test_data_entry c[CPU_SETSIZE];
 216 };
 217
 218 struct spinlock_thread_test_data {
 219         struct spinlock_test_data *data;
 220         long long reps;
 221         int reg;
 222 };
 223
 224 struct inc_test_data {
 225         struct test_data_entry c[CPU_SETSIZE];
 226 };
 227
 228 struct inc_thread_test_data {
 229         struct inc_test_data *data;
 230         long long reps;
 231         int reg;
 232 };
 233
 234 struct percpu_list_node {
 235         intptr_t data;
 236         struct percpu_list_node *next;
 237 };
 238
 239 struct percpu_list_entry {
 240         struct percpu_list_node *head;
 241 } __attribute__((aligned(128)));
 242
 243 struct percpu_list {
 244         struct percpu_list_entry c[CPU_SETSIZE];
 245 };
 246
 247 #define BUFFER_ITEM_PER_CPU     100
 248
 249 struct percpu_buffer_node {
 250         intptr_t data;
 251 };
 252
 253 struct percpu_buffer_entry {
 254         intptr_t offset;
 255         intptr_t buflen;
 256         struct percpu_buffer_node **array;
 257 } __attribute__((aligned(128)));
 258
 259 struct percpu_buffer {
 260         struct percpu_buffer_entry c[CPU_SETSIZE];
 261 };
 262
 263 #define MEMCPY_BUFFER_ITEM_PER_CPU      100
 264
 265 struct percpu_memcpy_buffer_node {
 266         intptr_t data1;
 267         uint64_t data2;
 268 };
 269
 270 struct percpu_memcpy_buffer_entry {
 271         intptr_t offset;
 272         intptr_t buflen;
 273         struct percpu_memcpy_buffer_node *array;
 274 } __attribute__((aligned(128)));
 275
 276 struct percpu_memcpy_buffer {
 277         struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];
 278 };
 279
 280 /* A simple percpu spinlock. Grabs lock on current cpu. */
 281 static int rseq_this_cpu_lock(struct percpu_lock *lock)
 282 {
 283         int cpu;
 284
 285         for (;;) {
 286                 int ret;
 287
 288                 cpu = rseq_cpu_start();
 289                 ret = rseq_cmpeqv_storev(&lock->c[cpu].v,
 290                                          0, 1, cpu);
 291                 if (rseq_likely(!ret))
 292                         break;
 293                 /* Retry if comparison fails or rseq aborts. */
 294         }
 295         /*
 296          * Acquire semantic when taking lock after control dependency.
 297          * Matches rseq_smp_store_release().
 298          */
 299         rseq_smp_acquire__after_ctrl_dep();
 300         return cpu;
 301 }
 302
 303 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)
 304 {
 305         assert(lock->c[cpu].v == 1);
 306         /*
 307          * Release lock, with release semantic. Matches
 308          * rseq_smp_acquire__after_ctrl_dep().
 309          */
 310         rseq_smp_store_release(&lock->c[cpu].v, 0);
 311 }
 312
 313 void *test_percpu_spinlock_thread(void *arg)
 314 {
 315         struct spinlock_thread_test_data *thread_data = arg;
 316         struct spinlock_test_data *data = thread_data->data;
 317         long long i, reps;
 318
 319         if (!opt_disable_rseq && thread_data->reg &&
 320             rseq_register_current_thread())
 321                 abort();
 322         reps = thread_data->reps;
 323         for (i = 0; i < reps; i++) {
 324                 int cpu = rseq_cpu_start();
 325
 326                 cpu = rseq_this_cpu_lock(&data->lock);
 327                 data->c[cpu].count++;
 328                 rseq_percpu_unlock(&data->lock, cpu);
 329 #ifndef BENCHMARK
 330                 if (i != 0 && !(i % (reps / 10)))
 331                         printf_verbose("tid %d: count %lld\n", (int) gettid(), i);
 332 #endif
 333         }
 334         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 335                        (int) gettid(), nr_abort, signals_delivered);
 336         if (!opt_disable_rseq && thread_data->reg &&
 337             rseq_unregister_current_thread())
 338                 abort();
 339         return NULL;
 340 }
 341
 342 /*
 343  * A simple test which implements a sharded counter using a per-cpu
 344  * lock.  Obviously real applications might prefer to simply use a
 345  * per-cpu increment; however, this is reasonable for a test and the
 346  * lock can be extended to synchronize more complicated operations.
 347  */
 348 void test_percpu_spinlock(void)
 349 {
 350         const int num_threads = opt_threads;
 351         int i, ret;
 352         uint64_t sum;
 353         pthread_t test_threads[num_threads];
 354         struct spinlock_test_data data;
 355         struct spinlock_thread_test_data thread_data[num_threads];
 356
 357         memset(&data, 0, sizeof(data));
 358         for (i = 0; i < num_threads; i++) {
 359                 thread_data[i].reps = opt_reps;
 360                 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
 361                         thread_data[i].reg = 1;
 362                 else
 363                         thread_data[i].reg = 0;
 364                 thread_data[i].data = &data;
 365                 ret = pthread_create(&test_threads[i], NULL,
 366                                      test_percpu_spinlock_thread,
 367                                      &thread_data[i]);
 368                 if (ret) {
 369                         errno = ret;
 370                         perror("pthread_create");
 371                         abort();
 372                 }
 373         }
 374
 375         for (i = 0; i < num_threads; i++) {
 376                 ret = pthread_join(test_threads[i], NULL);
 377                 if (ret) {
 378                         errno = ret;
 379                         perror("pthread_join");
 380                         abort();
 381                 }
 382         }
 383
 384         sum = 0;
 385         for (i = 0; i < CPU_SETSIZE; i++)
 386                 sum += data.c[i].count;
 387
 388         assert(sum == (uint64_t)opt_reps * num_threads);
 389 }
 390
 391 void *test_percpu_inc_thread(void *arg)
 392 {
 393         struct inc_thread_test_data *thread_data = arg;
 394         struct inc_test_data *data = thread_data->data;
 395         long long i, reps;
 396
 397         if (!opt_disable_rseq && thread_data->reg &&
 398             rseq_register_current_thread())
 399                 abort();
 400         reps = thread_data->reps;
 401         for (i = 0; i < reps; i++) {
 402                 int ret;
 403
 404                 do {
 405                         int cpu;
 406
 407                         cpu = rseq_cpu_start();
 408                         ret = rseq_addv(&data->c[cpu].count, 1, cpu);
 409                 } while (rseq_unlikely(ret));
 410 #ifndef BENCHMARK
 411                 if (i != 0 && !(i % (reps / 10)))
 412                         printf_verbose("tid %d: count %lld\n", (int) gettid(), i);
 413 #endif
 414         }
 415         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 416                        (int) gettid(), nr_abort, signals_delivered);
 417         if (!opt_disable_rseq && thread_data->reg &&
 418             rseq_unregister_current_thread())
 419                 abort();
 420         return NULL;
 421 }
 422
 423 void test_percpu_inc(void)
 424 {
 425         const int num_threads = opt_threads;
 426         int i, ret;
 427         uint64_t sum;
 428         pthread_t test_threads[num_threads];
 429         struct inc_test_data data;
 430         struct inc_thread_test_data thread_data[num_threads];
 431
 432         memset(&data, 0, sizeof(data));
 433         for (i = 0; i < num_threads; i++) {
 434                 thread_data[i].reps = opt_reps;
 435                 if (opt_disable_mod <= 0 || (i % opt_disable_mod))
 436                         thread_data[i].reg = 1;
 437                 else
 438                         thread_data[i].reg = 0;
 439                 thread_data[i].data = &data;
 440                 ret = pthread_create(&test_threads[i], NULL,
 441                                      test_percpu_inc_thread,
 442                                      &thread_data[i]);
 443                 if (ret) {
 444                         errno = ret;
 445                         perror("pthread_create");
 446                         abort();
 447                 }
 448         }
 449
 450         for (i = 0; i < num_threads; i++) {
 451                 ret = pthread_join(test_threads[i], NULL);
 452                 if (ret) {
 453                         errno = ret;
 454                         perror("pthread_join");
 455                         abort();
 456                 }
 457         }
 458
 459         sum = 0;
 460         for (i = 0; i < CPU_SETSIZE; i++)
 461                 sum += data.c[i].count;
 462
 463         assert(sum == (uint64_t)opt_reps * num_threads);
 464 }
 465
 466 void this_cpu_list_push(struct percpu_list *list,
 467                         struct percpu_list_node *node,
 468                         int *_cpu)
 469 {
 470         int cpu;
 471
 472         for (;;) {
 473                 intptr_t *targetptr, newval, expect;
 474                 int ret;
 475
 476                 cpu = rseq_cpu_start();
 477                 /* Load list->c[cpu].head with single-copy atomicity. */
 478                 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);
 479                 newval = (intptr_t)node;
 480                 targetptr = (intptr_t *)&list->c[cpu].head;
 481                 node->next = (struct percpu_list_node *)expect;
 482                 ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu);
 483                 if (rseq_likely(!ret))
 484                         break;
 485                 /* Retry if comparison fails or rseq aborts. */
 486         }
 487         if (_cpu)
 488                 *_cpu = cpu;
 489 }
 490
 491 /*
 492  * Unlike a traditional lock-less linked list; the availability of a
 493  * rseq primitive allows us to implement pop without concerns over
 494  * ABA-type races.
 495  */
 496 struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
 497                                            int *_cpu)
 498 {
 499         struct percpu_list_node *node = NULL;
 500         int cpu;
 501
 502         for (;;) {
 503                 struct percpu_list_node *head;
 504                 intptr_t *targetptr, expectnot, *load;
 505                 off_t offset;
 506                 int ret;
 507
 508                 cpu = rseq_cpu_start();
 509                 targetptr = (intptr_t *)&list->c[cpu].head;
 510                 expectnot = (intptr_t)NULL;
 511                 offset = offsetof(struct percpu_list_node, next);
 512                 load = (intptr_t *)&head;
 513                 ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot,
 514                                                    offset, load, cpu);
 515                 if (rseq_likely(!ret)) {
 516                         node = head;
 517                         break;
 518                 }
 519                 if (ret > 0)
 520                         break;
 521                 /* Retry if rseq aborts. */
 522         }
 523         if (_cpu)
 524                 *_cpu = cpu;
 525         return node;
 526 }
 527
 528 /*
 529  * __percpu_list_pop is not safe against concurrent accesses. Should
 530  * only be used on lists that are not concurrently modified.
 531  */
 532 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)
 533 {
 534         struct percpu_list_node *node;
 535
 536         node = list->c[cpu].head;
 537         if (!node)
 538                 return NULL;
 539         list->c[cpu].head = node->next;
 540         return node;
 541 }
 542
 543 void *test_percpu_list_thread(void *arg)
 544 {
 545         long long i, reps;
 546         struct percpu_list *list = (struct percpu_list *)arg;
 547
 548         if (!opt_disable_rseq && rseq_register_current_thread())
 549                 abort();
 550
 551         reps = opt_reps;
 552         for (i = 0; i < reps; i++) {
 553                 struct percpu_list_node *node;
 554
 555                 node = this_cpu_list_pop(list, NULL);
 556                 if (opt_yield)
 557                         sched_yield();  /* encourage shuffling */
 558                 if (node)
 559                         this_cpu_list_push(list, node, NULL);
 560         }
 561
 562         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 563                        (int) gettid(), nr_abort, signals_delivered);
 564         if (!opt_disable_rseq && rseq_unregister_current_thread())
 565                 abort();
 566
 567         return NULL;
 568 }
 569
 570 /* Simultaneous modification to a per-cpu linked list from many threads.  */
 571 void test_percpu_list(void)
 572 {
 573         const int num_threads = opt_threads;
 574         int i, j, ret;
 575         uint64_t sum = 0, expected_sum = 0;
 576         struct percpu_list list;
 577         pthread_t test_threads[num_threads];
 578         cpu_set_t allowed_cpus;
 579
 580         memset(&list, 0, sizeof(list));
 581
 582         /* Generate list entries for every usable cpu. */
 583         sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
 584         for (i = 0; i < CPU_SETSIZE; i++) {
 585                 if (!CPU_ISSET(i, &allowed_cpus))
 586                         continue;
 587                 for (j = 1; j <= 100; j++) {
 588                         struct percpu_list_node *node;
 589
 590                         expected_sum += j;
 591
 592                         node = malloc(sizeof(*node));
 593                         assert(node);
 594                         node->data = j;
 595                         node->next = list.c[i].head;
 596                         list.c[i].head = node;
 597                 }
 598         }
 599
 600         for (i = 0; i < num_threads; i++) {
 601                 ret = pthread_create(&test_threads[i], NULL,
 602                                      test_percpu_list_thread, &list);
 603                 if (ret) {
 604                         errno = ret;
 605                         perror("pthread_create");
 606                         abort();
 607                 }
 608         }
 609
 610         for (i = 0; i < num_threads; i++) {
 611                 ret = pthread_join(test_threads[i], NULL);
 612                 if (ret) {
 613                         errno = ret;
 614                         perror("pthread_join");
 615                         abort();
 616                 }
 617         }
 618
 619         for (i = 0; i < CPU_SETSIZE; i++) {
 620                 struct percpu_list_node *node;
 621
 622                 if (!CPU_ISSET(i, &allowed_cpus))
 623                         continue;
 624
 625                 while ((node = __percpu_list_pop(&list, i))) {
 626                         sum += node->data;
 627                         free(node);
 628                 }
 629         }
 630
 631         /*
 632          * All entries should now be accounted for (unless some external
 633          * actor is interfering with our allowed affinity while this
 634          * test is running).
 635          */
 636         assert(sum == expected_sum);
 637 }
 638
 639 bool this_cpu_buffer_push(struct percpu_buffer *buffer,
 640                           struct percpu_buffer_node *node,
 641                           int *_cpu)
 642 {
 643         bool result = false;
 644         int cpu;
 645
 646         for (;;) {
 647                 intptr_t *targetptr_spec, newval_spec;
 648                 intptr_t *targetptr_final, newval_final;
 649                 intptr_t offset;
 650                 int ret;
 651
 652                 cpu = rseq_cpu_start();
 653                 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
 654                 if (offset == buffer->c[cpu].buflen)
 655                         break;
 656                 newval_spec = (intptr_t)node;
 657                 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];
 658                 newval_final = offset + 1;
 659                 targetptr_final = &buffer->c[cpu].offset;
 660                 if (opt_mb)
 661                         ret = rseq_cmpeqv_trystorev_storev_release(
 662                                 targetptr_final, offset, targetptr_spec,
 663                                 newval_spec, newval_final, cpu);
 664                 else
 665                         ret = rseq_cmpeqv_trystorev_storev(targetptr_final,
 666                                 offset, targetptr_spec, newval_spec,
 667                                 newval_final, cpu);
 668                 if (rseq_likely(!ret)) {
 669                         result = true;
 670                         break;
 671                 }
 672                 /* Retry if comparison fails or rseq aborts. */
 673         }
 674         if (_cpu)
 675                 *_cpu = cpu;
 676         return result;
 677 }
 678
 679 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,
 680                                                int *_cpu)
 681 {
 682         struct percpu_buffer_node *head;
 683         int cpu;
 684
 685         for (;;) {
 686                 intptr_t *targetptr, newval;
 687                 intptr_t offset;
 688                 int ret;
 689
 690                 cpu = rseq_cpu_start();
 691                 /* Load offset with single-copy atomicity. */
 692                 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
 693                 if (offset == 0) {
 694                         head = NULL;
 695                         break;
 696                 }
 697                 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);
 698                 newval = offset - 1;
 699                 targetptr = (intptr_t *)&buffer->c[cpu].offset;
 700                 ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset,
 701                         (intptr_t *)&buffer->c[cpu].array[offset - 1],
 702                         (intptr_t)head, newval, cpu);
 703                 if (rseq_likely(!ret))
 704                         break;
 705                 /* Retry if comparison fails or rseq aborts. */
 706         }
 707         if (_cpu)
 708                 *_cpu = cpu;
 709         return head;
 710 }
 711
 712 /*
 713  * __percpu_buffer_pop is not safe against concurrent accesses. Should
 714  * only be used on buffers that are not concurrently modified.
 715  */
 716 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,
 717                                                int cpu)
 718 {
 719         struct percpu_buffer_node *head;
 720         intptr_t offset;
 721
 722         offset = buffer->c[cpu].offset;
 723         if (offset == 0)
 724                 return NULL;
 725         head = buffer->c[cpu].array[offset - 1];
 726         buffer->c[cpu].offset = offset - 1;
 727         return head;
 728 }
 729
 730 void *test_percpu_buffer_thread(void *arg)
 731 {
 732         long long i, reps;
 733         struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
 734
 735         if (!opt_disable_rseq && rseq_register_current_thread())
 736                 abort();
 737
 738         reps = opt_reps;
 739         for (i = 0; i < reps; i++) {
 740                 struct percpu_buffer_node *node;
 741
 742                 node = this_cpu_buffer_pop(buffer, NULL);
 743                 if (opt_yield)
 744                         sched_yield();  /* encourage shuffling */
 745                 if (node) {
 746                         if (!this_cpu_buffer_push(buffer, node, NULL)) {
 747                                 /* Should increase buffer size. */
 748                                 abort();
 749                         }
 750                 }
 751         }
 752
 753         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 754                        (int) gettid(), nr_abort, signals_delivered);
 755         if (!opt_disable_rseq && rseq_unregister_current_thread())
 756                 abort();
 757
 758         return NULL;
 759 }
 760
 761 /* Simultaneous modification to a per-cpu buffer from many threads.  */
 762 void test_percpu_buffer(void)
 763 {
 764         const int num_threads = opt_threads;
 765         int i, j, ret;
 766         uint64_t sum = 0, expected_sum = 0;
 767         struct percpu_buffer buffer;
 768         pthread_t test_threads[num_threads];
 769         cpu_set_t allowed_cpus;
 770
 771         memset(&buffer, 0, sizeof(buffer));
 772
 773         /* Generate list entries for every usable cpu. */
 774         sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
 775         for (i = 0; i < CPU_SETSIZE; i++) {
 776                 if (!CPU_ISSET(i, &allowed_cpus))
 777                         continue;
 778                 /* Worse-case is every item in same CPU. */
 779                 buffer.c[i].array =
 780                         malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
 781                                BUFFER_ITEM_PER_CPU);
 782                 assert(buffer.c[i].array);
 783                 buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;
 784                 for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) {
 785                         struct percpu_buffer_node *node;
 786
 787                         expected_sum += j;
 788
 789                         /*
 790                          * We could theoretically put the word-sized
 791                          * "data" directly in the buffer. However, we
 792                          * want to model objects that would not fit
 793                          * within a single word, so allocate an object
 794                          * for each node.
 795                          */
 796                         node = malloc(sizeof(*node));
 797                         assert(node);
 798                         node->data = j;
 799                         buffer.c[i].array[j - 1] = node;
 800                         buffer.c[i].offset++;
 801                 }
 802         }
 803
 804         for (i = 0; i < num_threads; i++) {
 805                 ret = pthread_create(&test_threads[i], NULL,
 806                                      test_percpu_buffer_thread, &buffer);
 807                 if (ret) {
 808                         errno = ret;
 809                         perror("pthread_create");
 810                         abort();
 811                 }
 812         }
 813
 814         for (i = 0; i < num_threads; i++) {
 815                 ret = pthread_join(test_threads[i], NULL);
 816                 if (ret) {
 817                         errno = ret;
 818                         perror("pthread_join");
 819                         abort();
 820                 }
 821         }
 822
 823         for (i = 0; i < CPU_SETSIZE; i++) {
 824                 struct percpu_buffer_node *node;
 825
 826                 if (!CPU_ISSET(i, &allowed_cpus))
 827                         continue;
 828
 829                 while ((node = __percpu_buffer_pop(&buffer, i))) {
 830                         sum += node->data;
 831                         free(node);
 832                 }
 833                 free(buffer.c[i].array);
 834         }
 835
 836         /*
 837          * All entries should now be accounted for (unless some external
 838          * actor is interfering with our allowed affinity while this
 839          * test is running).
 840          */
 841         assert(sum == expected_sum);
 842 }
 843
 844 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,
 845                                  struct percpu_memcpy_buffer_node item,
 846                                  int *_cpu)
 847 {
 848         bool result = false;
 849         int cpu;
 850
 851         for (;;) {
 852                 intptr_t *targetptr_final, newval_final, offset;
 853                 char *destptr, *srcptr;
 854                 size_t copylen;
 855                 int ret;
 856
 857                 cpu = rseq_cpu_start();
 858                 /* Load offset with single-copy atomicity. */
 859                 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
 860                 if (offset == buffer->c[cpu].buflen)
 861                         break;
 862                 destptr = (char *)&buffer->c[cpu].array[offset];
 863                 srcptr = (char *)&item;
 864                 /* copylen must be <= 4kB. */
 865                 copylen = sizeof(item);
 866                 newval_final = offset + 1;
 867                 targetptr_final = &buffer->c[cpu].offset;
 868                 if (opt_mb)
 869                         ret = rseq_cmpeqv_trymemcpy_storev_release(
 870                                 targetptr_final, offset,
 871                                 destptr, srcptr, copylen,
 872                                 newval_final, cpu);
 873                 else
 874                         ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final,
 875                                 offset, destptr, srcptr, copylen,
 876                                 newval_final, cpu);
 877                 if (rseq_likely(!ret)) {
 878                         result = true;
 879                         break;
 880                 }
 881                 /* Retry if comparison fails or rseq aborts. */
 882         }
 883         if (_cpu)
 884                 *_cpu = cpu;
 885         return result;
 886 }
 887
 888 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
 889                                 struct percpu_memcpy_buffer_node *item,
 890                                 int *_cpu)
 891 {
 892         bool result = false;
 893         int cpu;
 894
 895         for (;;) {
 896                 intptr_t *targetptr_final, newval_final, offset;
 897                 char *destptr, *srcptr;
 898                 size_t copylen;
 899                 int ret;
 900
 901                 cpu = rseq_cpu_start();
 902                 /* Load offset with single-copy atomicity. */
 903                 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
 904                 if (offset == 0)
 905                         break;
 906                 destptr = (char *)item;
 907                 srcptr = (char *)&buffer->c[cpu].array[offset - 1];
 908                 /* copylen must be <= 4kB. */
 909                 copylen = sizeof(*item);
 910                 newval_final = offset - 1;
 911                 targetptr_final = &buffer->c[cpu].offset;
 912                 ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final,
 913                         offset, destptr, srcptr, copylen,
 914                         newval_final, cpu);
 915                 if (rseq_likely(!ret)) {
 916                         result = true;
 917                         break;
 918                 }
 919                 /* Retry if comparison fails or rseq aborts. */
 920         }
 921         if (_cpu)
 922                 *_cpu = cpu;
 923         return result;
 924 }
 925
 926 /*
 927  * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
 928  * only be used on buffers that are not concurrently modified.
 929  */
 930 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
 931                                 struct percpu_memcpy_buffer_node *item,
 932                                 int cpu)
 933 {
 934         intptr_t offset;
 935
 936         offset = buffer->c[cpu].offset;
 937         if (offset == 0)
 938                 return false;
 939         memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));
 940         buffer->c[cpu].offset = offset - 1;
 941         return true;
 942 }
 943
 944 void *test_percpu_memcpy_buffer_thread(void *arg)
 945 {
 946         long long i, reps;
 947         struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
 948
 949         if (!opt_disable_rseq && rseq_register_current_thread())
 950                 abort();
 951
 952         reps = opt_reps;
 953         for (i = 0; i < reps; i++) {
 954                 struct percpu_memcpy_buffer_node item;
 955                 bool result;
 956
 957                 result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL);
 958                 if (opt_yield)
 959                         sched_yield();  /* encourage shuffling */
 960                 if (result) {
 961                         if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) {
 962                                 /* Should increase buffer size. */
 963                                 abort();
 964                         }
 965                 }
 966         }
 967
 968         printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
 969                        (int) gettid(), nr_abort, signals_delivered);
 970         if (!opt_disable_rseq && rseq_unregister_current_thread())
 971                 abort();
 972
 973         return NULL;
 974 }
 975
 976 /* Simultaneous modification to a per-cpu buffer from many threads.  */
 977 void test_percpu_memcpy_buffer(void)
 978 {
 979         const int num_threads = opt_threads;
 980         int i, j, ret;
 981         uint64_t sum = 0, expected_sum = 0;
 982         struct percpu_memcpy_buffer buffer;
 983         pthread_t test_threads[num_threads];
 984         cpu_set_t allowed_cpus;
 985
 986         memset(&buffer, 0, sizeof(buffer));
 987
 988         /* Generate list entries for every usable cpu. */
 989         sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
 990         for (i = 0; i < CPU_SETSIZE; i++) {
 991                 if (!CPU_ISSET(i, &allowed_cpus))
 992                         continue;
 993                 /* Worse-case is every item in same CPU. */
 994                 buffer.c[i].array =
 995                         malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
 996                                MEMCPY_BUFFER_ITEM_PER_CPU);
 997                 assert(buffer.c[i].array);
 998                 buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;
 999                 for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) {
1000                         expected_sum += 2 * j + 1;
1001
1002                         /*
1003                          * We could theoretically put the word-sized
1004                          * "data" directly in the buffer. However, we
1005                          * want to model objects that would not fit
1006                          * within a single word, so allocate an object
1007                          * for each node.
1008                          */
1009                         buffer.c[i].array[j - 1].data1 = j;
1010                         buffer.c[i].array[j - 1].data2 = j + 1;
1011                         buffer.c[i].offset++;
1012                 }
1013         }
1014
1015         for (i = 0; i < num_threads; i++) {
1016                 ret = pthread_create(&test_threads[i], NULL,
1017                                      test_percpu_memcpy_buffer_thread,
1018                                      &buffer);
1019                 if (ret) {
1020                         errno = ret;
1021                         perror("pthread_create");
1022                         abort();
1023                 }
1024         }
1025
1026         for (i = 0; i < num_threads; i++) {
1027                 ret = pthread_join(test_threads[i], NULL);
1028                 if (ret) {
1029                         errno = ret;
1030                         perror("pthread_join");
1031                         abort();
1032                 }
1033         }
1034
1035         for (i = 0; i < CPU_SETSIZE; i++) {
1036                 struct percpu_memcpy_buffer_node item;
1037
1038                 if (!CPU_ISSET(i, &allowed_cpus))
1039                         continue;
1040
1041                 while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {
1042                         sum += item.data1;
1043                         sum += item.data2;
1044                 }
1045                 free(buffer.c[i].array);
1046         }
1047
1048         /*
1049          * All entries should now be accounted for (unless some external
1050          * actor is interfering with our allowed affinity while this
1051          * test is running).
1052          */
1053         assert(sum == expected_sum);
1054 }
1055
1056 static void test_signal_interrupt_handler(int signo)
1057 {
1058         signals_delivered++;
1059 }
1060
1061 static int set_signal_handler(void)
1062 {
1063         int ret = 0;
1064         struct sigaction sa;
1065         sigset_t sigset;
1066
1067         ret = sigemptyset(&sigset);
1068         if (ret < 0) {
1069                 perror("sigemptyset");
1070                 return ret;
1071         }
1072
1073         sa.sa_handler = test_signal_interrupt_handler;
1074         sa.sa_mask = sigset;
1075         sa.sa_flags = 0;
1076         ret = sigaction(SIGUSR1, &sa, NULL);
1077         if (ret < 0) {
1078                 perror("sigaction");
1079                 return ret;
1080         }
1081
1082         printf_verbose("Signal handler set for SIGUSR1\n");
1083
1084         return ret;
1085 }
1086
1087 static void show_usage(int argc, char **argv)
1088 {
1089         printf("Usage : %s <OPTIONS>\n",
1090                 argv[0]);
1091         printf("OPTIONS:\n");
1092         printf("        [-1 loops] Number of loops for delay injection 1\n");
1093         printf("        [-2 loops] Number of loops for delay injection 2\n");
1094         printf("        [-3 loops] Number of loops for delay injection 3\n");
1095         printf("        [-4 loops] Number of loops for delay injection 4\n");
1096         printf("        [-5 loops] Number of loops for delay injection 5\n");
1097         printf("        [-6 loops] Number of loops for delay injection 6\n");
1098         printf("        [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1099         printf("        [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1100         printf("        [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1101         printf("        [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1102         printf("        [-y] Yield\n");
1103         printf("        [-k] Kill thread with signal\n");
1104         printf("        [-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1105         printf("        [-t N] Number of threads (default 200)\n");
1106         printf("        [-r N] Number of repetitions per thread (default 5000)\n");
1107         printf("        [-d] Disable rseq system call (no initialization)\n");
1108         printf("        [-D M] Disable rseq for each M threads\n");
1109         printf("        [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n");
1110         printf("        [-M] Push into buffer and memcpy buffer with memory barriers.\n");
1111         printf("        [-v] Verbose output.\n");
1112         printf("        [-h] Show this help.\n");
1113         printf("\n");
1114 }
1115
1116 int main(int argc, char **argv)
1117 {
1118         int i;
1119
1120         for (i = 1; i < argc; i++) {
1121                 if (argv[i][0] != '-')
1122                         continue;
1123                 switch (argv[i][1]) {
1124                 case '1':
1125                 case '2':
1126                 case '3':
1127                 case '4':
1128                 case '5':
1129                 case '6':
1130                 case '7':
1131                 case '8':
1132                 case '9':
1133                         if (argc < i + 2) {
1134                                 show_usage(argc, argv);
1135                                 goto error;
1136                         }
1137                         loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);
1138                         i++;
1139                         break;
1140                 case 'm':
1141                         if (argc < i + 2) {
1142                                 show_usage(argc, argv);
1143                                 goto error;
1144                         }
1145                         opt_modulo = atol(argv[i + 1]);
1146                         if (opt_modulo < 0) {
1147                                 show_usage(argc, argv);
1148                                 goto error;
1149                         }
1150                         i++;
1151                         break;
1152                 case 's':
1153                         if (argc < i + 2) {
1154                                 show_usage(argc, argv);
1155                                 goto error;
1156                         }
1157                         opt_sleep = atol(argv[i + 1]);
1158                         if (opt_sleep < 0) {
1159                                 show_usage(argc, argv);
1160                                 goto error;
1161                         }
1162                         i++;
1163                         break;
1164                 case 'y':
1165                         opt_yield = 1;
1166                         break;
1167                 case 'k':
1168                         opt_signal = 1;
1169                         break;
1170                 case 'd':
1171                         opt_disable_rseq = 1;
1172                         break;
1173                 case 'D':
1174                         if (argc < i + 2) {
1175                                 show_usage(argc, argv);
1176                                 goto error;
1177                         }
1178                         opt_disable_mod = atol(argv[i + 1]);
1179                         if (opt_disable_mod < 0) {
1180                                 show_usage(argc, argv);
1181                                 goto error;
1182                         }
1183                         i++;
1184                         break;
1185                 case 't':
1186                         if (argc < i + 2) {
1187                                 show_usage(argc, argv);
1188                                 goto error;
1189                         }
1190                         opt_threads = atol(argv[i + 1]);
1191                         if (opt_threads < 0) {
1192                                 show_usage(argc, argv);
1193                                 goto error;
1194                         }
1195                         i++;
1196                         break;
1197                 case 'r':
1198                         if (argc < i + 2) {
1199                                 show_usage(argc, argv);
1200                                 goto error;
1201                         }
1202                         opt_reps = atoll(argv[i + 1]);
1203                         if (opt_reps < 0) {
1204                                 show_usage(argc, argv);
1205                                 goto error;
1206                         }
1207                         i++;
1208                         break;
1209                 case 'h':
1210                         show_usage(argc, argv);
1211                         goto end;
1212                 case 'T':
1213                         if (argc < i + 2) {
1214                                 show_usage(argc, argv);
1215                                 goto error;
1216                         }
1217                         opt_test = *argv[i + 1];
1218                         switch (opt_test) {
1219                         case 's':
1220                         case 'l':
1221                         case 'i':
1222                         case 'b':
1223                         case 'm':
1224                                 break;
1225                         default:
1226                                 show_usage(argc, argv);
1227                                 goto error;
1228                         }
1229                         i++;
1230                         break;
1231                 case 'v':
1232                         verbose = 1;
1233                         break;
1234                 case 'M':
1235                         opt_mb = 1;
1236                         break;
1237                 default:
1238                         show_usage(argc, argv);
1239                         goto error;
1240                 }
1241         }
1242
1243         loop_cnt_1 = loop_cnt[1];
1244         loop_cnt_2 = loop_cnt[2];
1245         loop_cnt_3 = loop_cnt[3];
1246         loop_cnt_4 = loop_cnt[4];
1247         loop_cnt_5 = loop_cnt[5];
1248         loop_cnt_6 = loop_cnt[6];
1249
1250         if (set_signal_handler())
1251                 goto error;
1252
1253         if (!opt_disable_rseq && rseq_register_current_thread())
1254                 goto error;
1255         switch (opt_test) {
1256         case 's':
1257                 printf_verbose("spinlock\n");
1258                 test_percpu_spinlock();
1259                 break;
1260         case 'l':
1261                 printf_verbose("linked list\n");
1262                 test_percpu_list();
1263                 break;
1264         case 'b':
1265                 printf_verbose("buffer\n");
1266                 test_percpu_buffer();
1267                 break;
1268         case 'm':
1269                 printf_verbose("memcpy buffer\n");
1270                 test_percpu_memcpy_buffer();
1271                 break;
1272         case 'i':
1273                 printf_verbose("counter increment\n");
1274                 test_percpu_inc();
1275                 break;
1276         }
1277         if (!opt_disable_rseq && rseq_unregister_current_thread())
1278                 abort();
1279 end:
1280         return 0;
1281
1282 error:
1283         return -1;
1284 }