kernel/torture.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Common functions for in-kernel torture tests.
   4  *
   5  * Copyright (C) IBM Corporation, 2014
   6  *
   7  * Author: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *      Based on kernel/rcu/torture.c.
   9  */
  10
  11 #define pr_fmt(fmt) fmt
  12
  13 #include <linux/types.h>
  14 #include <linux/kernel.h>
  15 #include <linux/init.h>
  16 #include <linux/module.h>
  17 #include <linux/kthread.h>
  18 #include <linux/err.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/smp.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/sched.h>
  23 #include <linux/sched/clock.h>
  24 #include <linux/atomic.h>
  25 #include <linux/bitops.h>
  26 #include <linux/completion.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/percpu.h>
  29 #include <linux/notifier.h>
  30 #include <linux/reboot.h>
  31 #include <linux/freezer.h>
  32 #include <linux/cpu.h>
  33 #include <linux/delay.h>
  34 #include <linux/stat.h>
  35 #include <linux/slab.h>
  36 #include <linux/trace_clock.h>
  37 #include <linux/ktime.h>
  38 #include <asm/byteorder.h>
  39 #include <linux/torture.h>
  40 #include "rcu/rcu.h"
  41
  42 MODULE_LICENSE("GPL");
  43 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
  44
  45 static bool disable_onoff_at_boot;
  46 module_param(disable_onoff_at_boot, bool, 0444);
  47
  48 static bool ftrace_dump_at_shutdown;
  49 module_param(ftrace_dump_at_shutdown, bool, 0444);
  50
  51 static int verbose_sleep_frequency;
  52 module_param(verbose_sleep_frequency, int, 0444);
  53
  54 static int verbose_sleep_duration = 1;
  55 module_param(verbose_sleep_duration, int, 0444);
  56
  57 static char *torture_type;
  58 static int verbose;
  59
  60 /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
  61 #define FULLSTOP_DONTSTOP 0     /* Normal operation. */
  62 #define FULLSTOP_SHUTDOWN 1     /* System shutdown with torture running. */
  63 #define FULLSTOP_RMMOD    2     /* Normal rmmod of torture. */
  64 static int fullstop = FULLSTOP_RMMOD;
  65 static DEFINE_MUTEX(fullstop_mutex);
  66
  67 static atomic_t verbose_sleep_counter;
  68
  69 /*
  70  * Sleep if needed from VERBOSE_TOROUT*().
  71  */
  72 void verbose_torout_sleep(void)
  73 {
  74         if (verbose_sleep_frequency > 0 &&
  75             verbose_sleep_duration > 0 &&
  76             !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency))
  77                 schedule_timeout_uninterruptible(verbose_sleep_duration);
  78 }
  79 EXPORT_SYMBOL_GPL(verbose_torout_sleep);
  80
  81 /*
  82  * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit
  83  * nanosecond random fuzz.  This function and its friends desynchronize
  84  * testing from the timer wheel.
  85  */
  86 int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp)
  87 {
  88         ktime_t hto = baset_ns;
  89
  90         if (trsp)
  91                 hto += (torture_random(trsp) >> 3) % fuzzt_ns;
  92         set_current_state(TASK_UNINTERRUPTIBLE);
  93         return schedule_hrtimeout(&hto, HRTIMER_MODE_REL);
  94 }
  95 EXPORT_SYMBOL_GPL(torture_hrtimeout_ns);
  96
  97 /*
  98  * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit
  99  * nanosecond (not microsecond!) random fuzz.
 100  */
 101 int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp)
 102 {
 103         ktime_t baset_ns = baset_us * NSEC_PER_USEC;
 104
 105         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 106 }
 107 EXPORT_SYMBOL_GPL(torture_hrtimeout_us);
 108
 109 /*
 110  * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
 111  * microsecond (not millisecond!) random fuzz.
 112  */
 113 int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp)
 114 {
 115         ktime_t baset_ns = baset_ms * NSEC_PER_MSEC;
 116         u32 fuzzt_ns;
 117
 118         if ((u32)~0U / NSEC_PER_USEC < fuzzt_us)
 119                 fuzzt_ns = (u32)~0U;
 120         else
 121                 fuzzt_ns = fuzzt_us * NSEC_PER_USEC;
 122         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 123 }
 124 EXPORT_SYMBOL_GPL(torture_hrtimeout_ms);
 125
 126 /*
 127  * Schedule a high-resolution-timer sleep in jiffies, with an
 128  * implied one-jiffy random fuzz.  This is intended to replace calls to
 129  * schedule_timeout_interruptible() and friends.
 130  */
 131 int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp)
 132 {
 133         ktime_t baset_ns = jiffies_to_nsecs(baset_j);
 134
 135         return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp);
 136 }
 137 EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies);
 138
 139 /*
 140  * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
 141  * millisecond (not second!) random fuzz.
 142  */
 143 int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp)
 144 {
 145         ktime_t baset_ns = baset_s * NSEC_PER_SEC;
 146         u32 fuzzt_ns;
 147
 148         if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms)
 149                 fuzzt_ns = (u32)~0U;
 150         else
 151                 fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC;
 152         return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp);
 153 }
 154 EXPORT_SYMBOL_GPL(torture_hrtimeout_s);
 155
 156 #ifdef CONFIG_HOTPLUG_CPU
 157
 158 /*
 159  * Variables for online-offline handling.  Only present if CPU hotplug
 160  * is enabled, otherwise does nothing.
 161  */
 162
 163 static struct task_struct *onoff_task;
 164 static long onoff_holdoff;
 165 static long onoff_interval;
 166 static torture_ofl_func *onoff_f;
 167 static long n_offline_attempts;
 168 static long n_offline_successes;
 169 static unsigned long sum_offline;
 170 static int min_offline = -1;
 171 static int max_offline;
 172 static long n_online_attempts;
 173 static long n_online_successes;
 174 static unsigned long sum_online;
 175 static int min_online = -1;
 176 static int max_online;
 177
 178 /*
 179  * Attempt to take a CPU offline.  Return false if the CPU is already
 180  * offline or if it is not subject to CPU-hotplug operations.  The
 181  * caller can detect other failures by looking at the statistics.
 182  */
 183 bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
 184                      unsigned long *sum_offl, int *min_offl, int *max_offl)
 185 {
 186         unsigned long delta;
 187         int ret;
 188         char *s;
 189         unsigned long starttime;
 190
 191         if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
 192                 return false;
 193         if (num_online_cpus() <= 1)
 194                 return false;  /* Can't offline the last CPU. */
 195
 196         if (verbose > 1)
 197                 pr_alert("%s" TORTURE_FLAG
 198                          "torture_onoff task: offlining %d\n",
 199                          torture_type, cpu);
 200         starttime = jiffies;
 201         (*n_offl_attempts)++;
 202         ret = remove_cpu(cpu);
 203         if (ret) {
 204                 s = "";
 205                 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
 206                         // PCI probe frequently disables hotplug during boot.
 207                         (*n_offl_attempts)--;
 208                         s = " (-EBUSY forgiven during boot)";
 209                 }
 210                 if (verbose)
 211                         pr_alert("%s" TORTURE_FLAG
 212                                  "torture_onoff task: offline %d failed%s: errno %d\n",
 213                                  torture_type, cpu, s, ret);
 214         } else {
 215                 if (verbose > 1)
 216                         pr_alert("%s" TORTURE_FLAG
 217                                  "torture_onoff task: offlined %d\n",
 218                                  torture_type, cpu);
 219                 if (onoff_f)
 220                         onoff_f();
 221                 (*n_offl_successes)++;
 222                 delta = jiffies - starttime;
 223                 *sum_offl += delta;
 224                 if (*min_offl < 0) {
 225                         *min_offl = delta;
 226                         *max_offl = delta;
 227                 }
 228                 if (*min_offl > delta)
 229                         *min_offl = delta;
 230                 if (*max_offl < delta)
 231                         *max_offl = delta;
 232         }
 233
 234         return true;
 235 }
 236 EXPORT_SYMBOL_GPL(torture_offline);
 237
 238 /*
 239  * Attempt to bring a CPU online.  Return false if the CPU is already
 240  * online or if it is not subject to CPU-hotplug operations.  The
 241  * caller can detect other failures by looking at the statistics.
 242  */
 243 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
 244                     unsigned long *sum_onl, int *min_onl, int *max_onl)
 245 {
 246         unsigned long delta;
 247         int ret;
 248         char *s;
 249         unsigned long starttime;
 250
 251         if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
 252                 return false;
 253
 254         if (verbose > 1)
 255                 pr_alert("%s" TORTURE_FLAG
 256                          "torture_onoff task: onlining %d\n",
 257                          torture_type, cpu);
 258         starttime = jiffies;
 259         (*n_onl_attempts)++;
 260         ret = add_cpu(cpu);
 261         if (ret) {
 262                 s = "";
 263                 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
 264                         // PCI probe frequently disables hotplug during boot.
 265                         (*n_onl_attempts)--;
 266                         s = " (-EBUSY forgiven during boot)";
 267                 }
 268                 if (verbose)
 269                         pr_alert("%s" TORTURE_FLAG
 270                                  "torture_onoff task: online %d failed%s: errno %d\n",
 271                                  torture_type, cpu, s, ret);
 272         } else {
 273                 if (verbose > 1)
 274                         pr_alert("%s" TORTURE_FLAG
 275                                  "torture_onoff task: onlined %d\n",
 276                                  torture_type, cpu);
 277                 (*n_onl_successes)++;
 278                 delta = jiffies - starttime;
 279                 *sum_onl += delta;
 280                 if (*min_onl < 0) {
 281                         *min_onl = delta;
 282                         *max_onl = delta;
 283                 }
 284                 if (*min_onl > delta)
 285                         *min_onl = delta;
 286                 if (*max_onl < delta)
 287                         *max_onl = delta;
 288         }
 289
 290         return true;
 291 }
 292 EXPORT_SYMBOL_GPL(torture_online);
 293
 294 /*
 295  * Execute random CPU-hotplug operations at the interval specified
 296  * by the onoff_interval.
 297  */
 298 static int
 299 torture_onoff(void *arg)
 300 {
 301         int cpu;
 302         int maxcpu = -1;
 303         DEFINE_TORTURE_RANDOM(rand);
 304         int ret;
 305
 306         VERBOSE_TOROUT_STRING("torture_onoff task started");
 307         for_each_online_cpu(cpu)
 308                 maxcpu = cpu;
 309         WARN_ON(maxcpu < 0);
 310         if (!IS_MODULE(CONFIG_TORTURE_TEST)) {
 311                 for_each_possible_cpu(cpu) {
 312                         if (cpu_online(cpu))
 313                                 continue;
 314                         ret = add_cpu(cpu);
 315                         if (ret && verbose) {
 316                                 pr_alert("%s" TORTURE_FLAG
 317                                          "%s: Initial online %d: errno %d\n",
 318                                          __func__, torture_type, cpu, ret);
 319                         }
 320                 }
 321         }
 322
 323         if (maxcpu == 0) {
 324                 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled");
 325                 goto stop;
 326         }
 327
 328         if (onoff_holdoff > 0) {
 329                 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
 330                 schedule_timeout_interruptible(onoff_holdoff);
 331                 VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
 332         }
 333         while (!torture_must_stop()) {
 334                 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
 335                         schedule_timeout_interruptible(HZ / 10);
 336                         continue;
 337                 }
 338                 cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);
 339                 if (!torture_offline(cpu,
 340                                      &n_offline_attempts, &n_offline_successes,
 341                                      &sum_offline, &min_offline, &max_offline))
 342                         torture_online(cpu,
 343                                        &n_online_attempts, &n_online_successes,
 344                                        &sum_online, &min_online, &max_online);
 345                 schedule_timeout_interruptible(onoff_interval);
 346         }
 347
 348 stop:
 349         torture_kthread_stopping("torture_onoff");
 350         return 0;
 351 }
 352
 353 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 354
 355 /*
 356  * Initiate online-offline handling.
 357  */
 358 int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f)
 359 {
 360 #ifdef CONFIG_HOTPLUG_CPU
 361         onoff_holdoff = ooholdoff;
 362         onoff_interval = oointerval;
 363         onoff_f = f;
 364         if (onoff_interval <= 0)
 365                 return 0;
 366         return torture_create_kthread(torture_onoff, NULL, onoff_task);
 367 #else /* #ifdef CONFIG_HOTPLUG_CPU */
 368         return 0;
 369 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
 370 }
 371 EXPORT_SYMBOL_GPL(torture_onoff_init);
 372
 373 /*
 374  * Clean up after online/offline testing.
 375  */
 376 static void torture_onoff_cleanup(void)
 377 {
 378 #ifdef CONFIG_HOTPLUG_CPU
 379         if (onoff_task == NULL)
 380                 return;
 381         VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
 382         kthread_stop(onoff_task);
 383         onoff_task = NULL;
 384 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 385 }
 386
 387 /*
 388  * Print online/offline testing statistics.
 389  */
 390 void torture_onoff_stats(void)
 391 {
 392 #ifdef CONFIG_HOTPLUG_CPU
 393         pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
 394                 n_online_successes, n_online_attempts,
 395                 n_offline_successes, n_offline_attempts,
 396                 min_online, max_online,
 397                 min_offline, max_offline,
 398                 sum_online, sum_offline, HZ);
 399 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 400 }
 401 EXPORT_SYMBOL_GPL(torture_onoff_stats);
 402
 403 /*
 404  * Were all the online/offline operations successful?
 405  */
 406 bool torture_onoff_failures(void)
 407 {
 408 #ifdef CONFIG_HOTPLUG_CPU
 409         return n_online_successes != n_online_attempts ||
 410                n_offline_successes != n_offline_attempts;
 411 #else /* #ifdef CONFIG_HOTPLUG_CPU */
 412         return false;
 413 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
 414 }
 415 EXPORT_SYMBOL_GPL(torture_onoff_failures);
 416
 417 #define TORTURE_RANDOM_MULT     39916801  /* prime */
 418 #define TORTURE_RANDOM_ADD      479001701 /* prime */
 419 #define TORTURE_RANDOM_REFRESH  10000
 420
 421 /*
 422  * Crude but fast random-number generator.  Uses a linear congruential
 423  * generator, with occasional help from cpu_clock().
 424  */
 425 unsigned long
 426 torture_random(struct torture_random_state *trsp)
 427 {
 428         if (--trsp->trs_count < 0) {
 429                 trsp->trs_state += (unsigned long)local_clock();
 430                 trsp->trs_count = TORTURE_RANDOM_REFRESH;
 431         }
 432         trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
 433                 TORTURE_RANDOM_ADD;
 434         return swahw32(trsp->trs_state);
 435 }
 436 EXPORT_SYMBOL_GPL(torture_random);
 437
 438 /*
 439  * Variables for shuffling.  The idea is to ensure that each CPU stays
 440  * idle for an extended period to test interactions with dyntick idle,
 441  * as well as interactions with any per-CPU variables.
 442  */
 443 struct shuffle_task {
 444         struct list_head st_l;
 445         struct task_struct *st_t;
 446 };
 447
 448 static long shuffle_interval;   /* In jiffies. */
 449 static struct task_struct *shuffler_task;
 450 static cpumask_var_t shuffle_tmp_mask;
 451 static int shuffle_idle_cpu;    /* Force all torture tasks off this CPU */
 452 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
 453 static DEFINE_MUTEX(shuffle_task_mutex);
 454
 455 /*
 456  * Register a task to be shuffled.  If there is no memory, just splat
 457  * and don't bother registering.
 458  */
 459 void torture_shuffle_task_register(struct task_struct *tp)
 460 {
 461         struct shuffle_task *stp;
 462
 463         if (WARN_ON_ONCE(tp == NULL))
 464                 return;
 465         stp = kmalloc(sizeof(*stp), GFP_KERNEL);
 466         if (WARN_ON_ONCE(stp == NULL))
 467                 return;
 468         stp->st_t = tp;
 469         mutex_lock(&shuffle_task_mutex);
 470         list_add(&stp->st_l, &shuffle_task_list);
 471         mutex_unlock(&shuffle_task_mutex);
 472 }
 473 EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
 474
 475 /*
 476  * Unregister all tasks, for example, at the end of the torture run.
 477  */
 478 static void torture_shuffle_task_unregister_all(void)
 479 {
 480         struct shuffle_task *stp;
 481         struct shuffle_task *p;
 482
 483         mutex_lock(&shuffle_task_mutex);
 484         list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
 485                 list_del(&stp->st_l);
 486                 kfree(stp);
 487         }
 488         mutex_unlock(&shuffle_task_mutex);
 489 }
 490
 491 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle.
 492  * A special case is when shuffle_idle_cpu = -1, in which case we allow
 493  * the tasks to run on all CPUs.
 494  */
 495 static void torture_shuffle_tasks(void)
 496 {
 497         struct shuffle_task *stp;
 498
 499         cpumask_setall(shuffle_tmp_mask);
 500         get_online_cpus();
 501
 502         /* No point in shuffling if there is only one online CPU (ex: UP) */
 503         if (num_online_cpus() == 1) {
 504                 put_online_cpus();
 505                 return;
 506         }
 507
 508         /* Advance to the next CPU.  Upon overflow, don't idle any CPUs. */
 509         shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
 510         if (shuffle_idle_cpu >= nr_cpu_ids)
 511                 shuffle_idle_cpu = -1;
 512         else
 513                 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
 514
 515         mutex_lock(&shuffle_task_mutex);
 516         list_for_each_entry(stp, &shuffle_task_list, st_l)
 517                 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
 518         mutex_unlock(&shuffle_task_mutex);
 519
 520         put_online_cpus();
 521 }
 522
 523 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
 524  * system to become idle at a time and cut off its timer ticks. This is meant
 525  * to test the support for such tickless idle CPU in RCU.
 526  */
 527 static int torture_shuffle(void *arg)
 528 {
 529         VERBOSE_TOROUT_STRING("torture_shuffle task started");
 530         do {
 531                 schedule_timeout_interruptible(shuffle_interval);
 532                 torture_shuffle_tasks();
 533                 torture_shutdown_absorb("torture_shuffle");
 534         } while (!torture_must_stop());
 535         torture_kthread_stopping("torture_shuffle");
 536         return 0;
 537 }
 538
 539 /*
 540  * Start the shuffler, with shuffint in jiffies.
 541  */
 542 int torture_shuffle_init(long shuffint)
 543 {
 544         shuffle_interval = shuffint;
 545
 546         shuffle_idle_cpu = -1;
 547
 548         if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
 549                 VERBOSE_TOROUT_ERRSTRING("Failed to alloc mask");
 550                 return -ENOMEM;
 551         }
 552
 553         /* Create the shuffler thread */
 554         return torture_create_kthread(torture_shuffle, NULL, shuffler_task);
 555 }
 556 EXPORT_SYMBOL_GPL(torture_shuffle_init);
 557
 558 /*
 559  * Stop the shuffling.
 560  */
 561 static void torture_shuffle_cleanup(void)
 562 {
 563         torture_shuffle_task_unregister_all();
 564         if (shuffler_task) {
 565                 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
 566                 kthread_stop(shuffler_task);
 567                 free_cpumask_var(shuffle_tmp_mask);
 568         }
 569         shuffler_task = NULL;
 570 }
 571
 572 /*
 573  * Variables for auto-shutdown.  This allows "lights out" torture runs
 574  * to be fully scripted.
 575  */
 576 static struct task_struct *shutdown_task;
 577 static ktime_t shutdown_time;           /* time to system shutdown. */
 578 static void (*torture_shutdown_hook)(void);
 579
 580 /*
 581  * Absorb kthreads into a kernel function that won't return, so that
 582  * they won't ever access module text or data again.
 583  */
 584 void torture_shutdown_absorb(const char *title)
 585 {
 586         while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 587                 pr_notice("torture thread %s parking due to system shutdown\n",
 588                           title);
 589                 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
 590         }
 591 }
 592 EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
 593
 594 /*
 595  * Cause the torture test to shutdown the system after the test has
 596  * run for the time specified by the shutdown_secs parameter.
 597  */
 598 static int torture_shutdown(void *arg)
 599 {
 600         ktime_t ktime_snap;
 601
 602         VERBOSE_TOROUT_STRING("torture_shutdown task started");
 603         ktime_snap = ktime_get();
 604         while (ktime_before(ktime_snap, shutdown_time) &&
 605                !torture_must_stop()) {
 606                 if (verbose)
 607                         pr_alert("%s" TORTURE_FLAG
 608                                  "torture_shutdown task: %llu ms remaining\n",
 609                                  torture_type,
 610                                  ktime_ms_delta(shutdown_time, ktime_snap));
 611                 set_current_state(TASK_INTERRUPTIBLE);
 612                 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS);
 613                 ktime_snap = ktime_get();
 614         }
 615         if (torture_must_stop()) {
 616                 torture_kthread_stopping("torture_shutdown");
 617                 return 0;
 618         }
 619
 620         /* OK, shut down the system. */
 621
 622         VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system");
 623         shutdown_task = NULL;   /* Avoid self-kill deadlock. */
 624         if (torture_shutdown_hook)
 625                 torture_shutdown_hook();
 626         else
 627                 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping.");
 628         if (ftrace_dump_at_shutdown)
 629                 rcu_ftrace_dump(DUMP_ALL);
 630         kernel_power_off();     /* Shut down the system. */
 631         return 0;
 632 }
 633
 634 /*
 635  * Start up the shutdown task.
 636  */
 637 int torture_shutdown_init(int ssecs, void (*cleanup)(void))
 638 {
 639         torture_shutdown_hook = cleanup;
 640         if (ssecs > 0) {
 641                 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0));
 642                 return torture_create_kthread(torture_shutdown, NULL,
 643                                              shutdown_task);
 644         }
 645         return 0;
 646 }
 647 EXPORT_SYMBOL_GPL(torture_shutdown_init);
 648
 649 /*
 650  * Detect and respond to a system shutdown.
 651  */
 652 static int torture_shutdown_notify(struct notifier_block *unused1,
 653                                    unsigned long unused2, void *unused3)
 654 {
 655         mutex_lock(&fullstop_mutex);
 656         if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
 657                 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
 658                 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
 659         } else {
 660                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
 661         }
 662         mutex_unlock(&fullstop_mutex);
 663         return NOTIFY_DONE;
 664 }
 665
 666 static struct notifier_block torture_shutdown_nb = {
 667         .notifier_call = torture_shutdown_notify,
 668 };
 669
 670 /*
 671  * Shut down the shutdown task.  Say what???  Heh!  This can happen if
 672  * the torture module gets an rmmod before the shutdown time arrives.  ;-)
 673  */
 674 static void torture_shutdown_cleanup(void)
 675 {
 676         unregister_reboot_notifier(&torture_shutdown_nb);
 677         if (shutdown_task != NULL) {
 678                 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task");
 679                 kthread_stop(shutdown_task);
 680         }
 681         shutdown_task = NULL;
 682 }
 683
 684 /*
 685  * Variables for stuttering, which means to periodically pause and
 686  * restart testing in order to catch bugs that appear when load is
 687  * suddenly applied to or removed from the system.
 688  */
 689 static struct task_struct *stutter_task;
 690 static int stutter_pause_test;
 691 static int stutter;
 692 static int stutter_gap;
 693
 694 /*
 695  * Block until the stutter interval ends.  This must be called periodically
 696  * by all running kthreads that need to be subject to stuttering.
 697  */
 698 bool stutter_wait(const char *title)
 699 {
 700         unsigned int i = 0;
 701         bool ret = false;
 702         int spt;
 703
 704         cond_resched_tasks_rcu_qs();
 705         spt = READ_ONCE(stutter_pause_test);
 706         for (; spt; spt = READ_ONCE(stutter_pause_test)) {
 707                 if (!ret) {
 708                         sched_set_normal(current, MAX_NICE);
 709                         ret = true;
 710                 }
 711                 if (spt == 1) {
 712                         schedule_timeout_interruptible(1);
 713                 } else if (spt == 2) {
 714                         while (READ_ONCE(stutter_pause_test)) {
 715                                 if (!(i++ & 0xffff))
 716                                         torture_hrtimeout_us(10, 0, NULL);
 717                                 cond_resched();
 718                         }
 719                 } else {
 720                         schedule_timeout_interruptible(round_jiffies_relative(HZ));
 721                 }
 722                 torture_shutdown_absorb(title);
 723         }
 724         return ret;
 725 }
 726 EXPORT_SYMBOL_GPL(stutter_wait);
 727
 728 /*
 729  * Cause the torture test to "stutter", starting and stopping all
 730  * threads periodically.
 731  */
 732 static int torture_stutter(void *arg)
 733 {
 734         DEFINE_TORTURE_RANDOM(rand);
 735         int wtime;
 736
 737         VERBOSE_TOROUT_STRING("torture_stutter task started");
 738         do {
 739                 if (!torture_must_stop() && stutter > 1) {
 740                         wtime = stutter;
 741                         if (stutter > 2) {
 742                                 WRITE_ONCE(stutter_pause_test, 1);
 743                                 wtime = stutter - 3;
 744                                 torture_hrtimeout_jiffies(wtime, &rand);
 745                                 wtime = 2;
 746                         }
 747                         WRITE_ONCE(stutter_pause_test, 2);
 748                         torture_hrtimeout_jiffies(wtime, NULL);
 749                 }
 750                 WRITE_ONCE(stutter_pause_test, 0);
 751                 if (!torture_must_stop())
 752                         torture_hrtimeout_jiffies(stutter_gap, NULL);
 753                 torture_shutdown_absorb("torture_stutter");
 754         } while (!torture_must_stop());
 755         torture_kthread_stopping("torture_stutter");
 756         return 0;
 757 }
 758
 759 /*
 760  * Initialize and kick off the torture_stutter kthread.
 761  */
 762 int torture_stutter_init(const int s, const int sgap)
 763 {
 764         stutter = s;
 765         stutter_gap = sgap;
 766         return torture_create_kthread(torture_stutter, NULL, stutter_task);
 767 }
 768 EXPORT_SYMBOL_GPL(torture_stutter_init);
 769
 770 /*
 771  * Cleanup after the torture_stutter kthread.
 772  */
 773 static void torture_stutter_cleanup(void)
 774 {
 775         if (!stutter_task)
 776                 return;
 777         VERBOSE_TOROUT_STRING("Stopping torture_stutter task");
 778         kthread_stop(stutter_task);
 779         stutter_task = NULL;
 780 }
 781
 782 /*
 783  * Initialize torture module.  Please note that this is -not- invoked via
 784  * the usual module_init() mechanism, but rather by an explicit call from
 785  * the client torture module.  This call must be paired with a later
 786  * torture_init_end().
 787  *
 788  * The runnable parameter points to a flag that controls whether or not
 789  * the test is currently runnable.  If there is no such flag, pass in NULL.
 790  */
 791 bool torture_init_begin(char *ttype, int v)
 792 {
 793         mutex_lock(&fullstop_mutex);
 794         if (torture_type != NULL) {
 795                 pr_alert("torture_init_begin: Refusing %s init: %s running.\n",
 796                          ttype, torture_type);
 797                 pr_alert("torture_init_begin: One torture test at a time!\n");
 798                 mutex_unlock(&fullstop_mutex);
 799                 return false;
 800         }
 801         torture_type = ttype;
 802         verbose = v;
 803         fullstop = FULLSTOP_DONTSTOP;
 804         return true;
 805 }
 806 EXPORT_SYMBOL_GPL(torture_init_begin);
 807
 808 /*
 809  * Tell the torture module that initialization is complete.
 810  */
 811 void torture_init_end(void)
 812 {
 813         mutex_unlock(&fullstop_mutex);
 814         register_reboot_notifier(&torture_shutdown_nb);
 815 }
 816 EXPORT_SYMBOL_GPL(torture_init_end);
 817
 818 /*
 819  * Clean up torture module.  Please note that this is -not- invoked via
 820  * the usual module_exit() mechanism, but rather by an explicit call from
 821  * the client torture module.  Returns true if a race with system shutdown
 822  * is detected, otherwise, all kthreads started by functions in this file
 823  * will be shut down.
 824  *
 825  * This must be called before the caller starts shutting down its own
 826  * kthreads.
 827  *
 828  * Both torture_cleanup_begin() and torture_cleanup_end() must be paired,
 829  * in order to correctly perform the cleanup. They are separated because
 830  * threads can still need to reference the torture_type type, thus nullify
 831  * only after completing all other relevant calls.
 832  */
 833 bool torture_cleanup_begin(void)
 834 {
 835         mutex_lock(&fullstop_mutex);
 836         if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 837                 pr_warn("Concurrent rmmod and shutdown illegal!\n");
 838                 mutex_unlock(&fullstop_mutex);
 839                 schedule_timeout_uninterruptible(10);
 840                 return true;
 841         }
 842         WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
 843         mutex_unlock(&fullstop_mutex);
 844         torture_shutdown_cleanup();
 845         torture_shuffle_cleanup();
 846         torture_stutter_cleanup();
 847         torture_onoff_cleanup();
 848         return false;
 849 }
 850 EXPORT_SYMBOL_GPL(torture_cleanup_begin);
 851
 852 void torture_cleanup_end(void)
 853 {
 854         mutex_lock(&fullstop_mutex);
 855         torture_type = NULL;
 856         mutex_unlock(&fullstop_mutex);
 857 }
 858 EXPORT_SYMBOL_GPL(torture_cleanup_end);
 859
 860 /*
 861  * Is it time for the current torture test to stop?
 862  */
 863 bool torture_must_stop(void)
 864 {
 865         return torture_must_stop_irq() || kthread_should_stop();
 866 }
 867 EXPORT_SYMBOL_GPL(torture_must_stop);
 868
 869 /*
 870  * Is it time for the current torture test to stop?  This is the irq-safe
 871  * version, hence no check for kthread_should_stop().
 872  */
 873 bool torture_must_stop_irq(void)
 874 {
 875         return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
 876 }
 877 EXPORT_SYMBOL_GPL(torture_must_stop_irq);
 878
 879 /*
 880  * Each kthread must wait for kthread_should_stop() before returning from
 881  * its top-level function, otherwise segfaults ensue.  This function
 882  * prints a "stopping" message and waits for kthread_should_stop(), and
 883  * should be called from all torture kthreads immediately prior to
 884  * returning.
 885  */
 886 void torture_kthread_stopping(char *title)
 887 {
 888         char buf[128];
 889
 890         snprintf(buf, sizeof(buf), "Stopping %s", title);
 891         VERBOSE_TOROUT_STRING(buf);
 892         while (!kthread_should_stop()) {
 893                 torture_shutdown_absorb(title);
 894                 schedule_timeout_uninterruptible(1);
 895         }
 896 }
 897 EXPORT_SYMBOL_GPL(torture_kthread_stopping);
 898
 899 /*
 900  * Create a generic torture kthread that is immediately runnable.  If you
 901  * need the kthread to be stopped so that you can do something to it before
 902  * it starts, you will need to open-code your own.
 903  */
 904 int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m,
 905                             char *f, struct task_struct **tp)
 906 {
 907         int ret = 0;
 908
 909         VERBOSE_TOROUT_STRING(m);
 910         *tp = kthread_run(fn, arg, "%s", s);
 911         if (IS_ERR(*tp)) {
 912                 ret = PTR_ERR(*tp);
 913                 VERBOSE_TOROUT_ERRSTRING(f);
 914                 *tp = NULL;
 915         }
 916         torture_shuffle_task_register(*tp);
 917         return ret;
 918 }
 919 EXPORT_SYMBOL_GPL(_torture_create_kthread);
 920
 921 /*
 922  * Stop a generic kthread, emitting a message.
 923  */
 924 void _torture_stop_kthread(char *m, struct task_struct **tp)
 925 {
 926         if (*tp == NULL)
 927                 return;
 928         VERBOSE_TOROUT_STRING(m);
 929         kthread_stop(*tp);
 930         *tp = NULL;
 931 }
 932 EXPORT_SYMBOL_GPL(_torture_stop_kthread);