tools/testing/selftests/rcutorture/bin/torture.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0+
   3 #
   4 # Run a series of torture tests, intended for overnight or
   5 # longer timeframes, and also for large systems.
   6 #
   7 # Usage: torture.sh [ options ]
   8 #
   9 # Copyright (C) 2020 Facebook, Inc.
  10 #
  11 # Authors: Paul E. McKenney <paulmck@kernel.org>
  12
  13 scriptname=$0
  14 args="$*"
  15
  16 KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM
  17 PATH=${KVM}/bin:$PATH; export PATH
  18 . functions.sh
  19
  20 TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
  21 MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
  22 HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
  23 if test "$HALF_ALLOTED_CPUS" -lt 1
  24 then
  25         HALF_ALLOTED_CPUS=1
  26 fi
  27 VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
  28 if test "$VERBOSE_BATCH_CPUS" -lt 2
  29 then
  30         VERBOSE_BATCH_CPUS=0
  31 fi
  32
  33 # Configurations/scenarios.
  34 configs_rcutorture=
  35 configs_locktorture=
  36 configs_scftorture=
  37 kcsan_kmake_args=
  38
  39 # Default compression, duration, and apportionment.
  40 compress_kasan_vmlinux="`identify_qemu_vcpus`"
  41 duration_base=10
  42 duration_rcutorture_frac=7
  43 duration_locktorture_frac=1
  44 duration_scftorture_frac=2
  45
  46 # "yes" or "no" parameters
  47 do_allmodconfig=yes
  48 do_rcutorture=yes
  49 do_locktorture=yes
  50 do_scftorture=yes
  51 do_rcuscale=yes
  52 do_refscale=yes
  53 do_kvfree=yes
  54 do_kasan=yes
  55 do_kcsan=no
  56 do_clocksourcewd=yes
  57
  58 # doyesno - Helper function for yes/no arguments
  59 function doyesno () {
  60         if test "$1" = "$2"
  61         then
  62                 echo yes
  63         else
  64                 echo no
  65         fi
  66 }
  67
  68 usage () {
  69         echo "Usage: $scriptname optional arguments:"
  70         echo "       --compress-kasan-vmlinux concurrency"
  71         echo "       --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
  72         echo "       --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
  73         echo "       --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
  74         echo "       --doall"
  75         echo "       --doallmodconfig / --do-no-allmodconfig"
  76         echo "       --do-clocksourcewd / --do-no-clocksourcewd"
  77         echo "       --do-kasan / --do-no-kasan"
  78         echo "       --do-kcsan / --do-no-kcsan"
  79         echo "       --do-kvfree / --do-no-kvfree"
  80         echo "       --do-locktorture / --do-no-locktorture"
  81         echo "       --do-none"
  82         echo "       --do-rcuscale / --do-no-rcuscale"
  83         echo "       --do-rcutorture / --do-no-rcutorture"
  84         echo "       --do-refscale / --do-no-refscale"
  85         echo "       --do-scftorture / --do-no-scftorture"
  86         echo "       --duration [ <minutes> | <hours>h | <days>d ]"
  87         echo "       --kcsan-kmake-arg kernel-make-arguments"
  88         exit 1
  89 }
  90
  91 while test $# -gt 0
  92 do
  93         case "$1" in
  94         --compress-kasan-vmlinux)
  95                 checkarg --compress-kasan-vmlinux "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error'
  96                 compress_kasan_vmlinux=$2
  97                 shift
  98                 ;;
  99         --config-rcutorture|--configs-rcutorture)
 100                 checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
 101                 configs_rcutorture="$configs_rcutorture $2"
 102                 shift
 103                 ;;
 104         --config-locktorture|--configs-locktorture)
 105                 checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
 106                 configs_locktorture="$configs_locktorture $2"
 107                 shift
 108                 ;;
 109         --config-scftorture|--configs-scftorture)
 110                 checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
 111                 configs_scftorture="$configs_scftorture $2"
 112                 shift
 113                 ;;
 114         --do-all|--doall)
 115                 do_allmodconfig=yes
 116                 do_rcutorture=yes
 117                 do_locktorture=yes
 118                 do_scftorture=yes
 119                 do_rcuscale=yes
 120                 do_refscale=yes
 121                 do_kvfree=yes
 122                 do_kasan=yes
 123                 do_kcsan=yes
 124                 do_clocksourcewd=yes
 125                 ;;
 126         --do-allmodconfig|--do-no-allmodconfig)
 127                 do_allmodconfig=`doyesno "$1" --do-allmodconfig`
 128                 ;;
 129         --do-clocksourcewd|--do-no-clocksourcewd)
 130                 do_clocksourcewd=`doyesno "$1" --do-clocksourcewd`
 131                 ;;
 132         --do-kasan|--do-no-kasan)
 133                 do_kasan=`doyesno "$1" --do-kasan`
 134                 ;;
 135         --do-kcsan|--do-no-kcsan)
 136                 do_kcsan=`doyesno "$1" --do-kcsan`
 137                 ;;
 138         --do-kvfree|--do-no-kvfree)
 139                 do_kvfree=`doyesno "$1" --do-kvfree`
 140                 ;;
 141         --do-locktorture|--do-no-locktorture)
 142                 do_locktorture=`doyesno "$1" --do-locktorture`
 143                 ;;
 144         --do-none|--donone)
 145                 do_allmodconfig=no
 146                 do_rcutorture=no
 147                 do_locktorture=no
 148                 do_scftorture=no
 149                 do_rcuscale=no
 150                 do_refscale=no
 151                 do_kvfree=no
 152                 do_kasan=no
 153                 do_kcsan=no
 154                 do_clocksourcewd=no
 155                 ;;
 156         --do-rcuscale|--do-no-rcuscale)
 157                 do_rcuscale=`doyesno "$1" --do-rcuscale`
 158                 ;;
 159         --do-rcutorture|--do-no-rcutorture)
 160                 do_rcutorture=`doyesno "$1" --do-rcutorture`
 161                 ;;
 162         --do-refscale|--do-no-refscale)
 163                 do_refscale=`doyesno "$1" --do-refscale`
 164                 ;;
 165         --do-scftorture|--do-no-scftorture)
 166                 do_scftorture=`doyesno "$1" --do-scftorture`
 167                 ;;
 168         --duration)
 169                 checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
 170                 mult=1
 171                 if echo "$2" | grep -q 'm$'
 172                 then
 173                         mult=1
 174                 elif echo "$2" | grep -q 'h$'
 175                 then
 176                         mult=60
 177                 elif echo "$2" | grep -q 'd$'
 178                 then
 179                         mult=1440
 180                 fi
 181                 ts=`echo $2 | sed -e 's/[smhd]$//'`
 182                 duration_base=$(($ts*mult))
 183                 shift
 184                 ;;
 185         --kcsan-kmake-arg|--kcsan-kmake-args)
 186                 checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
 187                 kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`"
 188                 shift
 189                 ;;
 190         *)
 191                 echo Unknown argument $1
 192                 usage
 193                 ;;
 194         esac
 195         shift
 196 done
 197
 198 ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
 199 startdate="`date`"
 200 starttime="`get_starttime`"
 201
 202 T=/tmp/torture.sh.$$
 203 trap 'rm -rf $T' 0 2
 204 mkdir $T
 205
 206 echo " --- " $scriptname $args | tee -a $T/log
 207 echo " --- Results directory: " $ds | tee -a $T/log
 208
 209 # Calculate rcutorture defaults and apportion time
 210 if test -z "$configs_rcutorture"
 211 then
 212         configs_rcutorture=CFLIST
 213 fi
 214 duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
 215 if test "$duration_rcutorture" -eq 0
 216 then
 217         echo " --- Zero time for rcutorture, disabling" | tee -a $T/log
 218         do_rcutorture=no
 219 fi
 220
 221 # Calculate locktorture defaults and apportion time
 222 if test -z "$configs_locktorture"
 223 then
 224         configs_locktorture=CFLIST
 225 fi
 226 duration_locktorture=$((duration_base*duration_locktorture_frac/10))
 227 if test "$duration_locktorture" -eq 0
 228 then
 229         echo " --- Zero time for locktorture, disabling" | tee -a $T/log
 230         do_locktorture=no
 231 fi
 232
 233 # Calculate scftorture defaults and apportion time
 234 if test -z "$configs_scftorture"
 235 then
 236         configs_scftorture=CFLIST
 237 fi
 238 duration_scftorture=$((duration_base*duration_scftorture_frac/10))
 239 if test "$duration_scftorture" -eq 0
 240 then
 241         echo " --- Zero time for scftorture, disabling" | tee -a $T/log
 242         do_scftorture=no
 243 fi
 244
 245 touch $T/failures
 246 touch $T/successes
 247
 248 # torture_one - Does a single kvm.sh run.
 249 #
 250 # Usage:
 251 #       torture_bootargs="[ kernel boot arguments ]"
 252 #       torture_one flavor [ kvm.sh arguments ]
 253 #
 254 # Note that "flavor" is an arbitrary string.  Supply --torture if needed.
 255 # Note that quoting is problematic.  So on the command line, pass multiple
 256 # values with multiple kvm.sh argument instances.
 257 function torture_one {
 258         local cur_bootargs=
 259         local boottag=
 260
 261         echo " --- $curflavor:" Start `date` | tee -a $T/log
 262         if test -n "$torture_bootargs"
 263         then
 264                 boottag="--bootargs"
 265                 cur_bootargs="$torture_bootargs"
 266         fi
 267         "$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
 268         retcode=$?
 269         resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
 270         if test -z "$resdir"
 271         then
 272                 cat $T/$curflavor.out | tee -a $T/log
 273                 echo retcode=$retcode | tee -a $T/log
 274         fi
 275         if test "$retcode" == 0
 276         then
 277                 echo "$curflavor($retcode)" $resdir >> $T/successes
 278         else
 279                 echo "$curflavor($retcode)" $resdir >> $T/failures
 280         fi
 281 }
 282
 283 # torture_set - Does a set of tortures with and without KASAN and KCSAN.
 284 #
 285 # Usage:
 286 #       torture_bootargs="[ kernel boot arguments ]"
 287 #       torture_set flavor [ kvm.sh arguments ]
 288 #
 289 # Note that "flavor" is an arbitrary string that does not affect kvm.sh
 290 # in any way.  So also supply --torture if you need something other than
 291 # the default.
 292 function torture_set {
 293         local cur_kcsan_kmake_args=
 294         local kcsan_kmake_tag=
 295         local flavor=$1
 296         shift
 297         curflavor=$flavor
 298         torture_one "$@"
 299         if test "$do_kasan" = "yes"
 300         then
 301                 curflavor=${flavor}-kasan
 302                 torture_one "$@" --kasan
 303         fi
 304         if test "$do_kcsan" = "yes"
 305         then
 306                 curflavor=${flavor}-kcsan
 307                 if test -n "$kcsan_kmake_args"
 308                 then
 309                         kcsan_kmake_tag="--kmake-args"
 310                         cur_kcsan_kmake_args="$kcsan_kmake_args"
 311                 fi
 312                 torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan
 313         fi
 314 }
 315
 316 # make allmodconfig
 317 if test "$do_allmodconfig" = "yes"
 318 then
 319         echo " --- allmodconfig:" Start `date` | tee -a $T/log
 320         amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
 321         mkdir -p "$amcdir"
 322         echo " --- make clean" > "$amcdir/Make.out" 2>&1
 323         make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1
 324         echo " --- make allmodconfig" >> "$amcdir/Make.out" 2>&1
 325         make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1
 326         echo " --- make " >> "$amcdir/Make.out" 2>&1
 327         make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1
 328         retcode="$?"
 329         echo $retcode > "$amcdir/Make.exitcode"
 330         if test "$retcode" == 0
 331         then
 332                 echo "allmodconfig($retcode)" $amcdir >> $T/successes
 333         else
 334                 echo "allmodconfig($retcode)" $amcdir >> $T/failures
 335         fi
 336 fi
 337
 338 # --torture rcu
 339 if test "$do_rcutorture" = "yes"
 340 then
 341         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
 342         torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make
 343 fi
 344
 345 if test "$do_locktorture" = "yes"
 346 then
 347         torture_bootargs="torture.disable_onoff_at_boot"
 348         torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make
 349 fi
 350
 351 if test "$do_scftorture" = "yes"
 352 then
 353         torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot"
 354         torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
 355 fi
 356
 357 if test "$do_refscale" = yes
 358 then
 359         primlist="`grep '\.name[        ]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
 360 else
 361         primlist=
 362 fi
 363 for prim in $primlist
 364 do
 365         torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
 366         torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make
 367 done
 368
 369 if test "$do_rcuscale" = yes
 370 then
 371         primlist="`grep '\.name[        ]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
 372 else
 373         primlist=
 374 fi
 375 for prim in $primlist
 376 do
 377         torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
 378         torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
 379 done
 380
 381 if test "$do_kvfree" = "yes"
 382 then
 383         torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
 384         torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make
 385 fi
 386
 387 if test "$do_clocksourcewd" = "yes"
 388 then
 389         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
 390         torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
 391
 392         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 clocksource.max_cswd_read_retries=1"
 393         torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
 394
 395         # In case our work is already done...
 396         if test "$do_rcutorture" != "yes"
 397         then
 398                 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
 399                 torture_set "clocksourcewd-3" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --trust-make
 400         fi
 401 fi
 402
 403 echo " --- " $scriptname $args
 404 echo " --- " Done `date` | tee -a $T/log
 405 ret=0
 406 nsuccesses=0
 407 echo SUCCESSES: | tee -a $T/log
 408 if test -s "$T/successes"
 409 then
 410         cat "$T/successes" | tee -a $T/log
 411         nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
 412 fi
 413 nfailures=0
 414 echo FAILURES: | tee -a $T/log
 415 if test -s "$T/failures"
 416 then
 417         cat "$T/failures" | tee -a $T/log
 418         nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
 419         ret=2
 420 fi
 421 echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
 422 echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
 423 tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
 424 if test -n "$tdir" && test $compress_kasan_vmlinux -gt 0
 425 then
 426         # KASAN vmlinux files can approach 1GB in size, so compress them.
 427         echo Looking for KASAN files to compress: `date` > "$tdir/log-xz" 2>&1
 428         find "$tdir" -type d -name '*-kasan' -print > $T/xz-todo
 429         ncompresses=0
 430         batchno=1
 431         if test -s $T/xz-todo
 432         then
 433                 echo Size before compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
 434                 for i in `cat $T/xz-todo`
 435                 do
 436                         echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1
 437                         for j in $i/*/vmlinux
 438                         do
 439                                 xz "$j" >> "$tdir/log-xz" 2>&1 &
 440                                 ncompresses=$((ncompresses+1))
 441                                 if test $ncompresses -ge $compress_kasan_vmlinux
 442                                 then
 443                                         echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
 444                                         wait
 445                                         ncompresses=0
 446                                         batchno=$((batchno+1))
 447                                 fi
 448                         done
 449                 done
 450                 if test $ncompresses -gt 0
 451                 then
 452                         echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
 453                 fi
 454                 wait
 455                 echo Size after compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
 456                 echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log
 457         else
 458                 echo No compression needed: `date` >> "$tdir/log-xz" 2>&1
 459         fi
 460 fi
 461 if test -n "$tdir"
 462 then
 463         cp $T/log "$tdir"
 464 fi
 465 exit $ret