tools/testing/selftests/rcutorture/bin/torture.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0+
   3 #
   4 # Run a series of torture tests, intended for overnight or
   5 # longer timeframes, and also for large systems.
   6 #
   7 # Usage: torture.sh [ options ]
   8 #
   9 # Copyright (C) 2020 Facebook, Inc.
  10 #
  11 # Authors: Paul E. McKenney <paulmck@kernel.org>
  12
  13 scriptname=$0
  14 args="$*"
  15
  16 KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM
  17 PATH=${KVM}/bin:$PATH; export PATH
  18 . functions.sh
  19
  20 TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
  21 MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
  22 HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
  23 if test "$HALF_ALLOTED_CPUS" -lt 1
  24 then
  25         HALF_ALLOTED_CPUS=1
  26 fi
  27 VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
  28 if test "$VERBOSE_BATCH_CPUS" -lt 2
  29 then
  30         VERBOSE_BATCH_CPUS=0
  31 fi
  32
  33 # Configurations/scenarios.
  34 configs_rcutorture=
  35 configs_locktorture=
  36 configs_scftorture=
  37 kcsan_kmake_args=
  38
  39 # Default compression, duration, and apportionment.
  40 compress_kasan_vmlinux="`identify_qemu_vcpus`"
  41 duration_base=10
  42 duration_rcutorture_frac=7
  43 duration_locktorture_frac=1
  44 duration_scftorture_frac=2
  45
  46 # "yes" or "no" parameters
  47 do_allmodconfig=yes
  48 do_rcutorture=yes
  49 do_locktorture=yes
  50 do_scftorture=yes
  51 do_rcuscale=yes
  52 do_refscale=yes
  53 do_kvfree=yes
  54 do_kasan=yes
  55 do_kcsan=no
  56
  57 # doyesno - Helper function for yes/no arguments
  58 function doyesno () {
  59         if test "$1" = "$2"
  60         then
  61                 echo yes
  62         else
  63                 echo no
  64         fi
  65 }
  66
  67 usage () {
  68         echo "Usage: $scriptname optional arguments:"
  69         echo "       --compress-kasan-vmlinux concurrency"
  70         echo "       --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
  71         echo "       --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
  72         echo "       --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
  73         echo "       --doall"
  74         echo "       --doallmodconfig / --do-no-allmodconfig"
  75         echo "       --do-kasan / --do-no-kasan"
  76         echo "       --do-kcsan / --do-no-kcsan"
  77         echo "       --do-kvfree / --do-no-kvfree"
  78         echo "       --do-locktorture / --do-no-locktorture"
  79         echo "       --do-none"
  80         echo "       --do-rcuscale / --do-no-rcuscale"
  81         echo "       --do-rcutorture / --do-no-rcutorture"
  82         echo "       --do-refscale / --do-no-refscale"
  83         echo "       --do-scftorture / --do-no-scftorture"
  84         echo "       --duration [ <minutes> | <hours>h | <days>d ]"
  85         echo "       --kcsan-kmake-arg kernel-make-arguments"
  86         exit 1
  87 }
  88
  89 while test $# -gt 0
  90 do
  91         case "$1" in
  92         --compress-kasan-vmlinux)
  93                 checkarg --compress-kasan-vmlinux "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error'
  94                 compress_kasan_vmlinux=$2
  95                 shift
  96                 ;;
  97         --config-rcutorture|--configs-rcutorture)
  98                 checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
  99                 configs_rcutorture="$configs_rcutorture $2"
 100                 shift
 101                 ;;
 102         --config-locktorture|--configs-locktorture)
 103                 checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
 104                 configs_locktorture="$configs_locktorture $2"
 105                 shift
 106                 ;;
 107         --config-scftorture|--configs-scftorture)
 108                 checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
 109                 configs_scftorture="$configs_scftorture $2"
 110                 shift
 111                 ;;
 112         --doall)
 113                 do_allmodconfig=yes
 114                 do_rcutorture=yes
 115                 do_locktorture=yes
 116                 do_scftorture=yes
 117                 do_rcuscale=yes
 118                 do_refscale=yes
 119                 do_kvfree=yes
 120                 do_kasan=yes
 121                 do_kcsan=yes
 122                 ;;
 123         --do-allmodconfig|--do-no-allmodconfig)
 124                 do_allmodconfig=`doyesno "$1" --do-allmodconfig`
 125                 ;;
 126         --do-kasan|--do-no-kasan)
 127                 do_kasan=`doyesno "$1" --do-kasan`
 128                 ;;
 129         --do-kcsan|--do-no-kcsan)
 130                 do_kcsan=`doyesno "$1" --do-kcsan`
 131                 ;;
 132         --do-kvfree|--do-no-kvfree)
 133                 do_kvfree=`doyesno "$1" --do-kvfree`
 134                 ;;
 135         --do-locktorture|--do-no-locktorture)
 136                 do_locktorture=`doyesno "$1" --do-locktorture`
 137                 ;;
 138         --do-none)
 139                 do_allmodconfig=no
 140                 do_rcutorture=no
 141                 do_locktorture=no
 142                 do_scftorture=no
 143                 do_rcuscale=no
 144                 do_refscale=no
 145                 do_kvfree=no
 146                 do_kasan=no
 147                 do_kcsan=no
 148                 ;;
 149         --do-rcuscale|--do-no-rcuscale)
 150                 do_rcuscale=`doyesno "$1" --do-rcuscale`
 151                 ;;
 152         --do-rcutorture|--do-no-rcutorture)
 153                 do_rcutorture=`doyesno "$1" --do-rcutorture`
 154                 ;;
 155         --do-refscale|--do-no-refscale)
 156                 do_refscale=`doyesno "$1" --do-refscale`
 157                 ;;
 158         --do-scftorture|--do-no-scftorture)
 159                 do_scftorture=`doyesno "$1" --do-scftorture`
 160                 ;;
 161         --duration)
 162                 checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
 163                 mult=1
 164                 if echo "$2" | grep -q 'm$'
 165                 then
 166                         mult=1
 167                 elif echo "$2" | grep -q 'h$'
 168                 then
 169                         mult=60
 170                 elif echo "$2" | grep -q 'd$'
 171                 then
 172                         mult=1440
 173                 fi
 174                 ts=`echo $2 | sed -e 's/[smhd]$//'`
 175                 duration_base=$(($ts*mult))
 176                 shift
 177                 ;;
 178         --kcsan-kmake-arg|--kcsan-kmake-args)
 179                 checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
 180                 kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`"
 181                 shift
 182                 ;;
 183         *)
 184                 echo Unknown argument $1
 185                 usage
 186                 ;;
 187         esac
 188         shift
 189 done
 190
 191 ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
 192 startdate="`date`"
 193 starttime="`get_starttime`"
 194
 195 T=/tmp/torture.sh.$$
 196 trap 'rm -rf $T' 0 2
 197 mkdir $T
 198
 199 echo " --- " $scriptname $args | tee -a $T/log
 200 echo " --- Results directory: " $ds | tee -a $T/log
 201
 202 # Calculate rcutorture defaults and apportion time
 203 if test -z "$configs_rcutorture"
 204 then
 205         configs_rcutorture=CFLIST
 206 fi
 207 duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
 208 if test "$duration_rcutorture" -eq 0
 209 then
 210         echo " --- Zero time for rcutorture, disabling" | tee -a $T/log
 211         do_rcutorture=no
 212 fi
 213
 214 # Calculate locktorture defaults and apportion time
 215 if test -z "$configs_locktorture"
 216 then
 217         configs_locktorture=CFLIST
 218 fi
 219 duration_locktorture=$((duration_base*duration_locktorture_frac/10))
 220 if test "$duration_locktorture" -eq 0
 221 then
 222         echo " --- Zero time for locktorture, disabling" | tee -a $T/log
 223         do_locktorture=no
 224 fi
 225
 226 # Calculate scftorture defaults and apportion time
 227 if test -z "$configs_scftorture"
 228 then
 229         configs_scftorture=CFLIST
 230 fi
 231 duration_scftorture=$((duration_base*duration_scftorture_frac/10))
 232 if test "$duration_scftorture" -eq 0
 233 then
 234         echo " --- Zero time for scftorture, disabling" | tee -a $T/log
 235         do_scftorture=no
 236 fi
 237
 238 touch $T/failures
 239 touch $T/successes
 240
 241 # torture_one - Does a single kvm.sh run.
 242 #
 243 # Usage:
 244 #       torture_bootargs="[ kernel boot arguments ]"
 245 #       torture_one flavor [ kvm.sh arguments ]
 246 #
 247 # Note that "flavor" is an arbitrary string.  Supply --torture if needed.
 248 # Note that quoting is problematic.  So on the command line, pass multiple
 249 # values with multiple kvm.sh argument instances.
 250 function torture_one {
 251         local cur_bootargs=
 252         local boottag=
 253
 254         echo " --- $curflavor:" Start `date` | tee -a $T/log
 255         if test -n "$torture_bootargs"
 256         then
 257                 boottag="--bootargs"
 258                 cur_bootargs="$torture_bootargs"
 259         fi
 260         "$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
 261         retcode=$?
 262         resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
 263         if test -z "$resdir"
 264         then
 265                 cat $T/$curflavor.out | tee -a $T/log
 266                 echo retcode=$retcode | tee -a $T/log
 267         fi
 268         if test "$retcode" == 0
 269         then
 270                 echo "$curflavor($retcode)" $resdir >> $T/successes
 271         else
 272                 echo "$curflavor($retcode)" $resdir >> $T/failures
 273         fi
 274 }
 275
 276 # torture_set - Does a set of tortures with and without KASAN and KCSAN.
 277 #
 278 # Usage:
 279 #       torture_bootargs="[ kernel boot arguments ]"
 280 #       torture_set flavor [ kvm.sh arguments ]
 281 #
 282 # Note that "flavor" is an arbitrary string.  Supply --torture if needed.
 283 # Note that quoting is problematic.  So on the command line, pass multiple
 284 # values with multiple kvm.sh argument instances.
 285 function torture_set {
 286         local cur_kcsan_kmake_args=
 287         local kcsan_kmake_tag=
 288         local flavor=$1
 289         shift
 290         curflavor=$flavor
 291         torture_one "$@"
 292         if test "$do_kasan" = "yes"
 293         then
 294                 curflavor=${flavor}-kasan
 295                 torture_one "$@" --kasan
 296         fi
 297         if test "$do_kcsan" = "yes"
 298         then
 299                 curflavor=${flavor}-kcsan
 300                 if test -n "$kcsan_kmake_args"
 301                 then
 302                         kcsan_kmake_tag="--kmake-args"
 303                         cur_kcsan_kmake_args="$kcsan_kmake_args"
 304                 fi
 305                 torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan
 306         fi
 307 }
 308
 309 # make allmodconfig
 310 if test "$do_allmodconfig" = "yes"
 311 then
 312         echo " --- allmodconfig:" Start `date` | tee -a $T/log
 313         amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
 314         mkdir -p "$amcdir"
 315         echo " --- make clean" > "$amcdir/Make.out" 2>&1
 316         make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1
 317         echo " --- make allmodconfig" >> "$amcdir/Make.out" 2>&1
 318         make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1
 319         echo " --- make " >> "$amcdir/Make.out" 2>&1
 320         make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1
 321         retcode="$?"
 322         echo $retcode > "$amcdir/Make.exitcode"
 323         if test "$retcode" == 0
 324         then
 325                 echo "allmodconfig($retcode)" $amcdir >> $T/successes
 326         else
 327                 echo "allmodconfig($retcode)" $amcdir >> $T/failures
 328         fi
 329 fi
 330
 331 # --torture rcu
 332 if test "$do_rcutorture" = "yes"
 333 then
 334         torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
 335         torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make
 336 fi
 337
 338 if test "$do_locktorture" = "yes"
 339 then
 340         torture_bootargs="torture.disable_onoff_at_boot"
 341         torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make
 342 fi
 343
 344 if test "$do_scftorture" = "yes"
 345 then
 346         torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot"
 347         torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
 348 fi
 349
 350 if test "$do_refscale" = yes
 351 then
 352         primlist="`grep '\.name[        ]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
 353 else
 354         primlist=
 355 fi
 356 for prim in $primlist
 357 do
 358         torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
 359         torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make
 360 done
 361
 362 if test "$do_rcuscale" = yes
 363 then
 364         primlist="`grep '\.name[        ]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
 365 else
 366         primlist=
 367 fi
 368 for prim in $primlist
 369 do
 370         torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
 371         torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
 372 done
 373
 374 if test "$do_kvfree" = "yes"
 375 then
 376         torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
 377         torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make
 378 fi
 379
 380 echo " --- " $scriptname $args
 381 echo " --- " Done `date` | tee -a $T/log
 382 ret=0
 383 nsuccesses=0
 384 echo SUCCESSES: | tee -a $T/log
 385 if test -s "$T/successes"
 386 then
 387         cat "$T/successes" | tee -a $T/log
 388         nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
 389 fi
 390 nfailures=0
 391 echo FAILURES: | tee -a $T/log
 392 if test -s "$T/failures"
 393 then
 394         cat "$T/failures" | tee -a $T/log
 395         nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
 396         ret=2
 397 fi
 398 echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
 399 echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
 400 tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
 401 if test -n "$tdir" && test $compress_kasan_vmlinux -gt 0
 402 then
 403         # KASAN vmlinux files can approach 1GB in size, so compress them.
 404         echo Looking for KASAN files to compress: `date` > "$tdir/log-xz" 2>&1
 405         find "$tdir" -type d -name '*-kasan' -print > $T/xz-todo
 406         ncompresses=0
 407         batchno=1
 408         if test -s $T/xz-todo
 409         then
 410                 echo Size before compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
 411                 for i in `cat $T/xz-todo`
 412                 do
 413                         echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1
 414                         for j in $i/*/vmlinux
 415                         do
 416                                 xz "$j" >> "$tdir/log-xz" 2>&1 &
 417                                 ncompresses=$((ncompresses+1))
 418                                 if test $ncompresses -ge $compress_kasan_vmlinux
 419                                 then
 420                                         echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
 421                                         wait
 422                                         ncompresses=0
 423                                         batchno=$((batchno+1))
 424                                 fi
 425                         done
 426                 done
 427                 if test $ncompresses -gt 0
 428                 then
 429                         echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
 430                 fi
 431                 wait
 432                 echo Size after compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
 433                 echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log
 434         else
 435                 echo No compression needed: `date` >> "$tdir/log-xz" 2>&1
 436         fi
 437 fi
 438 if test -n "$tdir"
 439 then
 440         cp $T/log "$tdir"
 441 fi
 442 exit $ret