tools/testing/kunit/kunit.py

   1 #!/usr/bin/env python3
   2 # SPDX-License-Identifier: GPL-2.0
   3 #
   4 # A thin wrapper on top of the KUnit Kernel
   5 #
   6 # Copyright (C) 2019, Google LLC.
   7 # Author: Felix Guo <felixguoxiuping@gmail.com>
   8 # Author: Brendan Higgins <brendanhiggins@google.com>
   9
  10 import argparse
  11 import sys
  12 import os
  13 import time
  14
  15 assert sys.version_info >= (3, 7), "Python version is too old"
  16
  17 from collections import namedtuple
  18 from enum import Enum, auto
  19
  20 import kunit_config
  21 import kunit_json
  22 import kunit_kernel
  23 import kunit_parser
  24
  25 KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time'])
  26
  27 KunitConfigRequest = namedtuple('KunitConfigRequest',
  28                                 ['build_dir', 'make_options'])
  29 KunitBuildRequest = namedtuple('KunitBuildRequest',
  30                                ['jobs', 'build_dir', 'alltests',
  31                                 'make_options'])
  32 KunitExecRequest = namedtuple('KunitExecRequest',
  33                               ['timeout', 'build_dir', 'alltests', 'filter_glob'])
  34 KunitParseRequest = namedtuple('KunitParseRequest',
  35                                ['raw_output', 'input_data', 'build_dir', 'json'])
  36 KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',
  37                                            'build_dir', 'alltests', 'filter_glob',
  38                                            'json', 'make_options'])
  39
  40 KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0]
  41
  42 class KunitStatus(Enum):
  43         SUCCESS = auto()
  44         CONFIG_FAILURE = auto()
  45         BUILD_FAILURE = auto()
  46         TEST_FAILURE = auto()
  47
  48 def get_kernel_root_path() -> str:
  49         path = sys.argv[0] if not __file__ else __file__
  50         parts = os.path.realpath(path).split('tools/testing/kunit')
  51         if len(parts) != 2:
  52                 sys.exit(1)
  53         return parts[0]
  54
  55 def config_tests(linux: kunit_kernel.LinuxSourceTree,
  56                  request: KunitConfigRequest) -> KunitResult:
  57         kunit_parser.print_with_timestamp('Configuring KUnit Kernel ...')
  58
  59         config_start = time.time()
  60         success = linux.build_reconfig(request.build_dir, request.make_options)
  61         config_end = time.time()
  62         if not success:
  63                 return KunitResult(KunitStatus.CONFIG_FAILURE,
  64                                    'could not configure kernel',
  65                                    config_end - config_start)
  66         return KunitResult(KunitStatus.SUCCESS,
  67                            'configured kernel successfully',
  68                            config_end - config_start)
  69
  70 def build_tests(linux: kunit_kernel.LinuxSourceTree,
  71                 request: KunitBuildRequest) -> KunitResult:
  72         kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
  73
  74         build_start = time.time()
  75         success = linux.build_kernel(request.alltests,
  76                                      request.jobs,
  77                                      request.build_dir,
  78                                      request.make_options)
  79         build_end = time.time()
  80         if not success:
  81                 return KunitResult(KunitStatus.BUILD_FAILURE,
  82                                    'could not build kernel',
  83                                    build_end - build_start)
  84         if not success:
  85                 return KunitResult(KunitStatus.BUILD_FAILURE,
  86                                    'could not build kernel',
  87                                    build_end - build_start)
  88         return KunitResult(KunitStatus.SUCCESS,
  89                            'built kernel successfully',
  90                            build_end - build_start)
  91
  92 def exec_tests(linux: kunit_kernel.LinuxSourceTree,
  93                request: KunitExecRequest) -> KunitResult:
  94         kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
  95         test_start = time.time()
  96         result = linux.run_kernel(
  97                 timeout=None if request.alltests else request.timeout,
  98                 filter_glob=request.filter_glob,
  99                 build_dir=request.build_dir)
 100
 101         test_end = time.time()
 102
 103         return KunitResult(KunitStatus.SUCCESS,
 104                            result,
 105                            test_end - test_start)
 106
 107 def parse_tests(request: KunitParseRequest) -> KunitResult:
 108         parse_start = time.time()
 109
 110         test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS,
 111                                               [],
 112                                               'Tests not Parsed.')
 113
 114         if request.raw_output:
 115                 kunit_parser.raw_output(request.input_data)
 116         else:
 117                 test_result = kunit_parser.parse_run_tests(request.input_data)
 118         parse_end = time.time()
 119
 120         if request.json:
 121                 json_obj = kunit_json.get_json_result(
 122                                         test_result=test_result,
 123                                         def_config='kunit_defconfig',
 124                                         build_dir=request.build_dir,
 125                                         json_path=request.json)
 126                 if request.json == 'stdout':
 127                         print(json_obj)
 128
 129         if test_result.status != kunit_parser.TestStatus.SUCCESS:
 130                 return KunitResult(KunitStatus.TEST_FAILURE, test_result,
 131                                    parse_end - parse_start)
 132
 133         return KunitResult(KunitStatus.SUCCESS, test_result,
 134                                 parse_end - parse_start)
 135
 136
 137 def run_tests(linux: kunit_kernel.LinuxSourceTree,
 138               request: KunitRequest) -> KunitResult:
 139         run_start = time.time()
 140
 141         config_request = KunitConfigRequest(request.build_dir,
 142                                             request.make_options)
 143         config_result = config_tests(linux, config_request)
 144         if config_result.status != KunitStatus.SUCCESS:
 145                 return config_result
 146
 147         build_request = KunitBuildRequest(request.jobs, request.build_dir,
 148                                           request.alltests,
 149                                           request.make_options)
 150         build_result = build_tests(linux, build_request)
 151         if build_result.status != KunitStatus.SUCCESS:
 152                 return build_result
 153
 154         exec_request = KunitExecRequest(request.timeout, request.build_dir,
 155                                         request.alltests, request.filter_glob)
 156         exec_result = exec_tests(linux, exec_request)
 157         if exec_result.status != KunitStatus.SUCCESS:
 158                 return exec_result
 159
 160         parse_request = KunitParseRequest(request.raw_output,
 161                                           exec_result.result,
 162                                           request.build_dir,
 163                                           request.json)
 164         parse_result = parse_tests(parse_request)
 165
 166         run_end = time.time()
 167
 168         kunit_parser.print_with_timestamp((
 169                 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
 170                 'building, %.3fs running\n') % (
 171                                 run_end - run_start,
 172                                 config_result.elapsed_time,
 173                                 build_result.elapsed_time,
 174                                 exec_result.elapsed_time))
 175         return parse_result
 176
 177 def add_common_opts(parser) -> None:
 178         parser.add_argument('--build_dir',
 179                             help='As in the make command, it specifies the build '
 180                             'directory.',
 181                             type=str, default='.kunit', metavar='build_dir')
 182         parser.add_argument('--make_options',
 183                             help='X=Y make option, can be repeated.',
 184                             action='append')
 185         parser.add_argument('--alltests',
 186                             help='Run all KUnit tests through allyesconfig',
 187                             action='store_true')
 188         parser.add_argument('--kunitconfig',
 189                              help='Path to Kconfig fragment that enables KUnit tests.'
 190                              ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" '
 191                              'will get  automatically appended.',
 192                              metavar='kunitconfig')
 193
 194         parser.add_argument('--arch',
 195                             help=('Specifies the architecture to run tests under. '
 196                                   'The architecture specified here must match the '
 197                                   'string passed to the ARCH make param, '
 198                                   'e.g. i386, x86_64, arm, um, etc. Non-UML '
 199                                   'architectures run on QEMU.'),
 200                             type=str, default='um', metavar='arch')
 201
 202         parser.add_argument('--cross_compile',
 203                             help=('Sets make\'s CROSS_COMPILE variable; it should '
 204                                   'be set to a toolchain path prefix (the prefix '
 205                                   'of gcc and other tools in your toolchain, for '
 206                                   'example `sparc64-linux-gnu-` if you have the '
 207                                   'sparc toolchain installed on your system, or '
 208                                   '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` '
 209                                   'if you have downloaded the microblaze toolchain '
 210                                   'from the 0-day website to a directory in your '
 211                                   'home directory called `toolchains`).'),
 212                             metavar='cross_compile')
 213
 214         parser.add_argument('--qemu_config',
 215                             help=('Takes a path to a path to a file containing '
 216                                   'a QemuArchParams object.'),
 217                             type=str, metavar='qemu_config')
 218
 219 def add_build_opts(parser) -> None:
 220         parser.add_argument('--jobs',
 221                             help='As in the make command, "Specifies  the number of '
 222                             'jobs (commands) to run simultaneously."',
 223                             type=int, default=8, metavar='jobs')
 224
 225 def add_exec_opts(parser) -> None:
 226         parser.add_argument('--timeout',
 227                             help='maximum number of seconds to allow for all tests '
 228                             'to run. This does not include time taken to build the '
 229                             'tests.',
 230                             type=int,
 231                             default=300,
 232                             metavar='timeout')
 233         parser.add_argument('filter_glob',
 234                             help='maximum number of seconds to allow for all tests '
 235                             'to run. This does not include time taken to build the '
 236                             'tests.',
 237                             type=str,
 238                             nargs='?',
 239                             default='',
 240                             metavar='filter_glob')
 241
 242 def add_parse_opts(parser) -> None:
 243         parser.add_argument('--raw_output', help='don\'t format output from kernel',
 244                             action='store_true')
 245         parser.add_argument('--json',
 246                             nargs='?',
 247                             help='Stores test results in a JSON, and either '
 248                             'prints to stdout or saves to file if a '
 249                             'filename is specified',
 250                             type=str, const='stdout', default=None)
 251
 252 def main(argv, linux=None):
 253         parser = argparse.ArgumentParser(
 254                         description='Helps writing and running KUnit tests.')
 255         subparser = parser.add_subparsers(dest='subcommand')
 256
 257         # The 'run' command will config, build, exec, and parse in one go.
 258         run_parser = subparser.add_parser('run', help='Runs KUnit tests.')
 259         add_common_opts(run_parser)
 260         add_build_opts(run_parser)
 261         add_exec_opts(run_parser)
 262         add_parse_opts(run_parser)
 263
 264         config_parser = subparser.add_parser('config',
 265                                                 help='Ensures that .config contains all of '
 266                                                 'the options in .kunitconfig')
 267         add_common_opts(config_parser)
 268
 269         build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests')
 270         add_common_opts(build_parser)
 271         add_build_opts(build_parser)
 272
 273         exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests')
 274         add_common_opts(exec_parser)
 275         add_exec_opts(exec_parser)
 276         add_parse_opts(exec_parser)
 277
 278         # The 'parse' option is special, as it doesn't need the kernel source
 279         # (therefore there is no need for a build_dir, hence no add_common_opts)
 280         # and the '--file' argument is not relevant to 'run', so isn't in
 281         # add_parse_opts()
 282         parse_parser = subparser.add_parser('parse',
 283                                             help='Parses KUnit results from a file, '
 284                                             'and parses formatted results.')
 285         add_parse_opts(parse_parser)
 286         parse_parser.add_argument('file',
 287                                   help='Specifies the file to read results from.',
 288                                   type=str, nargs='?', metavar='input_file')
 289
 290         cli_args = parser.parse_args(argv)
 291
 292         if get_kernel_root_path():
 293                 os.chdir(get_kernel_root_path())
 294
 295         if cli_args.subcommand == 'run':
 296                 if not os.path.exists(cli_args.build_dir):
 297                         os.mkdir(cli_args.build_dir)
 298
 299                 if not linux:
 300                         linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir,
 301                                         kunitconfig_path=cli_args.kunitconfig,
 302                                         arch=cli_args.arch,
 303                                         cross_compile=cli_args.cross_compile,
 304                                         qemu_config_path=cli_args.qemu_config)
 305
 306                 request = KunitRequest(cli_args.raw_output,
 307                                        cli_args.timeout,
 308                                        cli_args.jobs,
 309                                        cli_args.build_dir,
 310                                        cli_args.alltests,
 311                                        cli_args.filter_glob,
 312                                        cli_args.json,
 313                                        cli_args.make_options)
 314                 result = run_tests(linux, request)
 315                 if result.status != KunitStatus.SUCCESS:
 316                         sys.exit(1)
 317         elif cli_args.subcommand == 'config':
 318                 if cli_args.build_dir and (
 319                                 not os.path.exists(cli_args.build_dir)):
 320                         os.mkdir(cli_args.build_dir)
 321
 322                 if not linux:
 323                         linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir,
 324                                         kunitconfig_path=cli_args.kunitconfig,
 325                                         arch=cli_args.arch,
 326                                         cross_compile=cli_args.cross_compile,
 327                                         qemu_config_path=cli_args.qemu_config)
 328
 329                 request = KunitConfigRequest(cli_args.build_dir,
 330                                              cli_args.make_options)
 331                 result = config_tests(linux, request)
 332                 kunit_parser.print_with_timestamp((
 333                         'Elapsed time: %.3fs\n') % (
 334                                 result.elapsed_time))
 335                 if result.status != KunitStatus.SUCCESS:
 336                         sys.exit(1)
 337         elif cli_args.subcommand == 'build':
 338                 if not linux:
 339                         linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir,
 340                                         kunitconfig_path=cli_args.kunitconfig,
 341                                         arch=cli_args.arch,
 342                                         cross_compile=cli_args.cross_compile,
 343                                         qemu_config_path=cli_args.qemu_config)
 344
 345                 request = KunitBuildRequest(cli_args.jobs,
 346                                             cli_args.build_dir,
 347                                             cli_args.alltests,
 348                                             cli_args.make_options)
 349                 result = build_tests(linux, request)
 350                 kunit_parser.print_with_timestamp((
 351                         'Elapsed time: %.3fs\n') % (
 352                                 result.elapsed_time))
 353                 if result.status != KunitStatus.SUCCESS:
 354                         sys.exit(1)
 355         elif cli_args.subcommand == 'exec':
 356                 if not linux:
 357                         linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir,
 358                                         kunitconfig_path=cli_args.kunitconfig,
 359                                         arch=cli_args.arch,
 360                                         cross_compile=cli_args.cross_compile,
 361                                         qemu_config_path=cli_args.qemu_config)
 362
 363                 exec_request = KunitExecRequest(cli_args.timeout,
 364                                                 cli_args.build_dir,
 365                                                 cli_args.alltests,
 366                                                 cli_args.filter_glob)
 367                 exec_result = exec_tests(linux, exec_request)
 368                 parse_request = KunitParseRequest(cli_args.raw_output,
 369                                                   exec_result.result,
 370                                                   cli_args.build_dir,
 371                                                   cli_args.json)
 372                 result = parse_tests(parse_request)
 373                 kunit_parser.print_with_timestamp((
 374                         'Elapsed time: %.3fs\n') % (
 375                                 exec_result.elapsed_time))
 376                 if result.status != KunitStatus.SUCCESS:
 377                         sys.exit(1)
 378         elif cli_args.subcommand == 'parse':
 379                 if cli_args.file == None:
 380                         kunit_output = sys.stdin
 381                 else:
 382                         with open(cli_args.file, 'r') as f:
 383                                 kunit_output = f.read().splitlines()
 384                 request = KunitParseRequest(cli_args.raw_output,
 385                                             kunit_output,
 386                                             None,
 387                                             cli_args.json)
 388                 result = parse_tests(request)
 389                 if result.status != KunitStatus.SUCCESS:
 390                         sys.exit(1)
 391         else:
 392                 parser.print_help()
 393
 394 if __name__ == '__main__':
 395         main(sys.argv[1:])