tools/testing/kunit/kunit.py

   1 #!/usr/bin/env python3
   2 # SPDX-License-Identifier: GPL-2.0
   3 #
   4 # A thin wrapper on top of the KUnit Kernel
   5 #
   6 # Copyright (C) 2019, Google LLC.
   7 # Author: Felix Guo <felixguoxiuping@gmail.com>
   8 # Author: Brendan Higgins <brendanhiggins@google.com>
   9
  10 import argparse
  11 import sys
  12 import os
  13 import time
  14
  15 from collections import namedtuple
  16 from enum import Enum, auto
  17
  18 import kunit_config
  19 import kunit_json
  20 import kunit_kernel
  21 import kunit_parser
  22
  23 KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time'])
  24
  25 KunitConfigRequest = namedtuple('KunitConfigRequest',
  26                                 ['build_dir', 'make_options'])
  27 KunitBuildRequest = namedtuple('KunitBuildRequest',
  28                                ['jobs', 'build_dir', 'alltests',
  29                                 'make_options'])
  30 KunitExecRequest = namedtuple('KunitExecRequest',
  31                               ['timeout', 'build_dir', 'alltests'])
  32 KunitParseRequest = namedtuple('KunitParseRequest',
  33                                ['raw_output', 'input_data', 'build_dir', 'json'])
  34 KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',
  35                                            'build_dir', 'alltests', 'json',
  36                                            'make_options'])
  37
  38 KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0]
  39
  40 class KunitStatus(Enum):
  41         SUCCESS = auto()
  42         CONFIG_FAILURE = auto()
  43         BUILD_FAILURE = auto()
  44         TEST_FAILURE = auto()
  45
  46 def get_kernel_root_path():
  47         parts = sys.argv[0] if not __file__ else __file__
  48         parts = os.path.realpath(parts).split('tools/testing/kunit')
  49         if len(parts) != 2:
  50                 sys.exit(1)
  51         return parts[0]
  52
  53 def config_tests(linux: kunit_kernel.LinuxSourceTree,
  54                  request: KunitConfigRequest) -> KunitResult:
  55         kunit_parser.print_with_timestamp('Configuring KUnit Kernel ...')
  56
  57         config_start = time.time()
  58         success = linux.build_reconfig(request.build_dir, request.make_options)
  59         config_end = time.time()
  60         if not success:
  61                 return KunitResult(KunitStatus.CONFIG_FAILURE,
  62                                    'could not configure kernel',
  63                                    config_end - config_start)
  64         return KunitResult(KunitStatus.SUCCESS,
  65                            'configured kernel successfully',
  66                            config_end - config_start)
  67
  68 def build_tests(linux: kunit_kernel.LinuxSourceTree,
  69                 request: KunitBuildRequest) -> KunitResult:
  70         kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
  71
  72         build_start = time.time()
  73         success = linux.build_um_kernel(request.alltests,
  74                                         request.jobs,
  75                                         request.build_dir,
  76                                         request.make_options)
  77         build_end = time.time()
  78         if not success:
  79                 return KunitResult(KunitStatus.BUILD_FAILURE,
  80                                    'could not build kernel',
  81                                    build_end - build_start)
  82         if not success:
  83                 return KunitResult(KunitStatus.BUILD_FAILURE,
  84                                    'could not build kernel',
  85                                    build_end - build_start)
  86         return KunitResult(KunitStatus.SUCCESS,
  87                            'built kernel successfully',
  88                            build_end - build_start)
  89
  90 def exec_tests(linux: kunit_kernel.LinuxSourceTree,
  91                request: KunitExecRequest) -> KunitResult:
  92         kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
  93         test_start = time.time()
  94         result = linux.run_kernel(
  95                 timeout=None if request.alltests else request.timeout,
  96                 build_dir=request.build_dir)
  97
  98         test_end = time.time()
  99
 100         return KunitResult(KunitStatus.SUCCESS,
 101                            result,
 102                            test_end - test_start)
 103
 104 def parse_tests(request: KunitParseRequest) -> KunitResult:
 105         parse_start = time.time()
 106
 107         test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS,
 108                                               [],
 109                                               'Tests not Parsed.')
 110
 111         if request.raw_output:
 112                 kunit_parser.raw_output(request.input_data)
 113         else:
 114                 test_result = kunit_parser.parse_run_tests(request.input_data)
 115         parse_end = time.time()
 116
 117         if request.json:
 118                 json_obj = kunit_json.get_json_result(
 119                                         test_result=test_result,
 120                                         def_config='kunit_defconfig',
 121                                         build_dir=request.build_dir,
 122                                         json_path=request.json)
 123                 if request.json == 'stdout':
 124                         print(json_obj)
 125
 126         if test_result.status != kunit_parser.TestStatus.SUCCESS:
 127                 return KunitResult(KunitStatus.TEST_FAILURE, test_result,
 128                                    parse_end - parse_start)
 129
 130         return KunitResult(KunitStatus.SUCCESS, test_result,
 131                                 parse_end - parse_start)
 132
 133
 134 def run_tests(linux: kunit_kernel.LinuxSourceTree,
 135               request: KunitRequest) -> KunitResult:
 136         run_start = time.time()
 137
 138         config_request = KunitConfigRequest(request.build_dir,
 139                                             request.make_options)
 140         config_result = config_tests(linux, config_request)
 141         if config_result.status != KunitStatus.SUCCESS:
 142                 return config_result
 143
 144         build_request = KunitBuildRequest(request.jobs, request.build_dir,
 145                                           request.alltests,
 146                                           request.make_options)
 147         build_result = build_tests(linux, build_request)
 148         if build_result.status != KunitStatus.SUCCESS:
 149                 return build_result
 150
 151         exec_request = KunitExecRequest(request.timeout, request.build_dir,
 152                                         request.alltests)
 153         exec_result = exec_tests(linux, exec_request)
 154         if exec_result.status != KunitStatus.SUCCESS:
 155                 return exec_result
 156
 157         parse_request = KunitParseRequest(request.raw_output,
 158                                           exec_result.result,
 159                                           request.build_dir,
 160                                           request.json)
 161         parse_result = parse_tests(parse_request)
 162
 163         run_end = time.time()
 164
 165         kunit_parser.print_with_timestamp((
 166                 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
 167                 'building, %.3fs running\n') % (
 168                                 run_end - run_start,
 169                                 config_result.elapsed_time,
 170                                 build_result.elapsed_time,
 171                                 exec_result.elapsed_time))
 172         return parse_result
 173
 174 def add_common_opts(parser):
 175         parser.add_argument('--build_dir',
 176                             help='As in the make command, it specifies the build '
 177                             'directory.',
 178                             type=str, default='.kunit', metavar='build_dir')
 179         parser.add_argument('--make_options',
 180                             help='X=Y make option, can be repeated.',
 181                             action='append')
 182         parser.add_argument('--alltests',
 183                             help='Run all KUnit tests through allyesconfig',
 184                             action='store_true')
 185
 186 def add_build_opts(parser):
 187         parser.add_argument('--jobs',
 188                             help='As in the make command, "Specifies  the number of '
 189                             'jobs (commands) to run simultaneously."',
 190                             type=int, default=8, metavar='jobs')
 191
 192 def add_exec_opts(parser):
 193         parser.add_argument('--timeout',
 194                             help='maximum number of seconds to allow for all tests '
 195                             'to run. This does not include time taken to build the '
 196                             'tests.',
 197                             type=int,
 198                             default=300,
 199                             metavar='timeout')
 200
 201 def add_parse_opts(parser):
 202         parser.add_argument('--raw_output', help='don\'t format output from kernel',
 203                             action='store_true')
 204         parser.add_argument('--json',
 205                             nargs='?',
 206                             help='Stores test results in a JSON, and either '
 207                             'prints to stdout or saves to file if a '
 208                             'filename is specified',
 209                             type=str, const='stdout', default=None)
 210
 211 def main(argv, linux=None):
 212         parser = argparse.ArgumentParser(
 213                         description='Helps writing and running KUnit tests.')
 214         subparser = parser.add_subparsers(dest='subcommand')
 215
 216         # The 'run' command will config, build, exec, and parse in one go.
 217         run_parser = subparser.add_parser('run', help='Runs KUnit tests.')
 218         add_common_opts(run_parser)
 219         add_build_opts(run_parser)
 220         add_exec_opts(run_parser)
 221         add_parse_opts(run_parser)
 222
 223         config_parser = subparser.add_parser('config',
 224                                                 help='Ensures that .config contains all of '
 225                                                 'the options in .kunitconfig')
 226         add_common_opts(config_parser)
 227
 228         build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests')
 229         add_common_opts(build_parser)
 230         add_build_opts(build_parser)
 231
 232         exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests')
 233         add_common_opts(exec_parser)
 234         add_exec_opts(exec_parser)
 235         add_parse_opts(exec_parser)
 236
 237         # The 'parse' option is special, as it doesn't need the kernel source
 238         # (therefore there is no need for a build_dir, hence no add_common_opts)
 239         # and the '--file' argument is not relevant to 'run', so isn't in
 240         # add_parse_opts()
 241         parse_parser = subparser.add_parser('parse',
 242                                             help='Parses KUnit results from a file, '
 243                                             'and parses formatted results.')
 244         add_parse_opts(parse_parser)
 245         parse_parser.add_argument('file',
 246                                   help='Specifies the file to read results from.',
 247                                   type=str, nargs='?', metavar='input_file')
 248
 249         cli_args = parser.parse_args(argv)
 250
 251         if get_kernel_root_path():
 252                 os.chdir(get_kernel_root_path())
 253
 254         if cli_args.subcommand == 'run':
 255                 if not os.path.exists(cli_args.build_dir):
 256                         os.mkdir(cli_args.build_dir)
 257
 258                 if not linux:
 259                         linux = kunit_kernel.LinuxSourceTree()
 260
 261                 linux.create_kunitconfig(cli_args.build_dir)
 262                 linux.read_kunitconfig(cli_args.build_dir)
 263
 264                 request = KunitRequest(cli_args.raw_output,
 265                                        cli_args.timeout,
 266                                        cli_args.jobs,
 267                                        cli_args.build_dir,
 268                                        cli_args.alltests,
 269                                        cli_args.json,
 270                                        cli_args.make_options)
 271                 result = run_tests(linux, request)
 272                 if result.status != KunitStatus.SUCCESS:
 273                         sys.exit(1)
 274         elif cli_args.subcommand == 'config':
 275                 if cli_args.build_dir and (
 276                                 not os.path.exists(cli_args.build_dir)):
 277                         os.mkdir(cli_args.build_dir)
 278
 279                 if not linux:
 280                         linux = kunit_kernel.LinuxSourceTree()
 281
 282                 linux.create_kunitconfig(cli_args.build_dir)
 283                 linux.read_kunitconfig(cli_args.build_dir)
 284
 285                 request = KunitConfigRequest(cli_args.build_dir,
 286                                              cli_args.make_options)
 287                 result = config_tests(linux, request)
 288                 kunit_parser.print_with_timestamp((
 289                         'Elapsed time: %.3fs\n') % (
 290                                 result.elapsed_time))
 291                 if result.status != KunitStatus.SUCCESS:
 292                         sys.exit(1)
 293         elif cli_args.subcommand == 'build':
 294                 if not linux:
 295                         linux = kunit_kernel.LinuxSourceTree()
 296
 297                 linux.create_kunitconfig(cli_args.build_dir)
 298                 linux.read_kunitconfig(cli_args.build_dir)
 299
 300                 request = KunitBuildRequest(cli_args.jobs,
 301                                             cli_args.build_dir,
 302                                             cli_args.alltests,
 303                                             cli_args.make_options)
 304                 result = build_tests(linux, request)
 305                 kunit_parser.print_with_timestamp((
 306                         'Elapsed time: %.3fs\n') % (
 307                                 result.elapsed_time))
 308                 if result.status != KunitStatus.SUCCESS:
 309                         sys.exit(1)
 310         elif cli_args.subcommand == 'exec':
 311                 if not linux:
 312                         linux = kunit_kernel.LinuxSourceTree()
 313
 314                 linux.create_kunitconfig(cli_args.build_dir)
 315                 linux.read_kunitconfig(cli_args.build_dir)
 316
 317                 exec_request = KunitExecRequest(cli_args.timeout,
 318                                                 cli_args.build_dir,
 319                                                 cli_args.alltests)
 320                 exec_result = exec_tests(linux, exec_request)
 321                 parse_request = KunitParseRequest(cli_args.raw_output,
 322                                                   exec_result.result,
 323                                                   cli_args.build_dir,
 324                                                   cli_args.json)
 325                 result = parse_tests(parse_request)
 326                 kunit_parser.print_with_timestamp((
 327                         'Elapsed time: %.3fs\n') % (
 328                                 exec_result.elapsed_time))
 329                 if result.status != KunitStatus.SUCCESS:
 330                         sys.exit(1)
 331         elif cli_args.subcommand == 'parse':
 332                 if cli_args.file == None:
 333                         kunit_output = sys.stdin
 334                 else:
 335                         with open(cli_args.file, 'r') as f:
 336                                 kunit_output = f.read().splitlines()
 337                 request = KunitParseRequest(cli_args.raw_output,
 338                                             kunit_output,
 339                                             None,
 340                                             cli_args.json)
 341                 result = parse_tests(request)
 342                 if result.status != KunitStatus.SUCCESS:
 343                         sys.exit(1)
 344         else:
 345                 parser.print_help()
 346
 347 if __name__ == '__main__':
 348         main(sys.argv[1:])