tools/testing/kunit/kunit_tool_test.py

   1 #!/usr/bin/python3
   2 # SPDX-License-Identifier: GPL-2.0
   3 #
   4 # A collection of tests for tools/testing/kunit/kunit.py
   5 #
   6 # Copyright (C) 2019, Google LLC.
   7 # Author: Brendan Higgins <brendanhiggins@google.com>
   8
   9 import unittest
  10 from unittest import mock
  11
  12 import tempfile, shutil # Handling test_tmpdir
  13
  14 import os
  15
  16 import kunit_config
  17 import kunit_parser
  18 import kunit_kernel
  19 import kunit
  20
  21 test_tmpdir = ''
  22
  23 def setUpModule():
  24         global test_tmpdir
  25         test_tmpdir = tempfile.mkdtemp()
  26
  27 def tearDownModule():
  28         shutil.rmtree(test_tmpdir)
  29
  30 def get_absolute_path(path):
  31         return os.path.join(os.path.dirname(__file__), path)
  32
  33 class KconfigTest(unittest.TestCase):
  34
  35         def test_is_subset_of(self):
  36                 kconfig0 = kunit_config.Kconfig()
  37                 self.assertTrue(kconfig0.is_subset_of(kconfig0))
  38
  39                 kconfig1 = kunit_config.Kconfig()
  40                 kconfig1.add_entry(kunit_config.KconfigEntry('TEST', 'y'))
  41                 self.assertTrue(kconfig1.is_subset_of(kconfig1))
  42                 self.assertTrue(kconfig0.is_subset_of(kconfig1))
  43                 self.assertFalse(kconfig1.is_subset_of(kconfig0))
  44
  45         def test_read_from_file(self):
  46                 kconfig = kunit_config.Kconfig()
  47                 kconfig_path = get_absolute_path(
  48                         'test_data/test_read_from_file.kconfig')
  49
  50                 kconfig.read_from_file(kconfig_path)
  51
  52                 expected_kconfig = kunit_config.Kconfig()
  53                 expected_kconfig.add_entry(
  54                         kunit_config.KconfigEntry('UML', 'y'))
  55                 expected_kconfig.add_entry(
  56                         kunit_config.KconfigEntry('MMU', 'y'))
  57                 expected_kconfig.add_entry(
  58                         kunit_config.KconfigEntry('TEST', 'y'))
  59                 expected_kconfig.add_entry(
  60                         kunit_config.KconfigEntry('EXAMPLE_TEST', 'y'))
  61                 expected_kconfig.add_entry(
  62                         kunit_config.KconfigEntry('MK8', 'n'))
  63
  64                 self.assertEqual(kconfig.entries(), expected_kconfig.entries())
  65
  66         def test_write_to_file(self):
  67                 kconfig_path = os.path.join(test_tmpdir, '.config')
  68
  69                 expected_kconfig = kunit_config.Kconfig()
  70                 expected_kconfig.add_entry(
  71                         kunit_config.KconfigEntry('UML', 'y'))
  72                 expected_kconfig.add_entry(
  73                         kunit_config.KconfigEntry('MMU', 'y'))
  74                 expected_kconfig.add_entry(
  75                         kunit_config.KconfigEntry('TEST', 'y'))
  76                 expected_kconfig.add_entry(
  77                         kunit_config.KconfigEntry('EXAMPLE_TEST', 'y'))
  78                 expected_kconfig.add_entry(
  79                         kunit_config.KconfigEntry('MK8', 'n'))
  80
  81                 expected_kconfig.write_to_file(kconfig_path)
  82
  83                 actual_kconfig = kunit_config.Kconfig()
  84                 actual_kconfig.read_from_file(kconfig_path)
  85
  86                 self.assertEqual(actual_kconfig.entries(),
  87                                  expected_kconfig.entries())
  88
  89 class KUnitParserTest(unittest.TestCase):
  90
  91         def assertContains(self, needle, haystack):
  92                 for line in haystack:
  93                         if needle in line:
  94                                 return
  95                 raise AssertionError('"' +
  96                         str(needle) + '" not found in "' + str(haystack) + '"!')
  97
  98         def test_output_isolated_correctly(self):
  99                 log_path = get_absolute_path(
 100                         'test_data/test_output_isolated_correctly.log')
 101                 file = open(log_path)
 102                 result = kunit_parser.isolate_kunit_output(file.readlines())
 103                 self.assertContains('TAP version 14\n', result)
 104                 self.assertContains('   # Subtest: example', result)
 105                 self.assertContains('   1..2', result)
 106                 self.assertContains('   ok 1 - example_simple_test', result)
 107                 self.assertContains('   ok 2 - example_mock_test', result)
 108                 self.assertContains('ok 1 - example', result)
 109                 file.close()
 110
 111         def test_output_with_prefix_isolated_correctly(self):
 112                 log_path = get_absolute_path(
 113                         'test_data/test_pound_sign.log')
 114                 with open(log_path) as file:
 115                         result = kunit_parser.isolate_kunit_output(file.readlines())
 116                 self.assertContains('TAP version 14\n', result)
 117                 self.assertContains('   # Subtest: kunit-resource-test', result)
 118                 self.assertContains('   1..5', result)
 119                 self.assertContains('   ok 1 - kunit_resource_test_init_resources', result)
 120                 self.assertContains('   ok 2 - kunit_resource_test_alloc_resource', result)
 121                 self.assertContains('   ok 3 - kunit_resource_test_destroy_resource', result)
 122                 self.assertContains(' foo bar   #', result)
 123                 self.assertContains('   ok 4 - kunit_resource_test_cleanup_resources', result)
 124                 self.assertContains('   ok 5 - kunit_resource_test_proper_free_ordering', result)
 125                 self.assertContains('ok 1 - kunit-resource-test', result)
 126                 self.assertContains(' foo bar   # non-kunit output', result)
 127                 self.assertContains('   # Subtest: kunit-try-catch-test', result)
 128                 self.assertContains('   1..2', result)
 129                 self.assertContains('   ok 1 - kunit_test_try_catch_successful_try_no_catch',
 130                                     result)
 131                 self.assertContains('   ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch',
 132                                     result)
 133                 self.assertContains('ok 2 - kunit-try-catch-test', result)
 134                 self.assertContains('   # Subtest: string-stream-test', result)
 135                 self.assertContains('   1..3', result)
 136                 self.assertContains('   ok 1 - string_stream_test_empty_on_creation', result)
 137                 self.assertContains('   ok 2 - string_stream_test_not_empty_after_add', result)
 138                 self.assertContains('   ok 3 - string_stream_test_get_string', result)
 139                 self.assertContains('ok 3 - string-stream-test', result)
 140
 141         def test_parse_successful_test_log(self):
 142                 all_passed_log = get_absolute_path(
 143                         'test_data/test_is_test_passed-all_passed.log')
 144                 file = open(all_passed_log)
 145                 result = kunit_parser.parse_run_tests(file.readlines())
 146                 self.assertEqual(
 147                         kunit_parser.TestStatus.SUCCESS,
 148                         result.status)
 149                 file.close()
 150
 151         def test_parse_failed_test_log(self):
 152                 failed_log = get_absolute_path(
 153                         'test_data/test_is_test_passed-failure.log')
 154                 file = open(failed_log)
 155                 result = kunit_parser.parse_run_tests(file.readlines())
 156                 self.assertEqual(
 157                         kunit_parser.TestStatus.FAILURE,
 158                         result.status)
 159                 file.close()
 160
 161         def test_no_tests(self):
 162                 empty_log = get_absolute_path(
 163                         'test_data/test_is_test_passed-no_tests_run.log')
 164                 file = open(empty_log)
 165                 result = kunit_parser.parse_run_tests(
 166                         kunit_parser.isolate_kunit_output(file.readlines()))
 167                 self.assertEqual(0, len(result.suites))
 168                 self.assertEqual(
 169                         kunit_parser.TestStatus.NO_TESTS,
 170                         result.status)
 171                 file.close()
 172
 173         def test_crashed_test(self):
 174                 crashed_log = get_absolute_path(
 175                         'test_data/test_is_test_passed-crash.log')
 176                 file = open(crashed_log)
 177                 result = kunit_parser.parse_run_tests(file.readlines())
 178                 self.assertEqual(
 179                         kunit_parser.TestStatus.TEST_CRASHED,
 180                         result.status)
 181                 file.close()
 182
 183         def test_ignores_prefix_printk_time(self):
 184                 prefix_log = get_absolute_path(
 185                         'test_data/test_config_printk_time.log')
 186                 with open(prefix_log) as file:
 187                         result = kunit_parser.parse_run_tests(file.readlines())
 188                 self.assertEqual('kunit-resource-test', result.suites[0].name)
 189
 190         def test_ignores_multiple_prefixes(self):
 191                 prefix_log = get_absolute_path(
 192                         'test_data/test_multiple_prefixes.log')
 193                 with open(prefix_log) as file:
 194                         result = kunit_parser.parse_run_tests(file.readlines())
 195                 self.assertEqual('kunit-resource-test', result.suites[0].name)
 196
 197         def test_prefix_mixed_kernel_output(self):
 198                 mixed_prefix_log = get_absolute_path(
 199                         'test_data/test_interrupted_tap_output.log')
 200                 with open(mixed_prefix_log) as file:
 201                         result = kunit_parser.parse_run_tests(file.readlines())
 202                 self.assertEqual('kunit-resource-test', result.suites[0].name)
 203
 204         def test_prefix_poundsign(self):
 205                 pound_log = get_absolute_path('test_data/test_pound_sign.log')
 206                 with open(pound_log) as file:
 207                         result = kunit_parser.parse_run_tests(file.readlines())
 208                 self.assertEqual('kunit-resource-test', result.suites[0].name)
 209
 210         def test_kernel_panic_end(self):
 211                 panic_log = get_absolute_path('test_data/test_kernel_panic_interrupt.log')
 212                 with open(panic_log) as file:
 213                         result = kunit_parser.parse_run_tests(file.readlines())
 214                 self.assertEqual('kunit-resource-test', result.suites[0].name)
 215
 216         def test_pound_no_prefix(self):
 217                 pound_log = get_absolute_path('test_data/test_pound_no_prefix.log')
 218                 with open(pound_log) as file:
 219                         result = kunit_parser.parse_run_tests(file.readlines())
 220                 self.assertEqual('kunit-resource-test', result.suites[0].name)
 221
 222 class StrContains(str):
 223         def __eq__(self, other):
 224                 return self in other
 225
 226 class KUnitMainTest(unittest.TestCase):
 227         def setUp(self):
 228                 path = get_absolute_path('test_data/test_is_test_passed-all_passed.log')
 229                 file = open(path)
 230                 all_passed_log = file.readlines()
 231                 self.print_patch = mock.patch('builtins.print')
 232                 self.print_mock = self.print_patch.start()
 233                 self.linux_source_mock = mock.Mock()
 234                 self.linux_source_mock.build_reconfig = mock.Mock(return_value=True)
 235                 self.linux_source_mock.build_um_kernel = mock.Mock(return_value=True)
 236                 self.linux_source_mock.run_kernel = mock.Mock(return_value=all_passed_log)
 237
 238         def tearDown(self):
 239                 self.print_patch.stop()
 240                 pass
 241
 242         def test_config_passes_args_pass(self):
 243                 kunit.main(['config'], self.linux_source_mock)
 244                 assert self.linux_source_mock.build_reconfig.call_count == 1
 245                 assert self.linux_source_mock.run_kernel.call_count == 0
 246
 247         def test_build_passes_args_pass(self):
 248                 kunit.main(['build'], self.linux_source_mock)
 249                 assert self.linux_source_mock.build_reconfig.call_count == 0
 250                 self.linux_source_mock.build_um_kernel.assert_called_once_with(False, 8, '', None)
 251                 assert self.linux_source_mock.run_kernel.call_count == 0
 252
 253         def test_exec_passes_args_pass(self):
 254                 kunit.main(['exec'], self.linux_source_mock)
 255                 assert self.linux_source_mock.build_reconfig.call_count == 0
 256                 assert self.linux_source_mock.run_kernel.call_count == 1
 257                 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='', timeout=300)
 258                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 259
 260         def test_run_passes_args_pass(self):
 261                 kunit.main(['run'], self.linux_source_mock)
 262                 assert self.linux_source_mock.build_reconfig.call_count == 1
 263                 assert self.linux_source_mock.run_kernel.call_count == 1
 264                 self.linux_source_mock.run_kernel.assert_called_once_with(
 265                         build_dir='', timeout=300)
 266                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 267
 268         def test_exec_passes_args_fail(self):
 269                 self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
 270                 with self.assertRaises(SystemExit) as e:
 271                         kunit.main(['exec'], self.linux_source_mock)
 272                 assert type(e.exception) == SystemExit
 273                 assert e.exception.code == 1
 274
 275         def test_run_passes_args_fail(self):
 276                 self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
 277                 with self.assertRaises(SystemExit) as e:
 278                         kunit.main(['run'], self.linux_source_mock)
 279                 assert type(e.exception) == SystemExit
 280                 assert e.exception.code == 1
 281                 assert self.linux_source_mock.build_reconfig.call_count == 1
 282                 assert self.linux_source_mock.run_kernel.call_count == 1
 283                 self.print_mock.assert_any_call(StrContains(' 0 tests run'))
 284
 285         def test_exec_raw_output(self):
 286                 self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
 287                 kunit.main(['exec', '--raw_output'], self.linux_source_mock)
 288                 assert self.linux_source_mock.run_kernel.call_count == 1
 289                 for kall in self.print_mock.call_args_list:
 290                         assert kall != mock.call(StrContains('Testing complete.'))
 291                         assert kall != mock.call(StrContains(' 0 tests run'))
 292
 293         def test_run_raw_output(self):
 294                 self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
 295                 kunit.main(['run', '--raw_output'], self.linux_source_mock)
 296                 assert self.linux_source_mock.build_reconfig.call_count == 1
 297                 assert self.linux_source_mock.run_kernel.call_count == 1
 298                 for kall in self.print_mock.call_args_list:
 299                         assert kall != mock.call(StrContains('Testing complete.'))
 300                         assert kall != mock.call(StrContains(' 0 tests run'))
 301
 302         def test_exec_timeout(self):
 303                 timeout = 3453
 304                 kunit.main(['exec', '--timeout', str(timeout)], self.linux_source_mock)
 305                 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='', timeout=timeout)
 306                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 307
 308         def test_run_timeout(self):
 309                 timeout = 3453
 310                 kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock)
 311                 assert self.linux_source_mock.build_reconfig.call_count == 1
 312                 self.linux_source_mock.run_kernel.assert_called_once_with(
 313                         build_dir='', timeout=timeout)
 314                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 315
 316         def test_run_builddir(self):
 317                 build_dir = '.kunit'
 318                 kunit.main(['run', '--build_dir', build_dir], self.linux_source_mock)
 319                 assert self.linux_source_mock.build_reconfig.call_count == 1
 320                 self.linux_source_mock.run_kernel.assert_called_once_with(
 321                         build_dir=build_dir, timeout=300)
 322                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 323
 324         def test_config_builddir(self):
 325                 build_dir = '.kunit'
 326                 kunit.main(['config', '--build_dir', build_dir], self.linux_source_mock)
 327                 assert self.linux_source_mock.build_reconfig.call_count == 1
 328
 329         def test_build_builddir(self):
 330                 build_dir = '.kunit'
 331                 kunit.main(['build', '--build_dir', build_dir], self.linux_source_mock)
 332                 self.linux_source_mock.build_um_kernel.assert_called_once_with(False, 8, build_dir, None)
 333
 334         def test_exec_builddir(self):
 335                 build_dir = '.kunit'
 336                 kunit.main(['exec', '--build_dir', build_dir], self.linux_source_mock)
 337                 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir=build_dir, timeout=300)
 338                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 339
 340 if __name__ == '__main__':
 341         unittest.main()