mtest: add option to slice tests

Executing tests can take a very long time. As an example, the Git test
suite on Windows takes around 4 hours to execute. The Git project has
been working around the issue by splitting up CI jobs into multiple
slices: one job creates the build artifacts, and then we spawn N test
jobs with those artifacts, where each test job executes 1/Nth of the
tests.

This can be scripted rather easily by using `meson test --list`,
selecting every Nth line, but there may be other projects that have a
similar need. Wire up a new option "--slice i/n" to `meson test` that
does implements this logic.

Signed-off-by: Patrick Steinhardt <[email protected]>

Author: pks-t

Diff for: docs/markdown/Unit-tests.md

@@ -206,6 +206,11 @@ name(s), the test name(s) must be contained in the suite(s). This
 however is redundant-- it would be more useful to specify either
 specific test names or suite(s).
 
+Since version *1.8.0*, you can pass `--slice i/n` to split up the set of tests
+into `n` slices and execute the `ith` such slice. This allows you to distribute
+a set of long-running tests across multiple machines to decrease the overall
+runtime of tests.
+
 ### Other test options
 
 Sometimes you need to run the tests multiple times, which is done like this:

Diff for: docs/markdown/snippets/test-slicing.md

@@ -0,0 +1,6 @@
+## New option to execute a slice of tests
+
+When tests take a long time to run a common strategy is to slice up the tests
+into multiple sets, where each set is executed on a separate machine. You can
+now use the `--slice i/n` argument for `meson test` to create `n` slices and
+execute the `ith` slice.

Diff for: man/meson.1

@@ -328,6 +328,9 @@ a multiplier to use for test timeout values (usually something like 100 for Valg
 .TP
 \fB\-\-setup\fR
 use the specified test setup
+.Tp
+\fB\-\-slice SLICE/NUM_SLICES\fR
+Split tests into NUM_SLICES slices and execute slice number SLICE.  (Since 1.8.0)
 
 .SH The wrap command
 
@@ -410,7 +413,7 @@ Manage the packagefiles overlay
 
 .B meson rewrite
 modifies the project definition.
- 
+
 .B meson rewrite [
 .I options
 .B ] [

Diff for: mesonbuild/mtest.py

@@ -99,6 +99,29 @@ def uniwidth(s: str) -> int:
         result += UNIWIDTH_MAPPING[w]
     return result
 
+def test_slice(arg: str) -> T.Tuple[int, int]:
+    values = arg.split('/')
+    if len(values) != 2:
+        raise argparse.ArgumentTypeError("value does not conform to format 'SLICE/NUM_SLICES'")
+
+    try:
+        nrslices = int(values[1])
+    except ValueError:
+        raise argparse.ArgumentTypeError('NUM_SLICES is not an integer')
+    if nrslices <= 0:
+        raise argparse.ArgumentTypeError('NUM_SLICES is not a positive integer')
+
+    try:
+        subslice = int(values[0])
+    except ValueError:
+        raise argparse.ArgumentTypeError('SLICE is not an integer')
+    if subslice <= 0:
+        raise argparse.ArgumentTypeError('SLICE is not a positive integer')
+    if subslice > nrslices:
+        raise argparse.ArgumentTypeError('SLICE exceeds NUM_SLICES')
+
+    return subslice, nrslices
+
 # Note: when adding arguments, please also add them to the completion
 # scripts in $MESONSRC/data/shell-completions/
 def add_arguments(parser: argparse.ArgumentParser) -> None:
@@ -149,12 +172,13 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
                         help='Arguments to pass to the specified test(s) or all tests')
     parser.add_argument('--max-lines', default=100, dest='max_lines', type=int,
                         help='Maximum number of lines to show from a long test log. Since 1.5.0.')
+    parser.add_argument('--slice', default=None, type=test_slice, metavar='SLICE/NUM_SLICES',
+                        help='Split tests into NUM_SLICES slices and execute slice SLICE. Since 1.8.0.')
     parser.add_argument('args', nargs='*',
                         help='Optional list of test names to run. "testname" to run all tests with that name, '
                         '"subprojname:testname" to specifically run "testname" from "subprojname", '
                         '"subprojname:" to run all tests defined by "subprojname".')
 
-
 def print_safe(s: str) -> None:
     end = '' if s[-1] == '\n' else '\n'
     try:
@@ -1977,6 +2001,11 @@ def get_tests(self, errorfile: T.Optional[T.IO] = None) -> T.List[TestSerialisat
         tests = [t for t in self.tests if self.test_suitable(t)]
         if self.options.args:
             tests = list(self.tests_from_args(tests))
+        if self.options.slice:
+            our_slice, nslices = self.options.slice
+            if nslices > len(tests):
+                raise MesonException(f'number of slices ({nslices}) exceeds number of tests ({len(tests)})')
+            tests = tests[our_slice - 1::nslices]
 
         if not tests:
             print('No suitable tests defined.', file=errorfile)

Diff for: test cases/unit/124 test slice/meson.build

@@ -0,0 +1,12 @@
+project('test_slice')
+
+python = import('python').find_installation('python3')
+
+foreach i : range(10)
+  test('test-' + (i + 1).to_string(),
+    python,
+    args: [
+      meson.current_source_dir() / 'test.py'
+    ],
+  )
+endforeach

Diff for: test cases/unit/124 test slice/test.py

@@ -5130,6 +5130,35 @@ def test_objc_objcpp_stds(self) -> None:
     def test_c_cpp_objc_objcpp_stds(self) -> None:
         self.__test_multi_stds(test_objc=True)
 
+    def test_slice(self):
+        testdir = os.path.join(self.unit_test_dir, '124 test slice')
+        self.init(testdir)
+        self.build()
+
+        for arg, expectation in {'1/1': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+                                 '1/2': [1, 3, 5, 7, 9],
+                                 '2/2': [2, 4, 6, 8, 10],
+                                 '1/10': [1],
+                                 '2/10': [2],
+                                 '10/10': [10],
+                                 }.items():
+            output = self._run(self.mtest_command + ['--slice=' + arg])
+            tests = sorted([ int(x[5:]) for x in re.findall(r'test-[0-9]*', output) ])
+            self.assertEqual(tests, expectation)
+
+        for arg, expectation in {'': 'error: argument --slice: value does not conform to format \'SLICE/NUM_SLICES\'',
+                                 '0': 'error: argument --slice: value does not conform to format \'SLICE/NUM_SLICES\'',
+                                 '0/1': 'error: argument --slice: SLICE is not a positive integer',
+                                 'a/1': 'error: argument --slice: SLICE is not an integer',
+                                 '1/0': 'error: argument --slice: NUM_SLICES is not a positive integer',
+                                 '1/a': 'error: argument --slice: NUM_SLICES is not an integer',
+                                 '2/1': 'error: argument --slice: SLICE exceeds NUM_SLICES',
+                                 '1/11': 'ERROR: number of slices (11) exceeds number of tests (10)',
+                                 }.items():
+            with self.assertRaises(subprocess.CalledProcessError) as cm:
+                self._run(self.mtest_command + ['--slice=' + arg])
+            self.assertIn(expectation, cm.exception.output)
+
     def test_rsp_support(self):
         env = get_fake_env()
         cc = detect_c_compiler(env, MachineChoice.HOST)