Bug 1003417 - Add ability to run subsuites to |mach python-test|, r?ted
This adds the ability to use manifestparser subsuites to |mach python-test|.
Subsuites are based on the premise of a "default" set that gets run when no
subsuites are explicitly specified. When a test is labelled with a subsuite,
that test is removed from the default set and will only run if that subsuite
is explicitly specified. This will allow us to chunk python unittests out of
'make check' piecemeal. The default set will run in 'make check', and
individual tasks (e.g mozbase), will specify a subsuite explicitly.
The |mach python-test| implementation is slightly different. By default,
subsuites are not considered if developers do not pass in --subsuite. This
means running |mach python-test| without arguments will still run the full set
of tests, and similarly, passing in test paths will *just work*.
If for some reason a developer needs to actually run the default set, a special
"default" subsuite has been create, so they can use
|mach python-test --subsuite default|. This default subsuite is also what 'make
check' will explicitly invoke.
MozReview-Commit-ID: FaHb4nvuoK9
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -12,16 +12,17 @@ import os
from concurrent.futures import (
ThreadPoolExecutor,
as_completed,
thread,
)
import mozinfo
from manifestparser import TestManifest
+from manifestparser import filters as mpf
from mozbuild.base import (
MachCommandBase,
)
from mach.decorators import (
CommandArgument,
CommandProvider,
@@ -60,16 +61,20 @@ class MachCommands(MachCommandBase):
default=False,
action='store_true',
help=('Collect all tests under given path instead of default '
'test resolution. Supports pytest-style tests.'))
@CommandArgument('-j', '--jobs',
default=1,
type=int,
help='Number of concurrent jobs to run. Default is 1.')
+ @CommandArgument('--subsuite',
+ default=None,
+ help=('Python subsuite to run. If not specified, all subsuites are run. '
+ 'Use the string `default` to only run tests without a subsuite.'))
@CommandArgument('tests', nargs='*',
metavar='TEST',
help=('Tests to run. Each test can be a single file or a directory. '
'Default test resolution relies on PYTHON_UNITTEST_MANIFESTS.'))
def python_test(self,
tests=[],
test_objects=None,
subsuite=None,
@@ -128,17 +133,24 @@ class MachCommands(MachCommandBase):
message = 'TEST-UNEXPECTED-FAIL | No tests collected'
if not path_only:
message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)'
self.log(logging.WARN, 'python-test', {}, message)
return 1
mp = TestManifest()
mp.tests.extend(test_objects)
- tests = mp.active_tests(disabled=False, **mozinfo.info)
+
+ filters = []
+ if subsuite == 'default':
+ filters.append(mpf.subsuite(None))
+ elif subsuite:
+ filters.append(mpf.subsuite(subsuite))
+
+ tests = mp.active_tests(filters=filters, disabled=False, **mozinfo.info)
self.jobs = jobs
self.terminate = False
self.verbose = verbose
return_code = 0
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
futures = [executor.submit(self._run_python_test, test['path'])