testing: refactor code for determining docker requirements (
bug 1363509); r?glob
The code for parsing test files to determine Docker requirements isn't
very reusable or extensible. This commit changes that.
Functionality should be unchaged. Only some APIs for functions
with single call sites changed. The code will be further refactored
in a future commit, so don't get too attached to it.
MozReview-Commit-ID: 70aKVVWXMny
--- a/testing/vcttesting/testing.py
+++ b/testing/vcttesting/testing.py
@@ -194,102 +194,115 @@ def get_test_files(extensions, venv):
return {
'extension': sorted(extension_tests),
'hook': sorted(hook_tests),
'unit': sorted(unit_tests),
'all': set(extension_tests) | set(hook_tests) | set(unit_tests),
}
-def docker_requirements(tests):
- """Whether any of the specified test files require Docker."""
+def docker_requirements_for_test(path):
+ """Given a path to a test, determine its Docker requirements.
+
+ Returns a set of strings describing which Docker features are
+ needed. String values are:
+
+ bmo
+ Requires images to run Bugzilla
+ hgmo
+ Requires images to run hg.mozilla.org
+ mozreview
+ Requires images to run mozreview
+ """
docker_keywords = (
b'MozReviewTest',
b'MozReviewWebDriverTest',
)
- hgmo = False
- mozreview = False
- bmo = False
- d = False
+ res = set()
+
+ with open(path, 'rb') as fh:
+ content = fh.read()
- for t in tests:
- with open(t, 'rb') as fh:
- content = fh.read()
+ if b'#require hgmodocker' in content:
+ res.add('hgmo')
- if b'#require hgmodocker' in content:
- d = True
- hgmo = True
+ if b'#require mozreviewdocker' in content:
+ res.add('bmo')
+ res.add('mozreview')
+
+ if b'#require bmodocker' in content:
+ res.add('bmo')
- if b'#require mozreviewdocker' in content:
- d = True
- mozreview = True
- bmo = True
+ for keyword in docker_keywords:
+ if keyword in content:
+ # This could probably be defined better.
+ res.add('hgmo')
+ res.add('mozreview')
- if b'#require bmodocker' in content:
- d = True
- bmo = True
+ return res
+
- for keyword in docker_keywords:
- if keyword in content:
- # This could probably be defined better.
- d = True
- hgmo = True
- mozreview = True
+def docker_requirements(tests):
+ """Determine what Docker features are needed by the given tests."""
- return d, hgmo, mozreview, bmo
+ res = set()
+ for test in tests:
+ res |= docker_requirements_for_test(test)
+
+ return res
def get_docker_state(docker, tests, verbose=False, use_last=False):
"""Obtain usable Docker images, possibly by building them.
Given a Docker client and list of .t test paths, determine what
Docker images are needed to run the tests and then return a dictionary
of environment variables that define the Docker image IDs.
If ``use_last`` is set, existing Docker images will be used. Otherwise,
Docker images are rebuilt to ensure they are up-to-date.
"""
- build_docker, hgmo, mozreview, bmo = docker_requirements(tests)
+ requirements = docker_requirements(tests)
- if not build_docker:
+ if not requirements:
return {}
env = {}
print('generating Docker images needed for tests')
t_start = time.time()
mr_images, hgmo_images, bmo_images = docker.build_all_images(
verbose=verbose,
use_last=use_last,
- hgmo=hgmo,
- mozreview=mozreview,
- bmo=bmo)
+ hgmo='hgmo' in requirements,
+ mozreview='mozreview' in requirements,
+ bmo='bmo' in requirements)
t_end = time.time()
print('got Docker images in %.2fs' % (t_end - t_start))
- if mozreview:
+ if 'mozreview' in requirements:
env['DOCKER_PULSE_IMAGE'] = mr_images['pulse']
env['DOCKER_HGRB_IMAGE'] = mr_images['hgrb']
env['DOCKER_AUTOLANDDB_IMAGE'] = mr_images['autolanddb']
env['DOCKER_AUTOLAND_IMAGE'] = mr_images['autoland']
env['DOCKER_RBWEB_IMAGE'] = mr_images['rbweb']
env['DOCKER_TREESTATUS_IMAGE'] = mr_images['treestatus']
env['DOCKER_LDAP_IMAGE'] = mr_images['ldap']
- if not hgmo:
+ if 'hgmo' not in requirements:
env['DOCKER_HGWEB_IMAGE'] = mr_images['hgweb']
- if not bmo:
+ if 'bmo' not in requirements:
env['DOCKER_BMOWEB_IMAGE'] = mr_images['bmoweb']
- if hgmo:
+ if 'hgmo' in requirements:
env['DOCKER_HGMASTER_IMAGE'] = hgmo_images['hgmaster']
env['DOCKER_HGWEB_IMAGE'] = hgmo_images['hgweb']
env['DOCKER_LDAP_IMAGE'] = hgmo_images['ldap']
env['DOCKER_PULSE_IMAGE'] = hgmo_images['pulse']
- if bmo:
+ if 'bmo' in requirements:
env['DOCKER_BMOWEB_IMAGE'] = bmo_images['bmoweb']
return env
def produce_coverage_reports(coverdir):
cov = Coverage(data_file='coverage')
cov.combine(data_paths=[os.path.join(coverdir, 'data')])