testing: refactor code for determining docker requirements (bug 1363509); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Tue, 09 May 2017 13:56:46 -0700
changeset 11323 d10ec339d25a0a97e4eea80a1bcca25c13d3bbc6
parent 11322 0d1b52cd638e95f20c82a23ddf1870447472946c
child 11324 ae2612777cd96ac7e7fd6ecc61207fa1d825dc68
push id1719
push userbmo:gps@mozilla.com
push dateSat, 24 Jun 2017 00:30:23 +0000
reviewersglob
bugs1363509
testing: refactor code for determining docker requirements (bug 1363509); r?glob The code for parsing test files to determine Docker requirements isn't very reusable or extensible. This commit changes that. Functionality should be unchaged. Only some APIs for functions with single call sites changed. The code will be further refactored in a future commit, so don't get too attached to it. MozReview-Commit-ID: 70aKVVWXMny
testing/vcttesting/testing.py
--- a/testing/vcttesting/testing.py
+++ b/testing/vcttesting/testing.py
@@ -194,102 +194,115 @@ def get_test_files(extensions, venv):
     return {
         'extension': sorted(extension_tests),
         'hook': sorted(hook_tests),
         'unit': sorted(unit_tests),
         'all': set(extension_tests) | set(hook_tests) | set(unit_tests),
     }
 
 
-def docker_requirements(tests):
-    """Whether any of the specified test files require Docker."""
+def docker_requirements_for_test(path):
+    """Given a path to a test, determine its Docker requirements.
+
+    Returns a set of strings describing which Docker features are
+    needed. String values are:
+
+    bmo
+       Requires images to run Bugzilla
+    hgmo
+       Requires images to run hg.mozilla.org
+    mozreview
+       Requires images to run mozreview
+    """
     docker_keywords = (
         b'MozReviewTest',
         b'MozReviewWebDriverTest',
     )
 
-    hgmo = False
-    mozreview = False
-    bmo = False
-    d = False
+    res = set()
+
+    with open(path, 'rb') as fh:
+        content = fh.read()
 
-    for t in tests:
-        with open(t, 'rb') as fh:
-            content = fh.read()
+        if b'#require hgmodocker' in content:
+            res.add('hgmo')
 
-            if b'#require hgmodocker' in content:
-                d = True
-                hgmo = True
+        if b'#require mozreviewdocker' in content:
+            res.add('bmo')
+            res.add('mozreview')
+
+        if b'#require bmodocker' in content:
+            res.add('bmo')
 
-            if b'#require mozreviewdocker' in content:
-                d = True
-                mozreview = True
-                bmo = True
+        for keyword in docker_keywords:
+            if keyword in content:
+                # This could probably be defined better.
+                res.add('hgmo')
+                res.add('mozreview')
 
-            if b'#require bmodocker' in content:
-                d = True
-                bmo = True
+    return res
+
 
-            for keyword in docker_keywords:
-                if keyword in content:
-                    # This could probably be defined better.
-                    d = True
-                    hgmo = True
-                    mozreview = True
+def docker_requirements(tests):
+    """Determine what Docker features are needed by the given tests."""
 
-    return d, hgmo, mozreview, bmo
+    res = set()
+    for test in tests:
+        res |= docker_requirements_for_test(test)
+
+    return res
 
 
 def get_docker_state(docker, tests, verbose=False, use_last=False):
     """Obtain usable Docker images, possibly by building them.
 
     Given a Docker client and list of .t test paths, determine what
     Docker images are needed to run the tests and then return a dictionary
     of environment variables that define the Docker image IDs.
 
     If ``use_last`` is set, existing Docker images will be used. Otherwise,
     Docker images are rebuilt to ensure they are up-to-date.
     """
-    build_docker, hgmo, mozreview, bmo = docker_requirements(tests)
+    requirements = docker_requirements(tests)
 
-    if not build_docker:
+    if not requirements:
         return {}
 
     env = {}
     print('generating Docker images needed for tests')
     t_start = time.time()
     mr_images, hgmo_images, bmo_images = docker.build_all_images(
             verbose=verbose,
             use_last=use_last,
-            hgmo=hgmo,
-            mozreview=mozreview,
-            bmo=bmo)
+            hgmo='hgmo' in requirements,
+            mozreview='mozreview' in requirements,
+            bmo='bmo' in requirements)
 
     t_end = time.time()
     print('got Docker images in %.2fs' % (t_end - t_start))
-    if mozreview:
+    if 'mozreview' in requirements:
         env['DOCKER_PULSE_IMAGE'] = mr_images['pulse']
         env['DOCKER_HGRB_IMAGE'] = mr_images['hgrb']
         env['DOCKER_AUTOLANDDB_IMAGE'] = mr_images['autolanddb']
         env['DOCKER_AUTOLAND_IMAGE'] = mr_images['autoland']
         env['DOCKER_RBWEB_IMAGE'] = mr_images['rbweb']
         env['DOCKER_TREESTATUS_IMAGE'] = mr_images['treestatus']
         env['DOCKER_LDAP_IMAGE'] = mr_images['ldap']
-        if not hgmo:
+        if 'hgmo' not in requirements:
             env['DOCKER_HGWEB_IMAGE'] = mr_images['hgweb']
-        if not bmo:
+        if 'bmo' not in requirements:
             env['DOCKER_BMOWEB_IMAGE'] = mr_images['bmoweb']
 
-    if hgmo:
+    if 'hgmo' in requirements:
         env['DOCKER_HGMASTER_IMAGE'] = hgmo_images['hgmaster']
         env['DOCKER_HGWEB_IMAGE'] = hgmo_images['hgweb']
         env['DOCKER_LDAP_IMAGE'] = hgmo_images['ldap']
         env['DOCKER_PULSE_IMAGE'] = hgmo_images['pulse']
 
-    if bmo:
+    if 'bmo' in requirements:
         env['DOCKER_BMOWEB_IMAGE'] = bmo_images['bmoweb']
 
     return env
 
 
 def produce_coverage_reports(coverdir):
     cov = Coverage(data_file='coverage')
     cov.combine(data_paths=[os.path.join(coverdir, 'data')])