Bug 1355180: introduce run-on-projects: built-projects, and use it; r=jmaher draft
authorDustin J. Mitchell <dustin@mozilla.com>
Tue, 18 Apr 2017 15:54:15 +0000
changeset 569771 237ea2898767da8e4561d1f6884e5add92a7bde2
parent 569762 f7685ecd789dae5984ed2ce513b747dc5accc780
child 569772 2604e5c0b7518f9cda6cede49ba006ca85255d33
push id56275
push userdmitchell@mozilla.com
push dateThu, 27 Apr 2017 22:26:22 +0000
reviewersjmaher
bugs1355180
milestone55.0a1
Bug 1355180: introduce run-on-projects: built-projects, and use it; r=jmaher This test-only "run-on-projects" option indicates that the test should run on exactly the projects where its parent build runs. This avoids the situation where a build is configured to *not* run on a project, but some wayward test didn't get the memo and "pulls in" the build anyway. This has been particularly problematic with win64-asan/opt. MozReview-Commit-ID: 8UP6FsXBohy
taskcluster/ci/test/tests.yml
taskcluster/taskgraph/loader/test.py
taskcluster/taskgraph/transforms/tests.py
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -8,21 +8,21 @@
 cppunit:
     description: "CPP Unit Tests"
     suite: cppunittest
     treeherder-symbol: tc(Cpp)
     e10s: false
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
-            windows10-64-asan/opt: []
             windows.*: ['mozilla-central', 'try']
-            linux64-qr/.*: ['graphics']
+            windows10-64-asan/opt: []  # as an exception to windows.*
+            linux64-qr/.*: ['graphics']  # not on mozilla-central, even though it's built there
             linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
-            default: ['all']
+            default: built-projects
     mozharness:
         by-test-platform:
             android.*:
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
                 extra-options:
@@ -50,19 +50,18 @@ crashtest:
     instance-size:
         by-test-platform:
             android.*: xlarge
             default: default
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows10.*: []
-            linux64-qr/.*: ['graphics', 'mozilla-central']
             linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
-            default: ['all']
+            default: built-projects
     chunks:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 10
             android.*: 4
             default: 1
     e10s:
         by-test-platform:
             windows.*: false # Bug 1304435
@@ -101,17 +100,17 @@ external-media-tests-base:
     treeherder-symbol: tc-VP(b-m)
     e10s: false
     tier: 2
     max-run-time: 5400
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows.*: ['mozilla-central', 'try']
-            default: ['all']
+            default: built-projects
     mozharness:
         script: firefox_media_tests_taskcluster.py
         config:
             by-test-platform:
                 windows.*:
                     - mediatests/taskcluster_windows_config.py
                 default:
                     - mediatests/taskcluster_posix_config.py
@@ -127,17 +126,17 @@ external-media-tests-twitch:
         by-test-platform:
             default: default
     tier: 2
     max-run-time: 5400
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows.*: ['mozilla-central', 'try']
-            default: ['all']
+            default: built-projects
     mozharness:
         script: firefox_media_tests_taskcluster.py
         config:
             by-test-platform:
                 windows.*:
                     - mediatests/taskcluster_windows_config.py
                 default:
                     - mediatests/taskcluster_posix_config.py
@@ -152,17 +151,17 @@ external-media-tests-youtube:
     treeherder-symbol: tc-VP(b-y)
     e10s: false
     tier: 2
     max-run-time: 5400
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows.*: ['mozilla-central', 'try']
-            default: ['all']
+            default: built-projects
     mozharness:
         script: firefox_media_tests_taskcluster.py
         config:
             by-test-platform:
                 windows.*:
                     - mediatests/taskcluster_windows_config.py
                 default:
                     - mediatests/taskcluster_posix_config.py
@@ -172,40 +171,32 @@ external-media-tests-youtube:
 
 firefox-ui-functional-local:
     description: "Firefox-ui-tests functional run"
     suite: "firefox-ui/functional local"
     treeherder-symbol: tc-Fxfn-l(en-US)
     max-run-time: 5400
     tier: 1
     docker-image: {"in-tree": "desktop1604-test"}
-    run-on-projects:
-        by-test-platform:
-            windows10-64-asan/opt: []
-            default: ['all']
     mozharness:
         script: firefox_ui_tests/functional.py
         config:
             - firefox_ui_tests/taskcluster.py
             - remove_executables.py
         extra-options:
             - "--tag"
             - "local"
 
 firefox-ui-functional-remote:
     description: "Firefox-ui-tests functional run"
     suite: "firefox-ui/functional remote"
     treeherder-symbol: tc-Fxfn-r(en-US)
     max-run-time: 5400
     tier: 2
     docker-image: {"in-tree": "desktop1604-test"}
-    run-on-projects:
-        by-test-platform:
-            windows10-64-asan/opt: []
-            default: ['all']
     mozharness:
         script: firefox_ui_tests/functional.py
         config:
             - firefox_ui_tests/taskcluster.py
             - remove_executables.py
         extra-options:
             - "--tag"
             - "remote"
@@ -216,17 +207,17 @@ gtest:
     treeherder-symbol: tc(GTest)
     e10s: false
     instance-size: xlarge
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows10.*: []
             linux64-qr/.*: ['graphics']
-            default: ['all']
+            default: built-projects
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
                 macosx.*:
@@ -241,19 +232,18 @@ gtest:
 jittest:
     description: "JIT Test run"
     suite: jittest/jittest-chunked
     treeherder-symbol: tc(Jit)
     e10s: false
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
-            windows10-64-asan/opt: []
             linux64-qr/.*: ['graphics']
-            default: ['all']
+            default: built-projects
     chunks:
         by-test-platform:
             windows.*: 1
             macosx.*: 1
             default: 6
     mozharness:
         chunked:
           by-test-platform:
@@ -304,20 +294,19 @@ jsreftest:
             macosx.*: true
             default: both
     max-run-time:
         by-test-platform:
             android.*: 7200
             default: 3600
     run-on-projects:
         by-test-platform:
-            windows10-64-asan/opt: []
+            windows10-64-asan/opt: built-projects  # exception to windows.*, below
             windows.*: ['mozilla-central', 'try']
-            linux64-qr/.*: ['graphics', 'mozilla-central']
-            default: ['all']
+            default: built-projects
     mozharness:
         by-test-platform:
             android.*:
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
                 extra-options:
@@ -366,17 +355,17 @@ marionette:
             default: 1
     e10s:
         by-test-platform:
             macosx.*: true
             default: both
     run-on-projects:
         by-test-platform:
             windows.*: ['mozilla-central', 'try']
-            default: ['all']
+            default: built-projects
     mozharness:
         by-test-platform:
             android.*:
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
                 extra-options:
@@ -422,17 +411,17 @@ mochitest:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 10800
             linux64-jsdcov/opt: 10800
             default: 5400
     allow-software-gl-layers: false
     run-on-projects:
         by-test-platform:
             windows10.*: []
-            default: ['all']
+            default: built-projects
     mozharness:
         by-test-platform:
             android.*:
                 mochitest-flavor: plain
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
@@ -466,17 +455,17 @@ mochitest-a11y:
     treeherder-symbol: tc-M(a11y)
     loopback-video: true
     e10s: false
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows.*: ['mozilla-central', 'try']
             linux64-qr/.*: ['graphics']
-            default: ['all']
+            default: built-projects
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: false
         mochitest-flavor: a11y
         config:
             by-test-platform:
                 windows.*:
@@ -515,17 +504,17 @@ mochitest-browser-chrome:
             linux64-jsdcov/opt: 7200
             linux64-ccov/opt: 7200
             linux64/debug: 5400
             linux32/debug: 5400
             default: 3600
     run-on-projects:
         by-test-platform:
             windows10.*: []
-            default: ['all']
+            default: built-projects
     mozharness:
         mochitest-flavor: browser
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 windows.*:
@@ -594,17 +583,17 @@ mochitest-chrome:
             android-4.3-arm7-api-15/debug: 4
             android.*: 2
             default: 3
     max-run-time: 3600
     e10s: false
     run-on-projects:
         by-test-platform:
             windows10.*: []
-            default: ['all']
+            default: built-projects
     mozharness:
         by-test-platform:
             android.*:
                 mochitest-flavor: chrome
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     by-test-platform:
@@ -641,17 +630,17 @@ mochitest-clipboard:
     instance-size: xlarge
     e10s:
       by-test-platform:
         macosx64/debug: true
         default: both
     run-on-projects:
         by-test-platform:
             windows10.*: []
-            default: ['all']
+            default: built-projects
     mozharness:
         by-test-platform:
             android.*:
                 mochitest-flavor: plain
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
@@ -689,17 +678,17 @@ mochitest-devtools-chrome:
         by-test-platform:
             windows.*: 8
             macosx.*: 8
             default: 10
     run-on-projects:
         by-test-platform:
             windows10.*: []
             windows7.*: ['mozilla-central', 'try']
-            default: ['all']
+            default: built-projects
     e10s:
         by-test-platform:
             linux64-jsdcov/opt: false
             # Bug 1304433: mochitest-devtools-chrome (e10s) not greened on windows
             windows.*: false
             macosx64/debug: true
             default: both
     mozharness:
@@ -736,17 +725,17 @@ mochitest-gpu:
     suite: mochitest/gpu
     treeherder-symbol: tc-M(gpu)
     loopback-video: true
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows10.*: []
             linux64-qr/.*: ['graphics']
-            default: ['all']
+            default: built-projects
     e10s:
         by-test-platform:
             windows.*: both
             android.*: false
             macosx64/opt: both
             default: true
     mozharness:
         by-test-platform:
@@ -782,19 +771,19 @@ mochitest-jetpack:
     suite: mochitest/jetpack-package
     treeherder-symbol: tc-M(JP)
     loopback-video: true
     e10s: false
     max-run-time: 5400
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
-            windows10-64-asan/opt: []
+            windows10-64-asan/opt: built-projects  # exception to windows.*, below
             windows.*: ['mozilla-central', 'try']
-            default: ['all']
+            default: built-projects
     mozharness:
         mochitest-flavor: jetpack-package
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: false
         config:
             by-test-platform:
                 windows.*:
@@ -828,17 +817,17 @@ mochitest-media:
             android.*: 3
             macosx64.*: 1
             default: 3
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows10.*: []
             linux64-qr/.*: ['graphics']
-            default: ['all']
+            default: built-projects
     mozharness:
         by-test-platform:
             android.*:
                 mochitest-flavor: plain
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
@@ -896,17 +885,17 @@ mochitest-webgl:
     description: "Mochitest webgl run"
     suite: mochitest/mochitest-gl
     treeherder-symbol: tc-M(gl)
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows10.*: []
             linux64-qr/.*: ['graphics', 'mozilla-central']
-            default: ['all']
+            default: built-projects
     chunks:
         by-test-platform:
             android.*: 10
             default: 3
     e10s:
       by-test-platform:
         macosx.*: true
         default: both
@@ -952,20 +941,17 @@ mochitest-webgl:
 
 mochitest-style:
     description: "Mochitest plain run for style system"
     suite: mochitest/plain-style
     treeherder-symbol: tc-M(s)
     loopback-video: true
     docker-image: {"in-tree": "desktop1604-test"}
     e10s: both
-    run-on-projects:
-        by-test-platform:
-            linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
-            default: ['all']
+    run-on-projects: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
     mozharness:
         mochitest-flavor: plain
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 default:
                     - unittests/linux_unittest.py
@@ -974,20 +960,17 @@ mochitest-style:
             - --mochitest-suite=plain-style
 
 mochitest-chrome-style:
     description: "Mochitest chrome run for style system"
     suite: mochitest/chrome-style
     treeherder-symbol: tc-M(cs)
     loopback-video: true
     docker-image: {"in-tree": "desktop1604-test"}
-    run-on-projects:
-        by-test-platform:
-            linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
-            default: ['all']
+    run-on-projects: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
     e10s: false
     mozharness:
         mochitest-flavor: chrome
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 default:
@@ -1000,17 +983,17 @@ reftest:
     description: "Reftest run"
     suite: reftest/reftest
     treeherder-symbol: tc-R(R)
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows10.*: []
             linux64-qr/.*: ['graphics', 'mozilla-central']
-            default: ['all']
+            default: built-projects
     instance-size:
         by-test-platform:
             android.*: xlarge
             default: default
     chunks:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 48
             android.*: 16
@@ -1058,17 +1041,17 @@ reftest-no-accel:
     description: "Reftest not accelerated run"
     suite: reftest/reftest-no-accel
     treeherder-symbol: tc-R(Ru)
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows10.*: []
             linux64-qr/.*: ['graphics', 'mozilla-central']
-            default: ['all']
+            default: built-projects
     chunks:
         by-test-platform:
             macosx.*: 1
             default: 8
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
@@ -1089,17 +1072,17 @@ reftest-stylo:
     suite: reftest/reftest-stylo
     treeherder-symbol: tc-R(Rs)
     docker-image: {"in-tree": "desktop1604-test"}
     chunks: 16
     run-on-projects:
         by-test-platform:
             linux64-stylo/opt: [ 'stylo', 'autoland', 'mozilla-central', 'try' ]
             linux64-stylo/debug: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
-            default: []
+            default: built-projects
     e10s:
         # run only e10s tests for this platform see bug 1343301
         by-test-platform:
             linux64-stylo/opt:
                 # no opt on inbound see bug 1339604
                 by-project:
                     stylo: true
                     autoland: true
@@ -1401,17 +1384,17 @@ web-platform-tests:
     max-run-time: 7200
     instance-size: xlarge
     docker-image: {"in-tree": "desktop1604-test"}
     checkout: true
     run-on-projects:
         by-test-platform:
             windows10.*: []
             windows7.*: ['mozilla-central', 'try']
-            default: ['all']
+            default: built-projects
     mozharness:
         script: web_platform_tests.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 windows.*:
                     - web_platform_tests/prod_config_windows_taskcluster.py
@@ -1434,17 +1417,17 @@ web-platform-tests-reftests:
             macosx.*: true
             default: both
     docker-image: {"in-tree": "desktop1604-test"}
     checkout: true
     run-on-projects:
         by-test-platform:
             windows10.*: []
             windows7.*: ['mozilla-central', 'try']
-            default: ['all']
+            default: built-projects
     mozharness:
         script: web_platform_tests.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 windows.*:
                     - web_platform_tests/prod_config_windows_taskcluster.py
                 default:
@@ -1480,17 +1463,17 @@ xpcshell:
         by-test-platform:
             linux64-jsdcov/opt: xpcshell-coverage
             default: xpcshell
     treeherder-symbol: tc-X(X)
     run-on-projects:
         by-test-platform:
             windows.*: []
             linux64-qr/.*: ['graphics']
-            default: ['all']
+            default: built-projects
     docker-image: {"in-tree": "desktop1604-test"}
     chunks:
         by-test-platform:
             linux64/debug: 10
             android-4.2-x86/opt: 6
             # windows.*: 1
             macosx.*: 1
             default: 8
--- a/taskcluster/taskgraph/loader/test.py
+++ b/taskcluster/taskgraph/loader/test.py
@@ -39,16 +39,17 @@ def loader(kind, path, config, params, l
 
     # generate all tests for all test platforms
     for test_platform_name, test_platform in test_platforms.iteritems():
         for test_name in test_platform['test-names']:
             test = copy.deepcopy(test_descriptions[test_name])
             test['build-platform'] = test_platform['build-platform']
             test['test-platform'] = test_platform_name
             test['build-label'] = test_platform['build-label']
+            test['build-attributes'] = test_platform['build-attributes']
             test['test-name'] = test_name
             if test_platform['nightly']:
                 test.setdefault('attributes', {})['nightly'] = True
 
             logger.debug("Generating tasks for test {} on platform {}".format(
                 test_name, test['test-platform']))
             yield test
 
@@ -83,16 +84,17 @@ def get_test_platforms(test_platforms_cf
             logger.warning(
                 "No build task with platform {}; ignoring test platform {}".format(
                     build_platform, test_platform))
             continue
         test_platforms[test_platform] = {
             'nightly': builds_by_platform[build_platform].attributes.get('nightly', False),
             'build-platform': build_platform,
             'build-label': builds_by_platform[build_platform].label,
+            'build-attributes': builds_by_platform[build_platform].attributes,
         }
         test_platforms[test_platform].update(cfg)
     return test_platforms
 
 
 def expand_tests(test_sets_cfg, test_platforms):
     """Expand the test sets in `test_platforms` out to sets of test names.
     Returns a dictionary like `get_test_platforms`, with an additional
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -94,19 +94,23 @@ test_description_schema = Schema({
 
     # attributes to appear in the resulting task (later transforms will add the
     # common attributes)
     Optional('attributes'): {basestring: object},
 
     # The `run_on_projects` attribute, defaulting to "all".  This dictates the
     # projects on which this task should be included in the target task set.
     # See the attributes documentation for details.
-    Optional('run-on-projects', default=['all']): optionally_keyed_by(
+    #
+    # Note that the special case 'built-projects', the default, uses the parent
+    # build task's run-on-projects, meaning that tests run only on platforms
+    # that are built.
+    Optional('run-on-projects', default='built-projects'): optionally_keyed_by(
         'test-platform',
-        [basestring]),
+        Any([basestring], 'built-projects')),
 
     # the sheriffing tier for this task (default: set based on test platform)
     Optional('tier'): optionally_keyed_by(
         'test-platform',
         Any(int, 'default')),
 
     # number of chunks to create for this task.  This can be keyed by test
     # platform by passing a dictionary in the `by-test-platform` key.  If the
@@ -263,16 +267,19 @@ test_description_schema = Schema({
     # -- values supplied by the task-generation infrastructure
 
     # the platform of the build this task is testing
     'build-platform': basestring,
 
     # the label of the build task generating the materials to test
     'build-label': basestring,
 
+    # the build's attributes
+    'build-attributes': {basestring: object},
+
     # the platform on which the tests will run
     'test-platform': basestring,
 
     # the name of the test (the key in tests.yml)
     'test-name': basestring,
 
     # the product name, defaults to firefox
     Optional('product'): basestring,
@@ -321,17 +328,17 @@ def set_defaults(config, tests):
         # software-gl-layers is only meaningful on linux, where it defaults to True
         if test['test-platform'].startswith('linux'):
             test.setdefault('allow-software-gl-layers', True)
         else:
             test['allow-software-gl-layers'] = False
 
         test.setdefault('os-groups', [])
         test.setdefault('chunks', 1)
-        test.setdefault('run-on-projects', ['all'])
+        test.setdefault('run-on-projects', 'built-projects')
         test.setdefault('instance-size', 'default')
         test.setdefault('max-run-time', 3600)
         test.setdefault('reboot', True)
         test['mozharness'].setdefault('extra-options', [])
         yield test
 
 
 @transforms.add
@@ -506,16 +513,25 @@ def enable_code_coverage(config, tests):
             test['e10s'] = False
             test['run-on-projects'] = []
         elif test['build-platform'] == 'linux64-jsdcov/opt':
             test['run-on-projects'] = []
         yield test
 
 
 @transforms.add
+def handle_run_on_projects(config, tests):
+    """Handle translating `built-projects` appropriately"""
+    for test in tests:
+        if test['run-on-projects'] == 'built-projects':
+            test['run-on-projects'] = test['build-attributes'].get('run_on_projects', ['all'])
+        yield test
+
+
+@transforms.add
 def split_e10s(config, tests):
     for test in tests:
         e10s = test['e10s']
 
         test.setdefault('attributes', {})
         test['e10s'] = False
         test['attributes']['e10s'] = False
 
@@ -691,17 +707,17 @@ def make_job_description(config, tests):
         jobdesc['name'] = name
         jobdesc['label'] = label
         jobdesc['description'] = test['description']
         jobdesc['when'] = test.get('when', {})
         jobdesc['attributes'] = attributes
         jobdesc['dependencies'] = {'build': build_label}
         jobdesc['expires-after'] = test['expires-after']
         jobdesc['routes'] = []
-        jobdesc['run-on-projects'] = test.get('run-on-projects', ['all'])
+        jobdesc['run-on-projects'] = test['run-on-projects']
         jobdesc['scopes'] = []
         jobdesc['tags'] = test.get('tags', {})
         jobdesc['optimizations'] = [['seta']]  # always run SETA for tests
         jobdesc['extra'] = {
             'chunks': {
                 'current': test['this-chunk'],
                 'total': test['chunks'],
             },