Bug 1355180: introduce run-on-projects: built-projects, and use it; r=jmaher
This test-only "run-on-projects" option indicates that the test should run on
exactly the projects where its parent build runs. This avoids the situation
where a build is configured to *not* run on a project, but some wayward test
didn't get the memo and "pulls in" the build anyway. This has been
particularly problematic with win64-asan/opt.
MozReview-Commit-ID: 8UP6FsXBohy
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -8,21 +8,21 @@
cppunit:
description: "CPP Unit Tests"
suite: cppunittest
treeherder-symbol: tc(Cpp)
e10s: false
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
- windows10-64-asan/opt: []
windows.*: ['mozilla-central', 'try']
- linux64-qr/.*: ['graphics']
+ windows10-64-asan/opt: [] # as an exception to windows.*
+ linux64-qr/.*: ['graphics'] # not on mozilla-central, even though it's built there
linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
- default: ['all']
+ default: built-projects
mozharness:
by-test-platform:
android.*:
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
- android/androidarm_4_3.py
extra-options:
@@ -50,19 +50,18 @@ crashtest:
instance-size:
by-test-platform:
android.*: xlarge
default: default
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows10.*: []
- linux64-qr/.*: ['graphics', 'mozilla-central']
linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
- default: ['all']
+ default: built-projects
chunks:
by-test-platform:
android-4.3-arm7-api-15/debug: 10
android.*: 4
default: 1
e10s:
by-test-platform:
windows.*: false # Bug 1304435
@@ -101,17 +100,17 @@ external-media-tests-base:
treeherder-symbol: tc-VP(b-m)
e10s: false
tier: 2
max-run-time: 5400
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows.*: ['mozilla-central', 'try']
- default: ['all']
+ default: built-projects
mozharness:
script: firefox_media_tests_taskcluster.py
config:
by-test-platform:
windows.*:
- mediatests/taskcluster_windows_config.py
default:
- mediatests/taskcluster_posix_config.py
@@ -127,17 +126,17 @@ external-media-tests-twitch:
by-test-platform:
default: default
tier: 2
max-run-time: 5400
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows.*: ['mozilla-central', 'try']
- default: ['all']
+ default: built-projects
mozharness:
script: firefox_media_tests_taskcluster.py
config:
by-test-platform:
windows.*:
- mediatests/taskcluster_windows_config.py
default:
- mediatests/taskcluster_posix_config.py
@@ -152,17 +151,17 @@ external-media-tests-youtube:
treeherder-symbol: tc-VP(b-y)
e10s: false
tier: 2
max-run-time: 5400
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows.*: ['mozilla-central', 'try']
- default: ['all']
+ default: built-projects
mozharness:
script: firefox_media_tests_taskcluster.py
config:
by-test-platform:
windows.*:
- mediatests/taskcluster_windows_config.py
default:
- mediatests/taskcluster_posix_config.py
@@ -172,40 +171,32 @@ external-media-tests-youtube:
firefox-ui-functional-local:
description: "Firefox-ui-tests functional run"
suite: "firefox-ui/functional local"
treeherder-symbol: tc-Fxfn-l(en-US)
max-run-time: 5400
tier: 1
docker-image: {"in-tree": "desktop1604-test"}
- run-on-projects:
- by-test-platform:
- windows10-64-asan/opt: []
- default: ['all']
mozharness:
script: firefox_ui_tests/functional.py
config:
- firefox_ui_tests/taskcluster.py
- remove_executables.py
extra-options:
- "--tag"
- "local"
firefox-ui-functional-remote:
description: "Firefox-ui-tests functional run"
suite: "firefox-ui/functional remote"
treeherder-symbol: tc-Fxfn-r(en-US)
max-run-time: 5400
tier: 2
docker-image: {"in-tree": "desktop1604-test"}
- run-on-projects:
- by-test-platform:
- windows10-64-asan/opt: []
- default: ['all']
mozharness:
script: firefox_ui_tests/functional.py
config:
- firefox_ui_tests/taskcluster.py
- remove_executables.py
extra-options:
- "--tag"
- "remote"
@@ -216,17 +207,17 @@ gtest:
treeherder-symbol: tc(GTest)
e10s: false
instance-size: xlarge
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows10.*: []
linux64-qr/.*: ['graphics']
- default: ['all']
+ default: built-projects
mozharness:
script: desktop_unittest.py
no-read-buildbot-config: true
config:
by-test-platform:
windows.*:
- unittests/win_taskcluster_unittest.py
macosx.*:
@@ -241,19 +232,18 @@ gtest:
jittest:
description: "JIT Test run"
suite: jittest/jittest-chunked
treeherder-symbol: tc(Jit)
e10s: false
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
- windows10-64-asan/opt: []
linux64-qr/.*: ['graphics']
- default: ['all']
+ default: built-projects
chunks:
by-test-platform:
windows.*: 1
macosx.*: 1
default: 6
mozharness:
chunked:
by-test-platform:
@@ -304,20 +294,19 @@ jsreftest:
macosx.*: true
default: both
max-run-time:
by-test-platform:
android.*: 7200
default: 3600
run-on-projects:
by-test-platform:
- windows10-64-asan/opt: []
+ windows10-64-asan/opt: built-projects # exception to windows.*, below
windows.*: ['mozilla-central', 'try']
- linux64-qr/.*: ['graphics', 'mozilla-central']
- default: ['all']
+ default: built-projects
mozharness:
by-test-platform:
android.*:
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
- android/androidarm_4_3.py
extra-options:
@@ -366,17 +355,17 @@ marionette:
default: 1
e10s:
by-test-platform:
macosx.*: true
default: both
run-on-projects:
by-test-platform:
windows.*: ['mozilla-central', 'try']
- default: ['all']
+ default: built-projects
mozharness:
by-test-platform:
android.*:
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
- android/androidarm_4_3.py
extra-options:
@@ -422,17 +411,17 @@ mochitest:
by-test-platform:
android-4.3-arm7-api-15/debug: 10800
linux64-jsdcov/opt: 10800
default: 5400
allow-software-gl-layers: false
run-on-projects:
by-test-platform:
windows10.*: []
- default: ['all']
+ default: built-projects
mozharness:
by-test-platform:
android.*:
mochitest-flavor: plain
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
- android/androidarm_4_3.py
@@ -466,17 +455,17 @@ mochitest-a11y:
treeherder-symbol: tc-M(a11y)
loopback-video: true
e10s: false
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows.*: ['mozilla-central', 'try']
linux64-qr/.*: ['graphics']
- default: ['all']
+ default: built-projects
mozharness:
script: desktop_unittest.py
no-read-buildbot-config: true
chunked: false
mochitest-flavor: a11y
config:
by-test-platform:
windows.*:
@@ -515,17 +504,17 @@ mochitest-browser-chrome:
linux64-jsdcov/opt: 7200
linux64-ccov/opt: 7200
linux64/debug: 5400
linux32/debug: 5400
default: 3600
run-on-projects:
by-test-platform:
windows10.*: []
- default: ['all']
+ default: built-projects
mozharness:
mochitest-flavor: browser
script: desktop_unittest.py
no-read-buildbot-config: true
chunked: true
config:
by-test-platform:
windows.*:
@@ -594,17 +583,17 @@ mochitest-chrome:
android-4.3-arm7-api-15/debug: 4
android.*: 2
default: 3
max-run-time: 3600
e10s: false
run-on-projects:
by-test-platform:
windows10.*: []
- default: ['all']
+ default: built-projects
mozharness:
by-test-platform:
android.*:
mochitest-flavor: chrome
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
by-test-platform:
@@ -641,17 +630,17 @@ mochitest-clipboard:
instance-size: xlarge
e10s:
by-test-platform:
macosx64/debug: true
default: both
run-on-projects:
by-test-platform:
windows10.*: []
- default: ['all']
+ default: built-projects
mozharness:
by-test-platform:
android.*:
mochitest-flavor: plain
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
- android/androidarm_4_3.py
@@ -689,17 +678,17 @@ mochitest-devtools-chrome:
by-test-platform:
windows.*: 8
macosx.*: 8
default: 10
run-on-projects:
by-test-platform:
windows10.*: []
windows7.*: ['mozilla-central', 'try']
- default: ['all']
+ default: built-projects
e10s:
by-test-platform:
linux64-jsdcov/opt: false
# Bug 1304433: mochitest-devtools-chrome (e10s) not greened on windows
windows.*: false
macosx64/debug: true
default: both
mozharness:
@@ -736,17 +725,17 @@ mochitest-gpu:
suite: mochitest/gpu
treeherder-symbol: tc-M(gpu)
loopback-video: true
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows10.*: []
linux64-qr/.*: ['graphics']
- default: ['all']
+ default: built-projects
e10s:
by-test-platform:
windows.*: both
android.*: false
macosx64/opt: both
default: true
mozharness:
by-test-platform:
@@ -782,19 +771,19 @@ mochitest-jetpack:
suite: mochitest/jetpack-package
treeherder-symbol: tc-M(JP)
loopback-video: true
e10s: false
max-run-time: 5400
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
- windows10-64-asan/opt: []
+ windows10-64-asan/opt: built-projects # exception to windows.*, below
windows.*: ['mozilla-central', 'try']
- default: ['all']
+ default: built-projects
mozharness:
mochitest-flavor: jetpack-package
script: desktop_unittest.py
no-read-buildbot-config: true
chunked: false
config:
by-test-platform:
windows.*:
@@ -828,17 +817,17 @@ mochitest-media:
android.*: 3
macosx64.*: 1
default: 3
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows10.*: []
linux64-qr/.*: ['graphics']
- default: ['all']
+ default: built-projects
mozharness:
by-test-platform:
android.*:
mochitest-flavor: plain
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
- android/androidarm_4_3.py
@@ -896,17 +885,17 @@ mochitest-webgl:
description: "Mochitest webgl run"
suite: mochitest/mochitest-gl
treeherder-symbol: tc-M(gl)
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows10.*: []
linux64-qr/.*: ['graphics', 'mozilla-central']
- default: ['all']
+ default: built-projects
chunks:
by-test-platform:
android.*: 10
default: 3
e10s:
by-test-platform:
macosx.*: true
default: both
@@ -952,20 +941,17 @@ mochitest-webgl:
mochitest-style:
description: "Mochitest plain run for style system"
suite: mochitest/plain-style
treeherder-symbol: tc-M(s)
loopback-video: true
docker-image: {"in-tree": "desktop1604-test"}
e10s: both
- run-on-projects:
- by-test-platform:
- linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
- default: ['all']
+ run-on-projects: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
mozharness:
mochitest-flavor: plain
script: desktop_unittest.py
no-read-buildbot-config: true
config:
by-test-platform:
default:
- unittests/linux_unittest.py
@@ -974,20 +960,17 @@ mochitest-style:
- --mochitest-suite=plain-style
mochitest-chrome-style:
description: "Mochitest chrome run for style system"
suite: mochitest/chrome-style
treeherder-symbol: tc-M(cs)
loopback-video: true
docker-image: {"in-tree": "desktop1604-test"}
- run-on-projects:
- by-test-platform:
- linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
- default: ['all']
+ run-on-projects: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
e10s: false
mozharness:
mochitest-flavor: chrome
script: desktop_unittest.py
no-read-buildbot-config: true
config:
by-test-platform:
default:
@@ -1000,17 +983,17 @@ reftest:
description: "Reftest run"
suite: reftest/reftest
treeherder-symbol: tc-R(R)
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows10.*: []
linux64-qr/.*: ['graphics', 'mozilla-central']
- default: ['all']
+ default: built-projects
instance-size:
by-test-platform:
android.*: xlarge
default: default
chunks:
by-test-platform:
android-4.3-arm7-api-15/debug: 48
android.*: 16
@@ -1058,17 +1041,17 @@ reftest-no-accel:
description: "Reftest not accelerated run"
suite: reftest/reftest-no-accel
treeherder-symbol: tc-R(Ru)
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows10.*: []
linux64-qr/.*: ['graphics', 'mozilla-central']
- default: ['all']
+ default: built-projects
chunks:
by-test-platform:
macosx.*: 1
default: 8
mozharness:
script: desktop_unittest.py
no-read-buildbot-config: true
config:
@@ -1089,17 +1072,17 @@ reftest-stylo:
suite: reftest/reftest-stylo
treeherder-symbol: tc-R(Rs)
docker-image: {"in-tree": "desktop1604-test"}
chunks: 16
run-on-projects:
by-test-platform:
linux64-stylo/opt: [ 'stylo', 'autoland', 'mozilla-central', 'try' ]
linux64-stylo/debug: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
- default: []
+ default: built-projects
e10s:
# run only e10s tests for this platform see bug 1343301
by-test-platform:
linux64-stylo/opt:
# no opt on inbound see bug 1339604
by-project:
stylo: true
autoland: true
@@ -1401,17 +1384,17 @@ web-platform-tests:
max-run-time: 7200
instance-size: xlarge
docker-image: {"in-tree": "desktop1604-test"}
checkout: true
run-on-projects:
by-test-platform:
windows10.*: []
windows7.*: ['mozilla-central', 'try']
- default: ['all']
+ default: built-projects
mozharness:
script: web_platform_tests.py
no-read-buildbot-config: true
chunked: true
config:
by-test-platform:
windows.*:
- web_platform_tests/prod_config_windows_taskcluster.py
@@ -1434,17 +1417,17 @@ web-platform-tests-reftests:
macosx.*: true
default: both
docker-image: {"in-tree": "desktop1604-test"}
checkout: true
run-on-projects:
by-test-platform:
windows10.*: []
windows7.*: ['mozilla-central', 'try']
- default: ['all']
+ default: built-projects
mozharness:
script: web_platform_tests.py
no-read-buildbot-config: true
config:
by-test-platform:
windows.*:
- web_platform_tests/prod_config_windows_taskcluster.py
default:
@@ -1480,17 +1463,17 @@ xpcshell:
by-test-platform:
linux64-jsdcov/opt: xpcshell-coverage
default: xpcshell
treeherder-symbol: tc-X(X)
run-on-projects:
by-test-platform:
windows.*: []
linux64-qr/.*: ['graphics']
- default: ['all']
+ default: built-projects
docker-image: {"in-tree": "desktop1604-test"}
chunks:
by-test-platform:
linux64/debug: 10
android-4.2-x86/opt: 6
# windows.*: 1
macosx.*: 1
default: 8
--- a/taskcluster/taskgraph/loader/test.py
+++ b/taskcluster/taskgraph/loader/test.py
@@ -39,16 +39,17 @@ def loader(kind, path, config, params, l
# generate all tests for all test platforms
for test_platform_name, test_platform in test_platforms.iteritems():
for test_name in test_platform['test-names']:
test = copy.deepcopy(test_descriptions[test_name])
test['build-platform'] = test_platform['build-platform']
test['test-platform'] = test_platform_name
test['build-label'] = test_platform['build-label']
+ test['build-attributes'] = test_platform['build-attributes']
test['test-name'] = test_name
if test_platform['nightly']:
test.setdefault('attributes', {})['nightly'] = True
logger.debug("Generating tasks for test {} on platform {}".format(
test_name, test['test-platform']))
yield test
@@ -83,16 +84,17 @@ def get_test_platforms(test_platforms_cf
logger.warning(
"No build task with platform {}; ignoring test platform {}".format(
build_platform, test_platform))
continue
test_platforms[test_platform] = {
'nightly': builds_by_platform[build_platform].attributes.get('nightly', False),
'build-platform': build_platform,
'build-label': builds_by_platform[build_platform].label,
+ 'build-attributes': builds_by_platform[build_platform].attributes,
}
test_platforms[test_platform].update(cfg)
return test_platforms
def expand_tests(test_sets_cfg, test_platforms):
"""Expand the test sets in `test_platforms` out to sets of test names.
Returns a dictionary like `get_test_platforms`, with an additional
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -94,19 +94,23 @@ test_description_schema = Schema({
# attributes to appear in the resulting task (later transforms will add the
# common attributes)
Optional('attributes'): {basestring: object},
# The `run_on_projects` attribute, defaulting to "all". This dictates the
# projects on which this task should be included in the target task set.
# See the attributes documentation for details.
- Optional('run-on-projects', default=['all']): optionally_keyed_by(
+ #
+ # Note that the special case 'built-projects', the default, uses the parent
+ # build task's run-on-projects, meaning that tests run only on platforms
+ # that are built.
+ Optional('run-on-projects', default='built-projects'): optionally_keyed_by(
'test-platform',
- [basestring]),
+ Any([basestring], 'built-projects')),
# the sheriffing tier for this task (default: set based on test platform)
Optional('tier'): optionally_keyed_by(
'test-platform',
Any(int, 'default')),
# number of chunks to create for this task. This can be keyed by test
# platform by passing a dictionary in the `by-test-platform` key. If the
@@ -263,16 +267,19 @@ test_description_schema = Schema({
# -- values supplied by the task-generation infrastructure
# the platform of the build this task is testing
'build-platform': basestring,
# the label of the build task generating the materials to test
'build-label': basestring,
+ # the build's attributes
+ 'build-attributes': {basestring: object},
+
# the platform on which the tests will run
'test-platform': basestring,
# the name of the test (the key in tests.yml)
'test-name': basestring,
# the product name, defaults to firefox
Optional('product'): basestring,
@@ -321,17 +328,17 @@ def set_defaults(config, tests):
# software-gl-layers is only meaningful on linux, where it defaults to True
if test['test-platform'].startswith('linux'):
test.setdefault('allow-software-gl-layers', True)
else:
test['allow-software-gl-layers'] = False
test.setdefault('os-groups', [])
test.setdefault('chunks', 1)
- test.setdefault('run-on-projects', ['all'])
+ test.setdefault('run-on-projects', 'built-projects')
test.setdefault('instance-size', 'default')
test.setdefault('max-run-time', 3600)
test.setdefault('reboot', True)
test['mozharness'].setdefault('extra-options', [])
yield test
@transforms.add
@@ -506,16 +513,25 @@ def enable_code_coverage(config, tests):
test['e10s'] = False
test['run-on-projects'] = []
elif test['build-platform'] == 'linux64-jsdcov/opt':
test['run-on-projects'] = []
yield test
@transforms.add
+def handle_run_on_projects(config, tests):
+ """Handle translating `built-projects` appropriately"""
+ for test in tests:
+ if test['run-on-projects'] == 'built-projects':
+ test['run-on-projects'] = test['build-attributes'].get('run_on_projects', ['all'])
+ yield test
+
+
+@transforms.add
def split_e10s(config, tests):
for test in tests:
e10s = test['e10s']
test.setdefault('attributes', {})
test['e10s'] = False
test['attributes']['e10s'] = False
@@ -691,17 +707,17 @@ def make_job_description(config, tests):
jobdesc['name'] = name
jobdesc['label'] = label
jobdesc['description'] = test['description']
jobdesc['when'] = test.get('when', {})
jobdesc['attributes'] = attributes
jobdesc['dependencies'] = {'build': build_label}
jobdesc['expires-after'] = test['expires-after']
jobdesc['routes'] = []
- jobdesc['run-on-projects'] = test.get('run-on-projects', ['all'])
+ jobdesc['run-on-projects'] = test['run-on-projects']
jobdesc['scopes'] = []
jobdesc['tags'] = test.get('tags', {})
jobdesc['optimizations'] = [['seta']] # always run SETA for tests
jobdesc['extra'] = {
'chunks': {
'current': test['this-chunk'],
'total': test['chunks'],
},