Bug 1316877: allow 'test-sets' in test-platforms.yml; r?jmaher
This adds a simple layer of indirection to avoid needless repetition of long
lists of tests.
MozReview-Commit-ID: 4hHyoNMwwGy
--- a/taskcluster/ci/android-test/test-platforms.yml
+++ b/taskcluster/ci/android-test/test-platforms.yml
@@ -8,18 +8,22 @@
# for the platform, referring to tests defined in test-sets.yml.
#
# Note that set does not depend on the tree; tree-dependent job selection
# should be performed in the target task selection phase of task-graph
# generation.
android-4.3-arm7-api-15/debug:
build-platform: android-api-15/debug
- test-set: debug-tests
+ test-sets:
+ - debug-tests
android-4.3-arm7-api-15/opt:
build-platform: android-api-15/opt
- test-set: opt-tests
+ test-sets:
+ - opt-tests
android-4.2-x86/opt:
build-platform: android-x86/opt
- test-set: x86-tests
+ test-sets:
+ - x86-tests
android-4.3-arm7-api-15-gradle/opt:
build-platform: android-api-15-gradle/opt
- test-set: gradle-tests
+ test-sets:
+ - gradle-tests
--- a/taskcluster/ci/desktop-test/test-platforms.yml
+++ b/taskcluster/ci/desktop-test/test-platforms.yml
@@ -8,75 +8,102 @@
# for the platform, referring to tests defined in test-sets.yml.
#
# Note that set does not depend on the tree; tree-dependent job selection
# should be performed in the target task selection phase of task-graph
# generation.
linux64/debug:
build-platform: linux64/debug
- test-set: all-tests-debug
+ test-sets:
+ - common-tests
+ - external-media-tests
+ - web-platform-tests
linux64/opt:
build-platform: linux64/opt
- test-set: all-tests-opt
+ test-sets:
+ - common-tests
+ - external-media-tests
+ - web-platform-tests
+ - opt-only-tests
+ - talos
# TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt
linux64-pgo/opt:
build-platform: linux64-pgo/opt
- test-set: all-tests-debug
+ test-sets:
+ - common-tests
+ - external-media-tests
+ - web-platform-tests
+
linux64-asan/opt:
build-platform: linux64-asan/opt
- test-set: asan-tests
+ test-sets:
+ - common-tests
# Stylo builds only run a subset of tests for the moment. So give them
# their own test set.
linux64-stylo/debug:
build-platform: linux64-stylo/debug
- test-set: stylo-tests
+ test-sets:
+ - stylo-tests
linux64-stylo/opt:
build-platform: linux64-stylo/opt
- test-set: stylo-tests
+ test-sets:
+ - stylo-tests
linux64-ccov/opt:
build-platform: linux64-ccov/opt
- test-set: ccov-code-coverage-tests
+ test-sets:
+ - ccov-code-coverage-tests
linux64-jsdcov/opt:
build-platform: linux64-jsdcov/opt
- test-set: jsdcov-code-coverage-tests
+ test-sets:
+ - jsdcov-code-coverage-tests
# win32 vm
windows7-32-vm/debug:
build-platform: win32/debug
- test-set: windows-vm-tests
+ test-sets:
+ - windows-vm-tests
windows7-32-vm/opt:
build-platform: win32/opt
- test-set: windows-vm-tests
+ test-sets:
+ - windows-vm-tests
# win32 gpu
#windows7-32/debug:
# build-platform: win32/debug
-# test-set: windows-gpu-tests
+# test-sets:
+# - windows-gpu-tests
#windows7-32/opt:
# build-platform: win32/opt
-# test-set: windows-gpu-tests
+# test-sets:
+# - windows-gpu-tests
# win64 vm
windows10-64-vm/debug:
build-platform: win64/debug
- test-set: windows-vm-tests
+ test-sets:
+ - windows-vm-tests
windows10-64-vm/opt:
build-platform: win64/opt
- test-set: windows-vm-tests
+ test-sets:
+ - windows-vm-tests
# win64 gpu
#windows10-64/debug:
# build-platform: win64/debug
-# test-set: windows-gpu-tests
+# test-sets:
+# - windows-gpu-tests
#windows10-64/opt:
# build-platform: win64/opt
-# test-set: windows-gpu-tests
+# test-sets:
+# - windows-gpu-tests
macosx64/debug:
build-platform: macosx64/debug
- test-set: macosx64-tests
+ test-sets:
+ - macosx64-tests
# macosx64/opt:
# build-platform: macosx64/opt
-# test-set: macosx64-tests
+# test-sets:
+# - macosx64-tests
--- a/taskcluster/ci/desktop-test/test-sets.yml
+++ b/taskcluster/ci/desktop-test/test-sets.yml
@@ -4,89 +4,23 @@
# Note that set does not depend on the tree; tree-dependent job selection
# should be performed in the target task selection phase of task-graph
# generation.
#
# A test set has a name, and a list of tests that it contains.
#
# Test names given here reference tests.yml.
-all-tests-debug:
- - cppunit
- - crashtest
- - external-media-tests
- - firefox-ui-functional-local
- - firefox-ui-functional-remote
- - gtest
- - jittest
- - jsreftest
- - marionette
- - mochitest
- - mochitest-a11y
- - mochitest-browser-chrome
- - mochitest-chrome
- - mochitest-clipboard
- - mochitest-devtools-chrome
- - mochitest-gpu
- - mochitest-jetpack
- - mochitest-media
- - mochitest-webgl
- - reftest
- - reftest-no-accel
- - web-platform-tests
- - web-platform-tests-reftests
- - web-platform-tests-wdspec
- - xpcshell
+##
+# Universal tests
+#
+# Tests that run just about everywhere; common-tests is the shared core, with
+# a few add-on test sets that are included or omitted on some platforms.
-# This is the same as all-tests-debug,
-# with the addition of "mochitest-valgrind".
-all-tests-opt:
- - cppunit
- - crashtest
- - external-media-tests
- - firefox-ui-functional-local
- - firefox-ui-functional-remote
- - gtest
- - jittest
- - jsreftest
- - marionette
- - mochitest
- - mochitest-a11y
- - mochitest-browser-chrome
- - mochitest-chrome
- - mochitest-clipboard
- - mochitest-devtools-chrome
- - mochitest-gpu
- - mochitest-jetpack
- - mochitest-media
- - mochitest-webgl
- - reftest
- - reftest-no-accel
- - web-platform-tests
- - web-platform-tests-reftests
- - web-platform-tests-wdspec
- - xpcshell
- - mochitest-valgrind
- - talos-chrome
- - talos-dromaeojs
- - talos-g1
- - talos-g2
- - talos-g3
- - talos-g4
- - talos-other
- - talos-svgr
- - talos-tp5o
-
-stylo-tests:
- - cppunit
- - crashtest
- - reftest-stylo
- - mochitest-stylo
-
-asan-tests:
+common-tests:
- cppunit
- crashtest
- firefox-ui-functional-local
- firefox-ui-functional-remote
- gtest
- jittest
- jsreftest
- marionette
@@ -99,16 +33,60 @@ asan-tests:
- mochitest-gpu
- mochitest-jetpack
- mochitest-media
- mochitest-webgl
- reftest
- reftest-no-accel
- xpcshell
+web-platform-tests:
+ - web-platform-tests
+ - web-platform-tests-reftests
+ - web-platform-tests-wdspec
+
+external-media-tests:
+ - external-media-tests
+
+opt-only-tests:
+ - mochitest-valgrind
+
+talos:
+ - talos-chrome
+ - talos-dromaeojs
+ - talos-g1
+ - talos-g2
+ - talos-g3
+ - talos-g4
+ - talos-other
+ - talos-svgr
+ - talos-tp5o
+
+##
+# Limited test sets for specific platforms
+
+stylo-tests:
+ - cppunit
+ - crashtest
+ - reftest-stylo
+ - mochitest-stylo
+
+ccov-code-coverage-tests:
+ - mochitest
+ - mochitest-browser-chrome
+ - mochitest-devtools-chrome
+ - xpcshell
+
+jsdcov-code-coverage-tests:
+ - mochitest-browser-chrome
+ - mochitest-devtools-chrome
+
+##
+# Test sets still being greened up in various ways
+
windows-vm-tests:
- cppunit
#- crashtest
- external-media-tests
#- gtest
#- jittest
- jsreftest
#- marionette
@@ -128,26 +106,16 @@ windows-vm-tests:
# these tests currently run on hardware, but may migrate above when validated
# note: on win, mochitest-a11y and mochitest-chrome come under mochitest-other
# windows-hw-tests:
# - mochitest-clipboard
# - mochitest-gpu
# - mochitest-other
-ccov-code-coverage-tests:
- - mochitest
- - mochitest-browser-chrome
- - mochitest-devtools-chrome
- - xpcshell
-
-jsdcov-code-coverage-tests:
- - mochitest-browser-chrome
- - mochitest-devtools-chrome
-
macosx64-tests:
- cppunit
- crashtest
# - gtest
- jsreftest
# - marionette
# - mochitest
# - mochitest-browser-chrome
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -83,22 +83,23 @@ Tests
-----
Test tasks for Gecko products are divided into several kinds, but share a
common implementation. The process goes like this, based on a set of YAML
files named in ``kind.yml``:
* For each build task, determine the related test platforms based on the build
platform. For example, a Windows 2010 build might be tested on Windows 7
- and Windows 10. Each test platform specifies a "test set" indicating which
+ and Windows 10. Each test platform specifies "test sets" indicating which
tests to run. This is configured in the file named
``test-platforms.yml``.
* Each test set is expanded to a list of tests to run. This is configured in
- the file named by ``test-sets.yml``.
+ the file named by ``test-sets.yml``. A platform may specify several test
+ sets, in which case the union of those sets is used.
* Each named test is looked up in the file named by ``tests.yml`` to find a
test description. This test description indicates what the test does, how
it is reported to treeherder, and how to perform the test, all in a
platform-independent fashion.
* Each test description is converted into one or more tasks. This is
performed by a sequence of transforms defined in the ``transforms`` key in
--- a/taskcluster/taskgraph/task/test.py
+++ b/taskcluster/taskgraph/task/test.py
@@ -82,31 +82,33 @@ class TestTask(transform.TransformTask):
for test_platform, cfg in test_platforms_cfg.iteritems():
build_platform = cfg['build-platform']
if build_platform not in builds_by_platform:
logger.warning(
"No build task with platform {}; ignoring test platform {}".format(
build_platform, test_platform))
continue
test_platforms[test_platform] = {
- 'test-set': cfg['test-set'],
'build-platform': build_platform,
'build-label': builds_by_platform[build_platform],
}
+ test_platforms[test_platform].update(cfg)
return test_platforms
@classmethod
def expand_tests(cls, test_sets_cfg, test_platforms):
"""Expand the test sets in `test_platforms` out to sets of test names.
Returns a dictionary like `get_test_platforms`, with an additional
`test-names` key for each test platform, containing a set of test
names."""
rv = {}
for test_platform, cfg in test_platforms.iteritems():
- test_set = cfg['test-set']
- if test_set not in test_sets_cfg:
+ test_sets = cfg['test-sets']
+ if not set(test_sets) < set(test_sets_cfg):
raise Exception(
- "Test set '{}' for test platform {} is not defined".format(
- test_set, test_platform))
- test_names = test_sets_cfg[test_set]
+ "Test sets {} for test platform {} are not defined".format(
+ ', '.join(test_sets), test_platform))
+ test_names = set()
+ for test_set in test_sets:
+ test_names.update(test_sets_cfg[test_set])
rv[test_platform] = cfg.copy()
rv[test_platform]['test-names'] = test_names
return rv