Bug 1316877: allow 'test-sets' in test-platforms.yml; r?jmaher draft
authorDustin J. Mitchell <dustin@mozilla.com>
Sun, 04 Dec 2016 15:38:14 -0500
changeset 453181 36e349af04dfefb88c91e8507cb248520bbcc8d2
parent 453180 8ef0e968b7d2feb1376a2e70bd0555d66278fb9d
child 540378 4f577ee2f1493cc406b08fe23e6fc4784e89f121
push id39573
push userdmitchell@mozilla.com
push dateThu, 22 Dec 2016 22:03:08 +0000
reviewersjmaher
bugs1316877
milestone53.0a1
Bug 1316877: allow 'test-sets' in test-platforms.yml; r?jmaher This adds a simple layer of indirection to avoid needless repetition of long lists of tests. MozReview-Commit-ID: 4hHyoNMwwGy
taskcluster/ci/android-test/test-platforms.yml
taskcluster/ci/desktop-test/test-platforms.yml
taskcluster/ci/desktop-test/test-sets.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/task/test.py
--- a/taskcluster/ci/android-test/test-platforms.yml
+++ b/taskcluster/ci/android-test/test-platforms.yml
@@ -8,18 +8,22 @@
 # for the platform, referring to tests defined in test-sets.yml.
 #
 # Note that set does not depend on the tree; tree-dependent job selection
 # should be performed in the target task selection phase of task-graph
 # generation.
 
 android-4.3-arm7-api-15/debug:
     build-platform: android-api-15/debug
-    test-set: debug-tests
+    test-sets:
+        - debug-tests
 android-4.3-arm7-api-15/opt:
     build-platform: android-api-15/opt
-    test-set: opt-tests
+    test-sets:
+        - opt-tests
 android-4.2-x86/opt:
     build-platform: android-x86/opt
-    test-set: x86-tests
+    test-sets:
+        - x86-tests
 android-4.3-arm7-api-15-gradle/opt:
     build-platform: android-api-15-gradle/opt
-    test-set: gradle-tests
+    test-sets:
+        - gradle-tests
--- a/taskcluster/ci/desktop-test/test-platforms.yml
+++ b/taskcluster/ci/desktop-test/test-platforms.yml
@@ -8,75 +8,102 @@
 # for the platform, referring to tests defined in test-sets.yml.
 #
 # Note that set does not depend on the tree; tree-dependent job selection
 # should be performed in the target task selection phase of task-graph
 # generation.
 
 linux64/debug:
     build-platform: linux64/debug
-    test-set: all-tests-debug
+    test-sets:
+        - common-tests
+        - external-media-tests
+        - web-platform-tests
 linux64/opt:
     build-platform: linux64/opt
-    test-set: all-tests-opt
+    test-sets:
+        - common-tests
+        - external-media-tests
+        - web-platform-tests
+        - opt-only-tests
+        - talos
 
 # TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt
 linux64-pgo/opt:
     build-platform: linux64-pgo/opt
-    test-set: all-tests-debug
+    test-sets:
+        - common-tests
+        - external-media-tests
+        - web-platform-tests
+
 linux64-asan/opt:
     build-platform: linux64-asan/opt
-    test-set: asan-tests
+    test-sets:
+        - common-tests
 
 # Stylo builds only run a subset of tests for the moment. So give them
 # their own test set.
 linux64-stylo/debug:
     build-platform: linux64-stylo/debug
-    test-set: stylo-tests
+    test-sets:
+        - stylo-tests
 linux64-stylo/opt:
     build-platform: linux64-stylo/opt
-    test-set: stylo-tests
+    test-sets:
+        - stylo-tests
 
 linux64-ccov/opt:
     build-platform: linux64-ccov/opt
-    test-set: ccov-code-coverage-tests
+    test-sets:
+        - ccov-code-coverage-tests
 linux64-jsdcov/opt:
     build-platform: linux64-jsdcov/opt
-    test-set: jsdcov-code-coverage-tests
+    test-sets:
+        - jsdcov-code-coverage-tests
 
 # win32 vm
 windows7-32-vm/debug:
     build-platform: win32/debug
-    test-set: windows-vm-tests
+    test-sets:
+        - windows-vm-tests
 windows7-32-vm/opt:
     build-platform: win32/opt
-    test-set: windows-vm-tests
+    test-sets:
+        - windows-vm-tests
 
 # win32 gpu
 #windows7-32/debug:
 #    build-platform: win32/debug
-#    test-set: windows-gpu-tests
+#    test-sets:
+#        - windows-gpu-tests
 #windows7-32/opt:
 #    build-platform: win32/opt
-#    test-set: windows-gpu-tests
+#    test-sets:
+#        - windows-gpu-tests
 
 # win64 vm
 windows10-64-vm/debug:
     build-platform: win64/debug
-    test-set: windows-vm-tests
+    test-sets:
+        - windows-vm-tests
 windows10-64-vm/opt:
     build-platform: win64/opt
-    test-set: windows-vm-tests
+    test-sets:
+        - windows-vm-tests
 
 # win64 gpu
 #windows10-64/debug:
 #    build-platform: win64/debug
-#    test-set: windows-gpu-tests
+#    test-sets:
+#        - windows-gpu-tests
 #windows10-64/opt:
 #    build-platform: win64/opt
-#    test-set: windows-gpu-tests
+#    test-sets:
+#        - windows-gpu-tests
 
 macosx64/debug:
     build-platform: macosx64/debug
-    test-set: macosx64-tests
+    test-sets:
+        - macosx64-tests
 # macosx64/opt:
 #    build-platform: macosx64/opt
-#    test-set: macosx64-tests
+#    test-sets:
+#        - macosx64-tests
--- a/taskcluster/ci/desktop-test/test-sets.yml
+++ b/taskcluster/ci/desktop-test/test-sets.yml
@@ -4,89 +4,23 @@
 # Note that set does not depend on the tree; tree-dependent job selection
 # should be performed in the target task selection phase of task-graph
 # generation.
 #
 # A test set has a name, and a list of tests that it contains.
 #
 # Test names given here reference tests.yml.
 
-all-tests-debug:
-    - cppunit
-    - crashtest
-    - external-media-tests
-    - firefox-ui-functional-local
-    - firefox-ui-functional-remote
-    - gtest
-    - jittest
-    - jsreftest
-    - marionette
-    - mochitest
-    - mochitest-a11y
-    - mochitest-browser-chrome
-    - mochitest-chrome
-    - mochitest-clipboard
-    - mochitest-devtools-chrome
-    - mochitest-gpu
-    - mochitest-jetpack
-    - mochitest-media
-    - mochitest-webgl
-    - reftest
-    - reftest-no-accel
-    - web-platform-tests
-    - web-platform-tests-reftests
-    - web-platform-tests-wdspec
-    - xpcshell
+##
+# Universal tests
+#
+# Tests that run just about everywhere; common-tests is the shared core, with
+# a few add-on test sets that are included or omitted on some platforms.
 
-# This is the same as all-tests-debug,
-# with the addition of "mochitest-valgrind".
-all-tests-opt:
-    - cppunit
-    - crashtest
-    - external-media-tests
-    - firefox-ui-functional-local
-    - firefox-ui-functional-remote
-    - gtest
-    - jittest
-    - jsreftest
-    - marionette
-    - mochitest
-    - mochitest-a11y
-    - mochitest-browser-chrome
-    - mochitest-chrome
-    - mochitest-clipboard
-    - mochitest-devtools-chrome
-    - mochitest-gpu
-    - mochitest-jetpack
-    - mochitest-media
-    - mochitest-webgl
-    - reftest
-    - reftest-no-accel
-    - web-platform-tests
-    - web-platform-tests-reftests
-    - web-platform-tests-wdspec
-    - xpcshell
-    - mochitest-valgrind
-    - talos-chrome
-    - talos-dromaeojs
-    - talos-g1
-    - talos-g2
-    - talos-g3
-    - talos-g4
-    - talos-other
-    - talos-svgr
-    - talos-tp5o
-
-stylo-tests:
-    - cppunit
-    - crashtest
-    - reftest-stylo
-    - mochitest-stylo
-
-asan-tests:
+common-tests:
     - cppunit
     - crashtest
     - firefox-ui-functional-local
     - firefox-ui-functional-remote
     - gtest
     - jittest
     - jsreftest
     - marionette
@@ -99,16 +33,60 @@ asan-tests:
     - mochitest-gpu
     - mochitest-jetpack
     - mochitest-media
     - mochitest-webgl
     - reftest
     - reftest-no-accel
     - xpcshell
 
+web-platform-tests:
+    - web-platform-tests
+    - web-platform-tests-reftests
+    - web-platform-tests-wdspec
+
+external-media-tests:
+    - external-media-tests
+
+opt-only-tests:
+    - mochitest-valgrind
+
+talos:
+    - talos-chrome
+    - talos-dromaeojs
+    - talos-g1
+    - talos-g2
+    - talos-g3
+    - talos-g4
+    - talos-other
+    - talos-svgr
+    - talos-tp5o
+
+##
+# Limited test sets for specific platforms
+
+stylo-tests:
+    - cppunit
+    - crashtest
+    - reftest-stylo
+    - mochitest-stylo
+
+ccov-code-coverage-tests:
+    - mochitest
+    - mochitest-browser-chrome
+    - mochitest-devtools-chrome
+    - xpcshell
+
+jsdcov-code-coverage-tests:
+    - mochitest-browser-chrome
+    - mochitest-devtools-chrome
+
+##
+# Test sets still being greened up in various ways
+
 windows-vm-tests:
     - cppunit
     #- crashtest
     - external-media-tests
     #- gtest
     #- jittest
     - jsreftest
     #- marionette
@@ -128,26 +106,16 @@ windows-vm-tests:
 
 # these tests currently run on hardware, but may migrate above when validated
 # note: on win, mochitest-a11y and mochitest-chrome come under mochitest-other
 # windows-hw-tests:
 #    - mochitest-clipboard
 #    - mochitest-gpu
 #    - mochitest-other
 
-ccov-code-coverage-tests:
-    - mochitest
-    - mochitest-browser-chrome
-    - mochitest-devtools-chrome
-    - xpcshell
-
-jsdcov-code-coverage-tests:
-    - mochitest-browser-chrome
-    - mochitest-devtools-chrome
-
 macosx64-tests:
     - cppunit
     - crashtest
     # - gtest
     - jsreftest
     # - marionette
     # - mochitest
     # - mochitest-browser-chrome
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -83,22 +83,23 @@ Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
    platform.  For example, a Windows 2010 build might be tested on Windows 7
-   and Windows 10.  Each test platform specifies a "test set" indicating which
+   and Windows 10.  Each test platform specifies "test sets" indicating which
    tests to run.  This is configured in the file named
    ``test-platforms.yml``.
 
  * Each test set is expanded to a list of tests to run.  This is configured in
-   the file named by ``test-sets.yml``.
+   the file named by ``test-sets.yml``. A platform may specify several test
+   sets, in which case the union of those sets is used.
 
  * Each named test is looked up in the file named by ``tests.yml`` to find a
    test description.  This test description indicates what the test does, how
    it is reported to treeherder, and how to perform the test, all in a
    platform-independent fashion.
 
  * Each test description is converted into one or more tasks.  This is
    performed by a sequence of transforms defined in the ``transforms`` key in
--- a/taskcluster/taskgraph/task/test.py
+++ b/taskcluster/taskgraph/task/test.py
@@ -82,31 +82,33 @@ class TestTask(transform.TransformTask):
         for test_platform, cfg in test_platforms_cfg.iteritems():
             build_platform = cfg['build-platform']
             if build_platform not in builds_by_platform:
                 logger.warning(
                     "No build task with platform {}; ignoring test platform {}".format(
                         build_platform, test_platform))
                 continue
             test_platforms[test_platform] = {
-                'test-set': cfg['test-set'],
                 'build-platform': build_platform,
                 'build-label': builds_by_platform[build_platform],
             }
+            test_platforms[test_platform].update(cfg)
         return test_platforms
 
     @classmethod
     def expand_tests(cls, test_sets_cfg, test_platforms):
         """Expand the test sets in `test_platforms` out to sets of test names.
         Returns a dictionary like `get_test_platforms`, with an additional
         `test-names` key for each test platform, containing a set of test
         names."""
         rv = {}
         for test_platform, cfg in test_platforms.iteritems():
-            test_set = cfg['test-set']
-            if test_set not in test_sets_cfg:
+            test_sets = cfg['test-sets']
+            if not set(test_sets) < set(test_sets_cfg):
                 raise Exception(
-                    "Test set '{}' for test platform {} is not defined".format(
-                        test_set, test_platform))
-            test_names = test_sets_cfg[test_set]
+                    "Test sets {} for test platform {} are not defined".format(
+                        ', '.join(test_sets), test_platform))
+            test_names = set()
+            for test_set in test_sets:
+                test_names.update(test_sets_cfg[test_set])
             rv[test_platform] = cfg.copy()
             rv[test_platform]['test-names'] = test_names
         return rv