Bug 1475541 - Make QR test jobs tier-1 by default. r?jmaher,Aryx draft
authorKartikaya Gupta <kgupta@mozilla.com>
Fri, 13 Jul 2018 09:44:06 -0400
changeset 817821 5ecb3da17ab5ee59c6dcd6d1fdb4d23301bfa62f
parent 817669 e951f4ad123aa87d1d392c286db14cabb41a8560
push id116174
push userkgupta@mozilla.com
push dateFri, 13 Jul 2018 13:44:30 +0000
reviewersjmaher, Aryx
bugs1475541
milestone63.0a1
Bug 1475541 - Make QR test jobs tier-1 by default. r?jmaher,Aryx This patch makes the QR test platforms tier-1 by default, and removes the ad-hoc bits that were making individual QR jobs tier-1 before. However, it also explicitly downgrades some QR jobs to tier-2 or tier-3; comments in the yml files indicate why. MozReview-Commit-ID: 1UfPuhcMvIW
taskcluster/ci/test/mochitest.yml
taskcluster/ci/test/reftest.yml
taskcluster/ci/test/talos.yml
taskcluster/ci/test/web-platform.yml
taskcluster/taskgraph/transforms/tests.py
--- a/taskcluster/ci/test/mochitest.yml
+++ b/taskcluster/ci/test/mochitest.yml
@@ -38,16 +38,20 @@ mochitest:
             linux64-jsdcov/opt: mochitest/plain-chunked-coverage
             default: mochitest/plain-chunked
     treeherder-symbol: M()
     loopback-video: true
     run-on-projects:
         by-test-platform:
             windows10-64-qr/debug: []  # bug 1424755
             default: built-projects
+    tier:
+        by-test-platform:
+            windows10-64-qr/debug: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
+            default: default
     instance-size:
         by-test-platform:
             linux64-jsdcov/opt: xlarge
             android-em.*: xlarge
             default: default
     virtualization:
         by-test-platform:
             windows10-64-qr/.*: virtual-with-gpu
@@ -260,20 +264,16 @@ mochitest-gpu:
         mochitest-flavor: plain
         extra-options:
             by-test-platform:
                 android-em.*:
                     # note that Android runs fewer suites than other platforms
                     - --test-suite=mochitest-plain-gpu
                 default:
                     - --mochitest-suite=plain-gpu,chrome-gpu,browser-chrome-gpu
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-media:
     description: "Mochitest media run"
     suite: mochitest/mochitest-media
     treeherder-symbol: M(mda)
     max-run-time:
         by-test-platform:
             windows10-64-ccov/debug: 7200
@@ -300,17 +300,16 @@ mochitest-media:
             by-test-platform:
                 android.*: false
                 macosx64.*: false
                 windows10-64.*: false
                 default: true
     tier:
         by-test-platform:
             android-hw.*: 3
-            linux64-qr/.*: 1
             default: default
 
 mochitest-plain-headless:
     description: "Mochitest plain headless run"
     suite: mochitest/plain-chunked
     treeherder-symbol: M(h)
     loopback-video: true
     chunks:
@@ -361,20 +360,16 @@ mochitest-webgl1-core:
     max-run-time:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-webgl1-ext:
     description: "Mochitest webgl1-ext run"
     suite: mochitest/mochitest-webgl1-ext
     treeherder-symbol: M(gl1e)
     run-on-projects:
         by-test-platform:
             windows10-64-ccov/.*: []  # Do not run on Windows ccov, see bug 1419475.
@@ -388,20 +383,16 @@ mochitest-webgl1-ext:
     max-run-time:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-webgl2-core:
     description: "Mochitest webgl2-core run"
     suite: mochitest/mochitest-webgl2-core
     treeherder-symbol: M(gl2c)
     run-on-projects:
         by-test-platform:
             windows10-64-ccov/.*: []  # Do not run on Windows ccov, see bug 1419475.
@@ -415,20 +406,16 @@ mochitest-webgl2-core:
     max-run-time:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-webgl2-ext:
     description: "Mochitest webgl2-ext run"
     suite: mochitest/mochitest-webgl2-ext
     treeherder-symbol: M(gl2e)
     run-on-projects:
         by-test-platform:
             windows10-64-ccov/.*: []  # Do not run on Windows ccov, see bug 1419475.
@@ -444,20 +431,16 @@ mochitest-webgl2-ext:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
         chunked: true
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-webgl2-deqp:
     description: "Mochitest webgl2-deqp run"
     suite: mochitest/mochitest-webgl2-deqp
     treeherder-symbol: M(gl2d)
     run-on-projects:
         by-test-platform:
             windows10-64-ccov/.*: []  # Do not run on Windows ccov, see bug 1419475.
@@ -473,12 +456,8 @@ mochitest-webgl2-deqp:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
         chunked: true
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
--- a/taskcluster/ci/test/reftest.yml
+++ b/taskcluster/ci/test/reftest.yml
@@ -42,20 +42,16 @@ crashtest:
             android-em-4.3-arm7-api-16/debug: 10
             android-em-7.0-x86/opt: 1
             android-em.*: 4
             default: 1
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
-    tier:
-        by-test-platform:
-            .*-qr/.*: 1
-            default: default
 
 jsreftest:
     description: "JS Reftest run"
     suite: reftest/jsreftest
     schedules-component: jsreftest  # scheduling for this reftest is different from the others..
     treeherder-symbol: R(J)
     run-on-projects: built-projects
     instance-size:
@@ -74,20 +70,16 @@ jsreftest:
             linux32/debug: 5
             macosx.*: 2
             default: 3
     max-run-time:
         by-test-platform:
             android-em.*: 7200
             windows10-64-ccov/debug: 7200
             default: 3600
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 reftest:
     description: "Reftest run"
     suite: reftest/reftest
     treeherder-symbol: R(R)
     run-on-projects: built-projects
     instance-size:
         by-test-platform:
@@ -119,20 +111,16 @@ reftest:
             default: 3600
     mozharness:
         chunked:
             by-test-platform:
                 android-em.*: false
                 macosx64/opt: false
                 windows10-64.*/opt: false
                 default: true
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 reftest-gpu:
     description: "Reftest GPU run"
     suite: reftest/reftest-gpu
     treeherder-symbol: R(Rg)
     chunks:
         by-test-platform:
             windows.*/opt: 2
--- a/taskcluster/ci/test/talos.yml
+++ b/taskcluster/ci/test/talos.yml
@@ -28,29 +28,37 @@ job-defaults:
 talos-bcv:
     description: "Talos basic compositor video"
     try-name: bcv
     treeherder-symbol: T(bcv)
     run-on-projects:
         by-test-platform:
             .*-qr/.*: []  # this test is not useful with webrender
             default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
+    tier:
+        by-test-platform:
+            .*-qr/.*: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
+            default: default
     max-run-time: 1800
     mozharness:
         extra-options:
             - --suite=bcv
 
 talos-bcv-profiling:
     description: "Talos profiling basic compositor video"
     try-name: bcv-profiling
     treeherder-symbol: T-P(bcv)
     run-on-projects:
         by-test-platform:
             .*-qr/.*: []  # this test is not useful with webrender
             default: ['mozilla-central', 'try']
+    tier:
+        by-test-platform:
+            .*-qr/.*: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
+            default: default
     max-run-time: 1800
     mozharness:
         extra-options:
             - --suite=bcv
             - --geckoProfile
 
 talos-chrome:
     description: "Talos chrome"
@@ -479,15 +487,19 @@ talos-xperf:
     description: "Talos xperf"
     try-name: xperf
     treeherder-symbol: T(x)
     virtualization: virtual
     run-on-projects:
         by-test-platform:
             windows7-32.*: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
             default: []
+    tier:
+        by-test-platform:
+            windows7-32.*: default
+            default: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
     mozharness:
         extra-options:
             - --suite=xperf
         config:
             by-test-platform:
                 windows.*:
                     - talos/windows_vm_config.py
--- a/taskcluster/ci/test/web-platform.yml
+++ b/taskcluster/ci/test/web-platform.yml
@@ -29,18 +29,22 @@ web-platform-tests:
             windows10-64-ccov/debug: 10800
             default: 7200
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
     run-on-projects:
         by-test-platform:
-            linux64-qr/.*: ['release', 'try']
+            linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
+    tier:
+        by-test-platform:
+            linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
+            default: default
     mozharness:
         chunked: true
         extra-options:
             - --test-type=testharness
 
 web-platform-tests-headless:
     description: "Web platform test headless run"
     suite: web-platform-tests
@@ -79,18 +83,22 @@ web-platform-tests-reftests:
             windows.*: 1
             default: 6
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
     run-on-projects:
         by-test-platform:
-            linux64-qr/.*: ['release', 'try']
+            linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
+    tier:
+        by-test-platform:
+            linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
+            default: default
     mozharness:
         extra-options:
             - --test-type=reftest
 
 web-platform-tests-reftests-headless:
     description: "Web platform reftest headless run"
     suite: web-platform-tests-reftests
     treeherder-symbol: W(WrH)
@@ -119,18 +127,22 @@ web-platform-tests-wdspec:
     description: "Web platform webdriver-spec run"
     suite: web-platform-tests-wdspec
     treeherder-symbol: W(Wd)
     mozharness:
         extra-options:
             - --test-type=wdspec
     run-on-projects:
         by-test-platform:
-            linux64-qr/.*: ['release', 'try']
+            linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
+    tier:
+        by-test-platform:
+            linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
+            default: default
 
 web-platform-tests-wdspec-headless:
     description: "Web platform webdriver-spec headless run"
     suite: web-platform-tests-wdspec
     treeherder-symbol: W(WdH)
     run-on-projects: []  # disabled pending releng approval
     mozharness:
         extra-options:
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -592,31 +592,37 @@ def set_tier(config, tests):
                                          'linux32-nightly/opt',
                                          'linux32-devedition/opt',
                                          'linux64/opt',
                                          'linux64-nightly/opt',
                                          'linux64/debug',
                                          'linux64-pgo/opt',
                                          'linux64-devedition/opt',
                                          'linux64-asan/opt',
+                                         'linux64-qr/opt',
+                                         'linux64-qr/debug',
                                          'windows7-32/debug',
                                          'windows7-32/opt',
                                          'windows7-32-pgo/opt',
                                          'windows7-32-devedition/opt',
                                          'windows7-32-nightly/opt',
                                          'windows10-64/debug',
                                          'windows10-64/opt',
                                          'windows10-64-pgo/opt',
                                          'windows10-64-devedition/opt',
                                          'windows10-64-nightly/opt',
                                          'windows10-64-asan/opt',
+                                         'windows10-64-qr/opt',
+                                         'windows10-64-qr/debug',
                                          'macosx64/opt',
                                          'macosx64/debug',
                                          'macosx64-nightly/opt',
                                          'macosx64-devedition/opt',
+                                         'macosx64-qr/opt',
+                                         'macosx64-qr/debug',
                                          'android-em-4.3-arm7-api-16/opt',
                                          'android-em-4.3-arm7-api-16/debug',
                                          'android-em-4.2-x86/opt']:
                 test['tier'] = 1
             else:
                 test['tier'] = 2
 
         yield test