Bug 1363714: Split tests tasks between TC and BB. r=dustin
In order to migrate from buildbot to taskcluster gradually, we read an
external json file to tells the percentage of jobs that must run in
Taskcluster.
MozReview-Commit-ID: JQhvsXKpcLz
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -30,16 +30,18 @@ from taskgraph.util.schema import (
from voluptuous import (
Any,
Optional,
Required,
)
import copy
import logging
+import requests
+from collections import defaultdict
WORKER_TYPE = {
# default worker types keyed by instance-size
'large': 'aws-provisioner-v1/gecko-t-linux-large',
'xlarge': 'aws-provisioner-v1/gecko-t-linux-xlarge',
'legacy': 'aws-provisioner-v1/gecko-t-linux-medium',
'default': 'aws-provisioner-v1/gecko-t-linux-large',
# windows / os x worker types keyed by test-platform
@@ -389,31 +391,29 @@ def set_treeherder_machine_platform(conf
test['build-platform'], test['test-platform'])
yield test
@transforms.add
def set_worker_implementation(config, tests):
"""Set the worker implementation based on the test platform."""
for test in tests:
- if test['test-platform'].startswith('macosx'):
- # see if '-g' appears in try syntax
- if config.config['args'].generic_worker:
- test['worker-implementation'] = 'generic-worker'
- # see if '-w' appears in try syntax
- elif config.config['args'].taskcluster_worker:
+ test_platform = test['test-platform']
+ if test_platform.startswith('macosx'):
+ if config.config['args'].taskcluster_worker:
test['worker-implementation'] = 'native-engine'
else:
- test['worker-implementation'] = 'buildbot-bridge'
+ test['worker-implementation'] = 'generic-worker'
elif test.get('suite', '') == 'talos':
test['worker-implementation'] = 'buildbot-bridge'
- elif test['test-platform'].startswith('win'):
+ elif test_platform.startswith('win'):
test['worker-implementation'] = 'generic-worker'
else:
test['worker-implementation'] = 'docker-worker'
+
yield test
@transforms.add
def set_tier(config, tests):
"""Set the tier based on policy for all test descriptions that do not
specify a tier otherwise."""
for test in tests:
@@ -663,16 +663,43 @@ def parallel_stylo_tests(config, tests):
if test['test-platform'].startswith('linux64-stylo/'):
# add parallel stylo tests
test['mozharness'].setdefault('extra-options', [])\
.append('--parallel-stylo-traversal')
yield test
@transforms.add
+def allocate_to_bbb(config, tests):
+ """Make the load balancing between taskcluster and buildbot"""
+ j = get_load_balacing_settings()
+
+ tests_set = defaultdict(list)
+ for test in tests:
+ tests_set[test['test-platform']].append(test)
+
+ # Make the load balancing between taskcluster and buildbot
+ for test_platform, t in tests_set.iteritems():
+ # We sort the list to make the order of the tasks deterministic
+ t.sort(key=lambda x: (x['test-name'], x.get('this_chunk', 1)))
+ # The json file tells the percentage of tasks that run on
+ # taskcluster. The logic here is inverted, as tasks have been
+ # previously assigned to taskcluster. Therefore we assign the
+ # 1-p tasks to buildbot-bridge.
+ n = j.get(test_platform, 1.0)
+ if not (test_platform.startswith('mac')
+ and config.config['args'].taskcluster_worker):
+ for i in range(int(n * len(t)), len(t)):
+ t[i]['worker-implementation'] = 'buildbot-bridge'
+
+ for y in t:
+ yield y
+
+
+@transforms.add
def make_job_description(config, tests):
"""Convert *test* descriptions to *job* descriptions (input to
taskgraph.transforms.job)"""
for test in tests:
label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name'])
if test['chunks'] > 1:
label += '-{}'.format(test['this-chunk'])
@@ -762,8 +789,16 @@ def make_job_description(config, tests):
def normpath(path):
return path.replace('/', '\\')
def get_firefox_version():
with open('browser/config/version.txt', 'r') as f:
return f.readline().strip()
+
+
+def get_load_balacing_settings():
+ url = "https://s3.amazonaws.com/taskcluster-graph-scheduling/tests-load.json"
+ try:
+ return requests.get(url).json()
+ except Exception:
+ return {}
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -238,19 +238,16 @@ def parse_message(message):
parser.add_argument('--no-retry', dest='no_retry', action='store_true')
parser.add_argument('--include-nightly', dest='include_nightly', action='store_true')
# While we are transitioning from BB to TC, we want to push jobs to tc-worker
# machines but not overload machines with every try push. Therefore, we add
# this temporary option to be able to push jobs to tc-worker.
parser.add_argument('-w', '--taskcluster-worker',
dest='taskcluster_worker', action='store_true', default=False)
- # Similarly, an extra flag for enabling os x jobs in generic-worker
- parser.add_argument('-g', '--generic-worker',
- dest='generic_worker', action='store_true', default=False)
# In order to run test jobs multiple times
parser.add_argument('--rebuild', dest='trigger_tests', type=int, default=1)
parts = parts[try_idx:] if try_idx is not None else []
args, _ = parser.parse_known_args(parts)
return args