--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -47,8 +47,34 @@ treeherder:
'TMW': 'Toolchain builds for Windows MinGW'
'TW32': 'Toolchain builds for Windows 32-bits'
'TW64': 'Toolchain builds for Windows 64-bits'
'SM-tc': 'Spidermonkey builds'
'pub': 'APK publishing'
'p': 'Partial generation'
'ps': 'Partials signing'
'Rel': 'Release promotion'
+
+try:
+ # We have a few platforms for which we want to do some "extra" builds, or at
+ # least build-ish things. Sort of. Anyway, these other things are implemented
+ # as different "platforms". These do *not* automatically ride along with "-p
+ # all"
+ ridealong-builds:
+ 'android-api-16':
+ - 'android-api-16-l10n'
+ 'linux':
+ - 'linux-l10n'
+ 'linux64':
+ - 'linux64-l10n'
+ - 'sm-plain'
+ - 'sm-nonunified'
+ - 'sm-arm-sim'
+ - 'sm-arm64-sim'
+ - 'sm-compacting'
+ - 'sm-rootanalysis'
+ - 'sm-package'
+ - 'sm-tsan'
+ - 'sm-asan'
+ - 'sm-mozjs-sys'
+ - 'sm-msan'
+ - 'sm-fuzzing'
+ - 'sm-rust-bindings'
--- a/taskcluster/taskgraph/config.py
+++ b/taskcluster/taskgraph/config.py
@@ -6,14 +6,21 @@ from __future__ import absolute_import,
from .util.schema import validate_schema, Schema
from voluptuous import Required
graph_config_schema = Schema({
Required('treeherder'): {
# Mapping of treeherder group symbols to descriptive names
Required('group-names'): {basestring: basestring}
- }
+ },
+ Required('try'): {
+ # We have a few platforms for which we want to do some "extra" builds, or at
+ # least build-ish things. Sort of. Anyway, these other things are implemented
+ # as different "platforms". These do *not* automatically ride along with "-p
+ # all"
+ Required('ridealong-builds', default={}): {basestring: [basestring]},
+ },
})
def validate_graph_config(config):
return validate_schema(graph_config_schema, config, "Invalid graph configuration:")
--- a/taskcluster/taskgraph/filter_tasks.py
+++ b/taskcluster/taskgraph/filter_tasks.py
@@ -19,29 +19,29 @@ def filter_task(name):
"""Generator to declare a task filter function."""
def wrap(func):
filter_task_functions[name] = func
return func
return wrap
@filter_task('target_tasks_method')
-def filter_target_tasks(graph, parameters):
+def filter_target_tasks(graph, parameters, graph_config):
"""Proxy filter to use legacy target tasks code.
This should go away once target_tasks are converted to filters.
"""
attr = parameters.get('target_tasks_method', 'all_tasks')
fn = target_tasks.get_method(attr)
- return fn(graph, parameters)
+ return fn(graph, parameters, graph_config)
@filter_task('check_servo')
-def filter_servo(graph, parameters):
+def filter_servo(graph, parameters, graph_config):
"""Filter out tasks for Servo vendoring changesets.
If the change triggering is related to Servo vendoring, impact is minimal
because not all platforms use Servo code.
We filter out tests on platforms that don't run Servo tests because running
tests will accomplish little for these changes.
"""
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -253,17 +253,17 @@ class TaskGraphGenerator(object):
len(full_task_set.graph.nodes), len(edges)))
yield verifications('full_task_graph', full_task_graph)
logger.info("Generating target task set")
target_task_set = TaskGraph(dict(all_tasks),
Graph(set(all_tasks.keys()), set()))
for fltr in self.filters:
old_len = len(target_task_set.graph.nodes)
- target_tasks = set(fltr(target_task_set, self.parameters))
+ target_tasks = set(fltr(target_task_set, self.parameters, graph_config))
target_task_set = TaskGraph(
{l: all_tasks[l] for l in target_tasks},
Graph(target_tasks, set()))
logger.info('Filter %s pruned %d tasks (%d remain)' % (
fltr.__name__,
old_len - len(target_tasks),
len(target_tasks)))
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -88,25 +88,25 @@ def filter_beta_release_tasks(task, para
def standard_filter(task, parameters):
return all(
filter_func(task, parameters) for filter_func in
(filter_on_nightly, filter_for_project, filter_upload_symbols)
)
-def _try_task_config(full_task_graph, parameters):
+def _try_task_config(full_task_graph, parameters, graph_config):
requested_tasks = parameters['try_task_config']['tasks']
return list(set(requested_tasks) & full_task_graph.graph.nodes)
-def _try_option_syntax(full_task_graph, parameters):
+def _try_option_syntax(full_task_graph, parameters, graph_config):
"""Generate a list of target tasks based on try syntax in
parameters['message'] and, for context, the full task graph."""
- options = try_option_syntax.TryOptionSyntax(parameters, full_task_graph)
+ options = try_option_syntax.TryOptionSyntax(parameters, full_task_graph, graph_config)
target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues()
if options.task_matches(t)]
attributes = {
k: getattr(options, k) for k in [
'env',
'no_retry',
'tag',
@@ -142,38 +142,38 @@ def _try_option_syntax(full_task_graph,
elif options.notifications == 'failure':
routes.append("notify.email.{}.on-failed".format(owner))
routes.append("notify.email.{}.on-exception".format(owner))
return target_tasks_labels
@_target_task('try_tasks')
-def target_tasks_try(full_task_graph, parameters):
+def target_tasks_try(full_task_graph, parameters, graph_config):
try_mode = parameters['try_mode']
if try_mode == 'try_task_config':
- return _try_task_config(full_task_graph, parameters)
+ return _try_task_config(full_task_graph, parameters, graph_config)
elif try_mode == 'try_option_syntax':
- return _try_option_syntax(full_task_graph, parameters)
+ return _try_option_syntax(full_task_graph, parameters, graph_config)
else:
# With no try mode, we schedule nothing, allowing the user to add tasks
# later via treeherder.
return []
@_target_task('default')
-def target_tasks_default(full_task_graph, parameters):
+def target_tasks_default(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
return [l for l, t in full_task_graph.tasks.iteritems()
if standard_filter(t, parameters)]
@_target_task('ash_tasks')
-def target_tasks_ash(full_task_graph, parameters):
+def target_tasks_ash(full_task_graph, parameters, graph_config):
"""Target tasks that only run on the ash branch."""
def filter(task):
platform = task.attributes.get('build_platform')
# Early return if platform is None
if not platform:
return False
# Only on Linux platforms
if 'linux' not in platform:
@@ -199,64 +199,64 @@ def target_tasks_ash(full_task_graph, pa
# don't upload symbols
if task.attributes['kind'] == 'upload-symbols':
return False
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('cedar_tasks')
-def target_tasks_cedar(full_task_graph, parameters):
+def target_tasks_cedar(full_task_graph, parameters, graph_config):
"""Target tasks that only run on the cedar branch."""
def filter(task):
platform = task.attributes.get('build_platform')
# only select platforms
if platform not in ('linux64', 'macosx64'):
return False
if task.attributes.get('unittest_suite'):
if not (task.attributes['unittest_suite'].startswith('mochitest') or
'xpcshell' in task.attributes['unittest_suite']):
return False
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('graphics_tasks')
-def target_tasks_graphics(full_task_graph, parameters):
+def target_tasks_graphics(full_task_graph, parameters, graph_config):
"""In addition to doing the filtering by project that the 'default'
filter does, also remove artifact builds because we have csets on
the graphics branch that aren't on the candidate branches of artifact
builds"""
filtered_for_project = target_tasks_default(full_task_graph, parameters)
def filter(task):
if task.attributes['kind'] == 'artifact-build':
return False
return True
return [l for l in filtered_for_project if filter(full_task_graph[l])]
@_target_task('mochitest_valgrind')
-def target_tasks_valgrind(full_task_graph, parameters):
+def target_tasks_valgrind(full_task_graph, parameters, graph_config):
"""Target tasks that only run on the cedar branch."""
def filter(task):
platform = task.attributes.get('test_platform', '').split('/')[0]
if platform not in ['linux64']:
return False
if task.attributes.get('unittest_suite', '').startswith('mochitest') and \
task.attributes.get('unittest_flavor', '').startswith('valgrind-plain'):
return True
return False
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_fennec')
-def target_tasks_nightly_fennec(full_task_graph, parameters):
+def target_tasks_nightly_fennec(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of fennec. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
def filter(task):
platform = task.attributes.get('build_platform')
if platform in ('android-aarch64-nightly',
'android-api-16-nightly',
'android-api-16-old-id-nightly',
@@ -265,51 +265,51 @@ def target_tasks_nightly_fennec(full_tas
'android-x86-old-id-nightly'):
if not task.attributes.get('nightly', False):
return False
return filter_for_project(task, parameters)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_linux')
-def target_tasks_nightly_linux(full_task_graph, parameters):
+def target_tasks_nightly_linux(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of linux. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
def filter(task):
platform = task.attributes.get('build_platform')
if platform in ('linux64-nightly', 'linux-nightly'):
return task.attributes.get('nightly', False)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('mozilla_beta_tasks')
-def target_tasks_mozilla_beta(full_task_graph, parameters):
+def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a promotable beta or release build
of desktop, plus android CI. The candidates build process involves a pipeline
of builds and signing, but does not include beetmover or balrog jobs."""
return [l for l, t in full_task_graph.tasks.iteritems() if
filter_beta_release_tasks(t, parameters)]
@_target_task('mozilla_release_tasks')
-def target_tasks_mozilla_release(full_task_graph, parameters):
+def target_tasks_mozilla_release(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a promotable beta or release build
of desktop, plus android CI. The candidates build process involves a pipeline
of builds and signing, but does not include beetmover or balrog jobs."""
return [l for l, t in full_task_graph.tasks.iteritems() if
filter_beta_release_tasks(t, parameters)]
@_target_task('maple_desktop_promotion')
@_target_task('mozilla-beta_desktop_promotion')
@_target_task('mozilla-release_desktop_promotion')
-def target_tasks_mozilla_beta_desktop_promotion(full_task_graph, parameters):
+def target_tasks_mozilla_beta_desktop_promotion(full_task_graph, parameters, graph_config):
"""Select the superset of tasks required to promote a beta or release build
of desktop. This should include all non-android mozilla_beta tasks, plus
l10n, beetmover, balrog, etc."""
beta_tasks = [l for l, t in full_task_graph.tasks.iteritems() if
filter_beta_release_tasks(t, parameters,
ignore_kinds=[],
allow_l10n=True)]
@@ -344,17 +344,17 @@ def target_tasks_mozilla_beta_desktop_pr
# TODO: bouncer sub
# TODO: snap
# TODO: recompression tasks
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('publish_firefox')
-def target_tasks_publish_firefox(full_task_graph, parameters):
+def target_tasks_publish_firefox(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to publish a candidates build of firefox.
Previous build deps will be optimized out via action task."""
filtered_for_candidates = target_tasks_mozilla_beta_desktop_promotion(
full_task_graph, parameters
)
def filter(task):
# Include promotion tasks; these will be optimized out
@@ -370,17 +370,17 @@ def target_tasks_publish_firefox(full_ta
# TODO: bouncer aliases
# TODO: checksums
# TODO: shipit mark as shipped
return [l for l, t in full_task_graph.iteritems() if filter(t)]
@_target_task('candidates_fennec')
-def target_tasks_candidates_fennec(full_task_graph, parameters):
+def target_tasks_candidates_fennec(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a candidates build of fennec. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
filtered_for_project = target_tasks_nightly_fennec(full_task_graph, parameters)
def filter(task):
attr = task.attributes.get
# Don't ship single locale fennec anymore - Bug 1408083
@@ -398,17 +398,17 @@ def target_tasks_candidates_fennec(full_
if task.kind in ('release-notify-promote',
):
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])]
@_target_task('publish_fennec')
-def target_tasks_publish_fennec(full_task_graph, parameters):
+def target_tasks_publish_fennec(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to publish a candidates build of fennec.
Previous build deps will be optimized out via action task."""
filtered_for_candidates = target_tasks_candidates_fennec(full_task_graph, parameters)
def filter(task):
# Include candidates build tasks; these will be optimized out
if task.label in filtered_for_candidates:
return True
@@ -430,95 +430,95 @@ def target_tasks_publish_fennec(full_tas
if task.kind in ('push-apk', 'push-apk-breakpoint'):
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])]
@_target_task('pine_tasks')
-def target_tasks_pine(full_task_graph, parameters):
+def target_tasks_pine(full_task_graph, parameters, graph_config):
"""Bug 1339179 - no mobile automation needed on pine"""
def filter(task):
platform = task.attributes.get('build_platform')
# disable mobile jobs
if str(platform).startswith('android'):
return False
# disable asan
if platform == 'linux64-asan':
return False
# disable non-pine and nightly tasks
if standard_filter(task, parameters):
return True
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_macosx')
-def target_tasks_nightly_macosx(full_task_graph, parameters):
+def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of macosx. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
def filter(task):
platform = task.attributes.get('build_platform')
if platform in ('macosx64-nightly', ):
return task.attributes.get('nightly', False)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_win32')
-def target_tasks_nightly_win32(full_task_graph, parameters):
+def target_tasks_nightly_win32(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
def filter(task):
platform = task.attributes.get('build_platform')
if not filter_for_project(task, parameters):
return False
if platform in ('win32-nightly', ):
return task.attributes.get('nightly', False)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_win64')
-def target_tasks_nightly_win64(full_task_graph, parameters):
+def target_tasks_nightly_win64(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
def filter(task):
platform = task.attributes.get('build_platform')
if not filter_for_project(task, parameters):
return False
if platform in ('win64-nightly', ):
return task.attributes.get('nightly', False)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('nightly_desktop')
-def target_tasks_nightly_desktop(full_task_graph, parameters):
+def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of linux, mac,
windows."""
# Avoid duplicate tasks.
return list(
set(target_tasks_nightly_win32(full_task_graph, parameters))
| set(target_tasks_nightly_win64(full_task_graph, parameters))
| set(target_tasks_nightly_macosx(full_task_graph, parameters))
| set(target_tasks_nightly_linux(full_task_graph, parameters))
)
# Opt DMD builds should only run nightly
@_target_task('nightly_dmd')
-def target_tasks_dmd(full_task_graph, parameters):
+def target_tasks_dmd(full_task_graph, parameters, graph_config):
"""Target DMD that run nightly on the m-c branch."""
def filter(task):
platform = task.attributes.get('build_platform', '')
return platform.endswith('-dmd')
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@_target_task('file_update')
-def target_tasks_file_update(full_task_graph, parameters):
+def target_tasks_file_update(full_task_graph, parameters, graph_config):
"""Select the set of tasks required to perform nightly in-tree file updates
"""
def filter(task):
# For now any task in the repo-update kind is ok
return task.kind in ['repo-update']
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -43,24 +43,24 @@ class FakeKind(Kind):
def load_tasks(self, parameters, loaded_tasks):
FakeKind.loaded_kinds.append(self.name)
return super(FakeKind, self).load_tasks(parameters, loaded_tasks)
class WithFakeKind(TaskGraphGenerator):
- def _load_kinds(self):
+ def _load_kinds(self, graph_config):
for kind_name, cfg in self.parameters['_kinds']:
config = {
'transforms': [],
}
if cfg:
config.update(cfg)
- yield FakeKind(kind_name, '/fake', config)
+ yield FakeKind(kind_name, '/fake', config, graph_config)
def _load_graph_config(self):
return {}
class FakeParameters(dict):
strict = True
@@ -86,17 +86,17 @@ class TestGenerator(unittest.TestCase):
def patch(self, monkeypatch):
self.patch = monkeypatch
def maketgg(self, target_tasks=None, kinds=[('_fake', [])], params=None):
params = params or {}
FakeKind.loaded_kinds = []
self.target_tasks = target_tasks or []
- def target_tasks_method(full_task_graph, parameters):
+ def target_tasks_method(full_task_graph, parameters, graph_config):
return self.target_tasks
def make_fake_strategies():
return {mode: FakeOptimization(mode)
for mode in ('always', 'never', 'even', 'odd')}
target_tasks_mod._target_task_methods['test_method'] = target_tasks_method
self.patch.setattr(optimize_mod, '_make_default_strategies', make_fake_strategies)
--- a/taskcluster/taskgraph/test/test_target_tasks.py
+++ b/taskcluster/taskgraph/test/test_target_tasks.py
@@ -12,17 +12,17 @@ from taskgraph import try_option_syntax
from taskgraph.graph import Graph
from taskgraph.taskgraph import TaskGraph
from taskgraph.task import Task
from mozunit import main
class FakeTryOptionSyntax(object):
- def __init__(self, message, task_graph):
+ def __init__(self, message, task_graph, graph_config):
self.trigger_tests = 0
self.talos_trigger_tests = 0
self.notifications = None
self.env = []
self.profile = False
self.tag = None
self.no_retry = False
@@ -35,17 +35,17 @@ class TestTargetTasks(unittest.TestCase)
def default_matches(self, run_on_projects, project):
method = target_tasks.get_method('default')
graph = TaskGraph(tasks={
'a': Task(kind='build', label='a',
attributes={'run_on_projects': run_on_projects},
task={}),
}, graph=Graph(nodes={'a'}, edges=set()))
parameters = {'project': project}
- return 'a' in method(graph, parameters)
+ return 'a' in method(graph, parameters, {})
def test_default_all(self):
"""run_on_projects=[all] includes release, integration, and other projects"""
self.assertTrue(self.default_matches(['all'], 'mozilla-central'))
self.assertTrue(self.default_matches(['all'], 'mozilla-inbound'))
self.assertTrue(self.default_matches(['all'], 'baobab'))
def test_default_integration(self):
@@ -89,34 +89,34 @@ class TestTargetTasks(unittest.TestCase)
tg = self.make_task_graph()
method = target_tasks.get_method('try_tasks')
params = {
'try_mode': None,
'project': 'try',
'message': '',
}
# only runs the task with run_on_projects: try
- self.assertEqual(method(tg, params), [])
+ self.assertEqual(method(tg, params, {}), [])
def test_try_option_syntax(self):
"try_mode = try_option_syntax uses TryOptionSyntax"
tg = self.make_task_graph()
method = target_tasks.get_method('try_tasks')
with self.fake_TryOptionSyntax():
params = {
'try_mode': 'try_option_syntax',
'message': 'try: -p all',
}
- self.assertEqual(method(tg, params), ['b'])
+ self.assertEqual(method(tg, params, {}), ['b'])
def test_try_task_config(self):
"try_mode = try_task_config uses the try config"
tg = self.make_task_graph()
method = target_tasks.get_method('try_tasks')
params = {
'try_mode': 'try_task_config',
'try_task_config': {'tasks': ['a']},
}
- self.assertEqual(method(tg, params), ['a'])
+ self.assertEqual(method(tg, params, {}), ['a'])
if __name__ == '__main__':
main()
--- a/taskcluster/taskgraph/test/test_try_option_syntax.py
+++ b/taskcluster/taskgraph/test/test_try_option_syntax.py
@@ -2,17 +2,16 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import unittest
from taskgraph.try_option_syntax import TryOptionSyntax, parse_message
-from taskgraph.try_option_syntax import RIDEALONG_BUILDS
from taskgraph.graph import Graph
from taskgraph.taskgraph import TaskGraph
from taskgraph.task import Task
from mozunit import main
def unittest_task(n, tp, bt='opt'):
return (n, Task('test', n, {
@@ -44,16 +43,25 @@ tasks = {k: v for k, v in [
unittest_task('extra4', 'linux64/debug', 'debug'),
unittest_task('extra5', 'linux/this'),
unittest_task('extra6', 'linux/that'),
unittest_task('extra7', 'linux/other'),
unittest_task('extra8', 'linux64/asan'),
talos_task('extra9', 'linux64/psan'),
]}
+RIDEALONG_BUILDS = {
+ 'linux': ['linux-ridealong'],
+ 'linux64': ['linux64-ridealong'],
+}
+
+GRAPH_CONFIG = {
+ 'try': {'ridealong-builds': RIDEALONG_BUILDS},
+}
+
for r in RIDEALONG_BUILDS.values():
tasks.update({k: v for k, v in [
unittest_task(n + '-test', n) for n in r
]})
unittest_tasks = {k: v for k, v in tasks.iteritems()
if 'unittest_try_name' in v.attributes}
talos_tasks = {k: v for k, v in tasks.iteritems()
@@ -61,283 +69,283 @@ talos_tasks = {k: v for k, v in tasks.it
graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
class TestTryOptionSyntax(unittest.TestCase):
def test_unknown_args(self):
"unknown arguments are ignored"
parameters = {'try_options': parse_message('try: --doubledash -z extra')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
# equilvant to "try:"..
self.assertEqual(tos.build_types, [])
self.assertEqual(tos.jobs, [])
def test_apostrophe_in_message(self):
"apostrophe does not break parsing"
parameters = {'try_options': parse_message('Increase spammy log\'s log level. try: -b do')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.build_types), ['debug', 'opt'])
def test_b_do(self):
"-b do should produce both build_types"
parameters = {'try_options': parse_message('try: -b do')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.build_types), ['debug', 'opt'])
def test_b_d(self):
"-b d should produce build_types=['debug']"
parameters = {'try_options': parse_message('try: -b d')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.build_types), ['debug'])
def test_b_o(self):
"-b o should produce build_types=['opt']"
parameters = {'try_options': parse_message('try: -b o')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.build_types), ['opt'])
def test_build_o(self):
"--build o should produce build_types=['opt']"
parameters = {'try_options': parse_message('try: --build o')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.build_types), ['opt'])
def test_b_dx(self):
"-b dx should produce build_types=['debug'], silently ignoring the x"
parameters = {'try_options': parse_message('try: -b dx')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.build_types), ['debug'])
def test_j_job(self):
"-j somejob sets jobs=['somejob']"
parameters = {'try_options': parse_message('try: -j somejob')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.jobs), ['somejob'])
def test_j_jobs(self):
"-j job1,job2 sets jobs=['job1', 'job2']"
parameters = {'try_options': parse_message('try: -j job1,job2')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.jobs), ['job1', 'job2'])
def test_j_all(self):
"-j all sets jobs=None"
parameters = {'try_options': parse_message('try: -j all')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.jobs, None)
def test_j_twice(self):
"-j job1 -j job2 sets jobs=job1, job2"
parameters = {'try_options': parse_message('try: -j job1 -j job2')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.jobs), sorted(['job1', 'job2']))
def test_p_all(self):
"-p all sets platforms=None"
parameters = {'try_options': parse_message('try: -p all')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.platforms, None)
def test_p_linux(self):
- "-p linux sets platforms=['linux', 'linux-l10n']"
+ "-p linux sets platforms=['linux', 'linux-ridealong']"
parameters = {'try_options': parse_message('try: -p linux')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
- self.assertEqual(tos.platforms, ['linux', 'linux-l10n'])
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(tos.platforms, ['linux', 'linux-ridealong'])
def test_p_linux_win32(self):
- "-p linux,win32 sets platforms=['linux', 'linux-l10n', 'win32']"
+ "-p linux,win32 sets platforms=['linux', 'linux-ridealong', 'win32']"
parameters = {'try_options': parse_message('try: -p linux,win32')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
- self.assertEqual(sorted(tos.platforms), ['linux', 'linux-l10n', 'win32'])
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
+ self.assertEqual(sorted(tos.platforms), ['linux', 'linux-ridealong', 'win32'])
def test_p_expands_ridealongs(self):
"-p linux,linux64 includes the RIDEALONG_BUILDS"
parameters = {'try_options': parse_message('try: -p linux,linux64')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
platforms = set(['linux'] + RIDEALONG_BUILDS['linux'])
platforms |= set(['linux64'] + RIDEALONG_BUILDS['linux64'])
self.assertEqual(sorted(tos.platforms), sorted(platforms))
def test_u_none(self):
"-u none sets unittests=[]"
parameters = {'try_options': parse_message('try: -u none')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), [])
def test_u_all(self):
"-u all sets unittests=[..whole list..]"
parameters = {'try_options': parse_message('try: -u all')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([{'test': t} for t in unittest_tasks]))
def test_u_single(self):
"-u mochitest-webgl sets unittests=[mochitest-webgl]"
parameters = {'try_options': parse_message('try: -u mochitest-webgl')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
def test_u_alias(self):
"-u mochitest-gl sets unittests=[mochitest-webgl]"
parameters = {'try_options': parse_message('try: -u mochitest-gl')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
def test_u_multi_alias(self):
"-u e10s sets unittests=[all e10s unittests]"
parameters = {'try_options': parse_message('try: -u e10s')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': t} for t in unittest_tasks if 'e10s' in t
]))
def test_u_commas(self):
"-u mochitest-webgl,gtest sets unittests=both"
parameters = {'try_options': parse_message('try: -u mochitest-webgl,gtest')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'mochitest-webgl'},
{'test': 'gtest'},
]))
def test_u_chunks(self):
"-u gtest-3,gtest-4 selects the third and fourth chunk of gtest"
parameters = {'try_options': parse_message('try: -u gtest-3,gtest-4')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'only_chunks': set('34')},
]))
def test_u_platform(self):
"-u gtest[linux] selects the linux platform for gtest"
parameters = {'try_options': parse_message('try: -u gtest[linux]')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux']},
]))
def test_u_platforms(self):
"-u gtest[linux,win32] selects the linux and win32 platforms for gtest"
parameters = {'try_options': parse_message('try: -u gtest[linux,win32]')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux', 'win32']},
]))
def test_u_platforms_pretty(self):
"""-u gtest[Ubuntu] selects the linux, linux64, linux64-asan, linux64-stylo-disabled,
and linux64-stylo-sequential platforms for gtest"""
parameters = {'try_options': parse_message('try: -u gtest[Ubuntu]')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux32', 'linux64', 'linux64-asan',
'linux64-stylo-disabled', 'linux64-stylo-sequential']},
]))
def test_u_platforms_negated(self):
"-u gtest[-linux] selects all platforms but linux for gtest"
parameters = {'try_options': parse_message('try: -u gtest[-linux]')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
all_platforms = set([x.attributes['test_platform'] for x in unittest_tasks.values()])
self.assertEqual(sorted(tos.unittests[0]['platforms']), sorted(
[x for x in all_platforms if x != 'linux']
))
def test_u_platforms_negated_pretty(self):
"-u gtest[Ubuntu,-x64] selects just linux for gtest"
parameters = {'try_options': parse_message('try: -u gtest[Ubuntu,-x64]')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux32']},
]))
def test_u_chunks_platforms(self):
"-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest"
parameters = {'try_options': parse_message('try: -u gtest-1[linux,win32]')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux', 'win32'], 'only_chunks': set('1')},
]))
def test_t_none(self):
"-t none sets talos=[]"
parameters = {'try_options': parse_message('try: -t none')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.talos), [])
def test_t_all(self):
"-t all sets talos=[..whole list..]"
parameters = {'try_options': parse_message('try: -t all')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.talos), sorted([{'test': t} for t in talos_tasks]))
def test_t_single(self):
"-t mochitest-webgl sets talos=[mochitest-webgl]"
parameters = {'try_options': parse_message('try: -t mochitest-webgl')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(sorted(tos.talos), sorted([{'test': 'mochitest-webgl'}]))
# -t shares an implementation with -u, so it's not tested heavily
def test_trigger_tests(self):
"--rebuild 10 sets trigger_tests"
parameters = {'try_options': parse_message('try: --rebuild 10')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.trigger_tests, 10)
def test_talos_trigger_tests(self):
"--rebuild-talos 10 sets talos_trigger_tests"
parameters = {'try_options': parse_message('try: --rebuild-talos 10')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.talos_trigger_tests, 10)
def test_interactive(self):
"--interactive sets interactive"
parameters = {'try_options': parse_message('try: --interactive')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.interactive, True)
def test_all_email(self):
"--all-emails sets notifications"
parameters = {'try_options': parse_message('try: --all-emails')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.notifications, 'all')
def test_fail_email(self):
"--failure-emails sets notifications"
parameters = {'try_options': parse_message('try: --failure-emails')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.notifications, 'failure')
def test_no_email(self):
"no email settings don't set notifications"
parameters = {'try_options': parse_message('try:')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.notifications, None)
def test_setenv(self):
"--setenv VAR=value adds a environment variables setting to env"
parameters = {'try_options': parse_message(
'try: --setenv VAR1=value1 --setenv VAR2=value2')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.env, ['VAR1=value1', 'VAR2=value2'])
def test_profile(self):
"--geckoProfile sets profile to true"
parameters = {'try_options': parse_message('try: --geckoProfile')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertTrue(tos.profile)
def test_tag(self):
"--tag TAG sets tag to TAG value"
parameters = {'try_options': parse_message('try: --tag tagName')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertEqual(tos.tag, 'tagName')
def test_no_retry(self):
"--no-retry sets no_retry to true"
parameters = {'try_options': parse_message('try: --no-retry')}
- tos = TryOptionSyntax(parameters, graph_with_jobs)
+ tos = TryOptionSyntax(parameters, graph_with_jobs, GRAPH_CONFIG)
self.assertTrue(tos.no_retry)
if __name__ == '__main__':
main()
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -147,45 +147,16 @@ UNITTEST_PLATFORM_PRETTY_NAMES = {
'Windows 7 VM': ['windows7-32-vm'],
'Windows 8': ['windows8-64'],
'Windows 10': ['windows10-64'],
# 'Windows XP': [..TODO..],
# 'win32': [..TODO..],
# 'win64': [..TODO..],
}
-# We have a few platforms for which we want to do some "extra" builds, or at
-# least build-ish things. Sort of. Anyway, these other things are implemented
-# as different "platforms". These do *not* automatically ride along with "-p
-# all"
-RIDEALONG_BUILDS = {
- 'android-api-16': [
- 'android-api-16-l10n',
- ],
- 'linux': [
- 'linux-l10n',
- ],
- 'linux64': [
- 'linux64-l10n',
- 'sm-plain',
- 'sm-nonunified',
- 'sm-arm-sim',
- 'sm-arm64-sim',
- 'sm-compacting',
- 'sm-rootanalysis',
- 'sm-package',
- 'sm-tsan',
- 'sm-asan',
- 'sm-mozjs-sys',
- 'sm-msan',
- 'sm-fuzzing',
- 'sm-rust-bindings',
- ],
-}
-
TEST_CHUNK_SUFFIX = re.compile('(.*)-([0-9]+)$')
def escape_whitespace_in_brackets(input_str):
'''
In tests you may restrict them by platform [] inside of the brackets
whitespace may occur this is typically invalid shell syntax so we escape it
with backslash sequences .
@@ -257,17 +228,17 @@ def parse_message(message):
# In order to run test jobs multiple times
parser.add_argument('--rebuild', dest='trigger_tests', type=int, default=1)
args, _ = parser.parse_known_args(parts)
return vars(args)
class TryOptionSyntax(object):
- def __init__(self, parameters, full_task_graph):
+ def __init__(self, parameters, full_task_graph, graph_config):
"""
Apply the try options in parameters.
The resulting object has attributes:
- build_types: a list containing zero or more of 'opt' and 'debug'
- platforms: a list of selected platform names, or None for all
- unittests: a list of tests, of the form given below, or None for all
@@ -284,16 +255,17 @@ class TryOptionSyntax(object):
The unittests and talos lists contain dictionaries of the form:
{
'test': '<suite name>',
'platforms': [..platform names..], # to limit to only certain platforms
'only_chunks': set([..chunk numbers..]), # to limit only to certain chunks
}
"""
+ self.graph_config = graph_config
self.jobs = []
self.build_types = []
self.platforms = []
self.unittests = []
self.talos = []
self.trigger_tests = 0
self.interactive = False
self.notifications = None
@@ -347,16 +319,17 @@ class TryOptionSyntax(object):
raise Exception("Unknown build type(s) [%s] specified for try" % ','.join(bad_types))
return build_types
def parse_platforms(self, platform_arg, full_task_graph):
if platform_arg == 'all':
return None
+ RIDEALONG_BUILDS = self.graph_config['try']['ridealong-builds']
results = []
for build in platform_arg.split(','):
results.append(build)
if build in RIDEALONG_BUILDS:
results.extend(RIDEALONG_BUILDS[build])
logger.info("platform %s triggers ridealong builds %s" %
(build, ', '.join(RIDEALONG_BUILDS[build])))
