--- a/taskcluster/ci/build/kind.yml
+++ b/taskcluster/ci/build/kind.yml
@@ -16,10 +16,8 @@ transforms:
- taskgraph.transforms.task:transforms
jobs-from:
- android.yml
- android-stuff.yml
- linux.yml
- macosx.yml
- windows.yml
-
-parse-commit: taskgraph.try_option_syntax:parse_message
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -4,10 +4,8 @@ kind-dependencies:
- build
- build-signing
transforms:
- taskgraph.transforms.tests:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.coalesce:transforms
- taskgraph.transforms.task:transforms
-
-parse-commit: taskgraph.try_option_syntax:parse_message
--- a/taskcluster/docs/loading.rst
+++ b/taskcluster/docs/loading.rst
@@ -27,17 +27,8 @@ The return value is a list of inputs to
``transforms`` property. The specific format for the input depends on the first
transform - whatever it expects. The final transform should be
``taskgraph.transform.task:transforms``, which produces the output format the
task-graph generation infrastructure expects.
The ``transforms`` key in ``kind.yml`` is further documented in
:doc:`transforms`. For more information on how all of this works, consult the
docstrings and comments in the source code itself.
-
-Try option syntax
------------------
-
-The ``parse-commit`` optional field specified in ``kind.yml`` links to a
-function to parse the command line options in the ``--message`` mach parameter.
-Currently, the only valid value is ``taskgraph.try_option_syntax:parse_message``.
-The parsed arguments are stored in ``config.config['args']``, it corresponds
-to the same object returned by ``parse_args`` from ``argparse`` Python module.
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -74,50 +74,53 @@ Tree Information
``cedar``.
``level``
The `SCM level
<https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/>`_
associated with this tree. This dictates the names of resources used in the
generated tasks, and those tasks will fail if it is incorrect.
+Try Configuration
+-----------------
+
+``try_mode``
+ The mode in which a try push is operating. This can be one of
+ ``"try_task_config"``, ``"try_option_syntax"``, or ``None`` meaning no try
+ input was provided.
+
+``try_options``
+ The arguments given as try syntax (as a dictionary), or ``None`` if
+ ``try_mode`` is not ``try_option_syntax``.
+
+``try_task_config``
+ The contents of the ``try_task_config.json`` file, or ``None`` if
+ ``try_mode`` is not ``try_task_config``.
+
Target Set
----------
The "target set" is the set of task labels which must be included in a task
graph. The task graph generation process will include any tasks required by
those in the target set, recursively. In a decision task, this set can be
specified programmatically using one of a variety of methods (e.g., parsing try
syntax or reading a project-specific configuration file).
``filters``
List of filter functions (from ``taskcluster/taskgraph/filter_tasks.py``) to
apply. This is usually defined internally, as filters are typically
global.
-``target_task_labels``
- List of task labels to select. Labels not listed will be filtered out.
- Enabled on try only.
-
``target_tasks_method``
The method to use to determine the target task set. This is the suffix of
one of the functions in ``taskcluster/taskgraph/target_tasks.py``.
``optimize_target_tasks``
If true, then target tasks are eligible for optimization.
``include_nightly``
If true, then nightly tasks are eligible for optimization.
``release_history``
History of recent releases by platform and locale, used when generating
partial updates for nightly releases.
Suitable contents can be generated with ``mach release-history``,
which will print to the console by default.
-
-Morphed Set
------------
-
-``morph_templates``
- Dict of JSON-e templates to apply to each task, keyed by template name.
- Values are extra context that will be available to the template under the
- ``input.<template>`` key. Available templates live in
- ``taskcluster/taskgraph/templates``. Enabled on try only.
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -12,16 +12,17 @@ import re
import time
import yaml
from .generator import TaskGraphGenerator
from .create import create_tasks
from .parameters import Parameters
from .taskgraph import TaskGraph
+from .try_option_syntax import parse_message
from .actions import render_actions_json
from taskgraph.util.partials import populate_release_history
from . import GECKO
from taskgraph.util.templates import Templates
from taskgraph.util.time import (
json_time_from_now,
current_json_time,
@@ -31,20 +32,16 @@ logger = logging.getLogger(__name__)
ARTIFACTS_DIR = 'artifacts'
# For each project, this gives a set of parameters specific to the project.
# See `taskcluster/docs/parameters.rst` for information on parameters.
PER_PROJECT_PARAMETERS = {
'try': {
'target_tasks_method': 'try_tasks',
- # Always perform optimization. This makes it difficult to use try
- # pushes to run a task that would otherwise be optimized, but is a
- # compromise to avoid essentially disabling optimization in try.
- 'optimize_target_tasks': True,
# By default, the `try_option_syntax` `target_task_method` ignores this
# parameter, and enables/disables nightlies depending whether
# `--include-nightly` is specified in the commit message.
# We're setting the `include_nightly` parameter to True here for when
# we submit decision tasks against Try that use other
# `target_task_method`s, like `nightly_fennec` or `mozilla_beta_tasks`,
# which reference the `include_nightly` parameter.
'include_nightly': True,
@@ -163,18 +160,16 @@ def get_decision_parameters(options):
] if n in options}
# Define default filter list, as most configurations shouldn't need
# custom filters.
parameters['filters'] = [
'check_servo',
'target_tasks_method',
]
- parameters['target_task_labels'] = []
- parameters['morph_templates'] = {}
# owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
# case, fake it
if '@' not in parameters['owner']:
parameters['owner'] += '@noreply.mozilla.org'
# use the pushdate as build_date if given, else use current time
parameters['build_date'] = parameters['pushdate'] or int(time.time())
@@ -186,36 +181,63 @@ def get_decision_parameters(options):
try:
parameters.update(PER_PROJECT_PARAMETERS[project])
except KeyError:
logger.warning("using default project parameters; add {} to "
"PER_PROJECT_PARAMETERS in {} to customize behavior "
"for this project".format(project, __file__))
parameters.update(PER_PROJECT_PARAMETERS['default'])
- # morph_templates and target_task_labels are only used on try, so don't
- # bother loading them elsewhere
- task_config_file = os.path.join(GECKO, 'try_task_config.json')
- if project == 'try' and os.path.isfile(task_config_file):
- with open(task_config_file, 'r') as fh:
- task_config = json.load(fh)
- parameters['morph_templates'] = task_config.get('templates', {})
- parameters['target_task_labels'] = task_config.get('tasks')
-
# `target_tasks_method` has higher precedence than `project` parameters
if options.get('target_tasks_method'):
parameters['target_tasks_method'] = options['target_tasks_method']
# If the target method is nightly, we should build partials. This means
# knowing what has been released previously.
# An empty release_history is fine, it just means no partials will be built
parameters.setdefault('release_history', dict())
if 'nightly' in parameters.get('target_tasks_method', ''):
parameters['release_history'] = populate_release_history('Firefox', project)
+ # if try_task_config.json is present, load it
+ task_config_file = os.path.join(os.getcwd(), 'try_task_config.json')
+
+ # load try settings
+ parameters['try_mode'] = None
+ if os.path.isfile(task_config_file):
+ parameters['try_mode'] = 'try_task_config'
+ with open(task_config_file, 'r') as fh:
+ parameters['try_task_config'] = json.load(fh)
+ else:
+ parameters['try_task_config'] = None
+
+ if 'try:' in parameters['message']:
+ parameters['try_mode'] = 'try_option_syntax'
+ args = parse_message(parameters['message'])
+ parameters['try_options'] = args
+ else:
+ parameters['try_options'] = None
+
+ parameters['optimize_target_tasks'] = {
+ # The user has explicitly requested a set of jobs, so run them all
+ # regardless of optimization. Their dependencies can be optimized,
+ # though.
+ 'try_task_config': False,
+
+ # Always perform optimization. This makes it difficult to use try
+ # pushes to run a task that would otherwise be optimized, but is a
+ # compromise to avoid essentially disabling optimization in try.
+ # to run tasks that would otherwise be optimized, ues try_task_config.
+ 'try_option_syntax': True,
+
+ # since no try jobs have been specified, the standard target task will
+ # be applied, and tasks should be optimized out of that.
+ None: True,
+ }[parameters['try_mode']]
+
return Parameters(parameters)
def write_artifact(filename, data):
logger.info('writing artifact file `{}`'.format(filename))
if not os.path.isdir(ARTIFACTS_DIR):
os.mkdir(ARTIFACTS_DIR)
path = os.path.join(ARTIFACTS_DIR, filename)
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -37,22 +37,16 @@ class Kind(object):
except KeyError:
raise KeyError("{!r} does not define `loader`".format(self.path))
return find_object(loader)
def load_tasks(self, parameters, loaded_tasks):
loader = self._get_loader()
config = copy.deepcopy(self.config)
- if 'parse-commit' in self.config:
- parse_commit = find_object(config['parse-commit'])
- config['args'] = parse_commit(parameters['message'])
- else:
- config['args'] = None
-
kind_dependencies = config.get('kind-dependencies', [])
kind_dependencies_tasks = [task for task in loaded_tasks
if task.kind in kind_dependencies]
inputs = loader(self.name, self.path, config, parameters, loaded_tasks)
transforms = TransformSequence()
for xform_path in config['transforms']:
--- a/taskcluster/taskgraph/morph.py
+++ b/taskcluster/taskgraph/morph.py
@@ -278,13 +278,15 @@ class apply_jsone_templates(object):
return taskgraph, label_to_taskid
def morph(taskgraph, label_to_taskid, parameters):
"""Apply all morphs"""
morphs = [
add_index_tasks,
add_s3_uploader_task,
- apply_jsone_templates(parameters.get('morph_templates')),
]
+ if parameters['try_mode'] == 'try_task_config':
+ morphs.append(apply_jsone_templates(parameters['try_task_config'].get('templates')))
+
for m in morphs:
taskgraph, label_to_taskid = m(taskgraph, label_to_taskid)
return taskgraph, label_to_taskid
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -16,41 +16,37 @@ PARAMETER_NAMES = set([
'build_date',
'filters',
'head_ref',
'head_repository',
'head_rev',
'include_nightly',
'level',
'message',
- 'morph_templates',
'moz_build_date',
'optimize_target_tasks',
'owner',
'project',
'pushdate',
'pushlog_id',
'release_history',
- 'target_task_labels',
'target_tasks_method',
-])
-
-TRY_ONLY_PARAMETERS = set([
- 'morph_templates',
- 'target_task_labels',
+ 'try_mode',
+ 'try_options',
+ 'try_task_config',
])
class Parameters(ReadOnlyDict):
"""An immutable dictionary with nicer KeyError messages on failure"""
def check(self):
names = set(self)
msg = []
- missing = PARAMETER_NAMES - TRY_ONLY_PARAMETERS - names
+ missing = PARAMETER_NAMES - names
if missing:
msg.append("missing parameters: " + ", ".join(missing))
extra = names - PARAMETER_NAMES
if extra:
msg.append("extra parameters: " + ", ".join(extra))
if msg:
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -1,22 +1,19 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-import os
-
from taskgraph import try_option_syntax
from taskgraph.util.attributes import match_run_on_projects
-here = os.path.abspath(os.path.dirname(__file__))
_target_task_methods = {}
def _target_task(name):
def wrap(func):
_target_task_methods[name] = func
return func
return wrap
@@ -48,27 +45,24 @@ def filter_upload_symbols(task, paramete
def standard_filter(task, parameters):
return all(
filter_func(task, parameters) for filter_func in
(filter_on_nightly, filter_for_project, filter_upload_symbols)
)
def _try_task_config(full_task_graph, parameters):
- if not parameters.get('target_task_labels'):
- return []
-
- return [t.label for t in full_task_graph.tasks.itervalues()
- if t.label in parameters['target_task_labels']]
+ requested_tasks = parameters['try_task_config']['tasks']
+ return list(set(requested_tasks) & full_task_graph.graph.nodes)
def _try_option_syntax(full_task_graph, parameters):
"""Generate a list of target tasks based on try syntax in
parameters['message'] and, for context, the full task graph."""
- options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph)
+ options = try_option_syntax.TryOptionSyntax(parameters, full_task_graph)
target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues()
if options.task_matches(t)]
attributes = {
k: getattr(options, k) for k in [
'env',
'no_retry',
'tag',
@@ -105,29 +99,33 @@ def _try_option_syntax(full_task_graph,
routes.append("notify.email.{}.on-failed".format(owner))
routes.append("notify.email.{}.on-exception".format(owner))
return target_tasks_labels
@_target_task('try_tasks')
def target_tasks_try(full_task_graph, parameters):
- labels = _try_task_config(full_task_graph, parameters)
-
- if 'try:' in parameters['message'] or not labels:
- labels.extend(_try_option_syntax(full_task_graph, parameters))
-
- return labels
+ try_mode = parameters['try_mode']
+ if try_mode == 'try_task_config':
+ return _try_task_config(full_task_graph, parameters)
+ elif try_mode == 'try_option_syntax':
+ return _try_option_syntax(full_task_graph, parameters)
+ else:
+ # With no try mode, we would like to schedule everything (following
+ # run_on_projects) and let optimization trim it down. But optimization
+ # isn't yet up to the task, so instead we use try_option_syntax with
+ # an empty message (which basically just schedules `-j`objs)
+ return _try_option_syntax(full_task_graph, parameters)
@_target_task('default')
def target_tasks_default(full_task_graph, parameters):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
-
return [l for l, t in full_task_graph.tasks.iteritems()
if standard_filter(t, parameters)]
@_target_task('ash_tasks')
def target_tasks_ash(full_task_graph, parameters):
"""Target tasks that only run on the ash branch."""
def filter(task):
--- a/taskcluster/taskgraph/test/test_decision.py
+++ b/taskcluster/taskgraph/test/test_decision.py
@@ -7,17 +7,17 @@ from __future__ import absolute_import,
import os
import json
import yaml
import shutil
import unittest
import tempfile
from taskgraph import decision
-from mozunit import main
+from mozunit import main, MockedOpen
class TestDecision(unittest.TestCase):
def test_write_artifact_json(self):
data = [{'some': 'data'}]
tmpdir = tempfile.mkdtemp()
try:
@@ -39,10 +39,58 @@ class TestDecision(unittest.TestCase):
with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.yml")) as f:
self.assertEqual(yaml.safe_load(f), data)
finally:
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
decision.ARTIFACTS_DIR = 'artifacts'
+class TestGetDecisionParameters(unittest.TestCase):
+
+ def setUp(self):
+ self.options = {
+ 'base_repository': 'https://hg.mozilla.org/mozilla-unified',
+ 'head_repository': 'https://hg.mozilla.org/mozilla-central',
+ 'head_rev': 'abcd',
+ 'head_ref': 'ef01',
+ 'message': '',
+ 'project': 'mozilla-central',
+ 'pushlog_id': 143,
+ 'pushdate': 1503691511,
+ 'owner': 'nobody@mozilla.com',
+ 'level': 3,
+ }
+
+ def test_simple_options(self):
+ params = decision.get_decision_parameters(self.options)
+ self.assertEqual(params['pushlog_id'], 143)
+ self.assertEqual(params['build_date'], 1503691511)
+ self.assertEqual(params['moz_build_date'], '20170825200511')
+ self.assertEqual(params['try_mode'], None)
+ self.assertEqual(params['try_options'], None)
+ self.assertEqual(params['try_task_config'], None)
+
+ def test_no_email_owner(self):
+ self.options['owner'] = 'ffxbld'
+ params = decision.get_decision_parameters(self.options)
+ self.assertEqual(params['owner'], 'ffxbld@noreply.mozilla.org')
+
+ def test_try_options(self):
+ self.options['message'] = 'try: -b do -t all'
+ params = decision.get_decision_parameters(self.options)
+ self.assertEqual(params['try_mode'], 'try_option_syntax')
+ self.assertEqual(params['try_options']['build_types'], 'do')
+ self.assertEqual(params['try_options']['unittests'], 'all')
+ self.assertEqual(params['try_task_config'], None)
+
+ def test_try_task_config(self):
+ ttc = {'tasks': ['a', 'b'], 'templates': {}}
+ ttc_file = os.path.join(os.getcwd(), 'try_task_config.json')
+ with MockedOpen({ttc_file: json.dumps(ttc)}):
+ params = decision.get_decision_parameters(self.options)
+ self.assertEqual(params['try_mode'], 'try_task_config')
+ self.assertEqual(params['try_options'], None)
+ self.assertEqual(params['try_task_config'], ttc)
+
+
if __name__ == '__main__':
main()
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -54,16 +54,17 @@ class TestGenerator(unittest.TestCase):
def target_tasks_method(full_task_graph, parameters):
return self.target_tasks
target_tasks_mod._target_task_methods['test_method'] = target_tasks_method
parameters = {
'_kinds': kinds,
'target_tasks_method': 'test_method',
+ 'try_mode': None,
}
return WithFakeKind('/root', parameters)
def test_kind_ordering(self):
"When task kinds depend on each other, they are loaded in postorder"
self.tgg = self.maketgg(kinds=[
('_fake3', ['_fake2', '_fake1']),
--- a/taskcluster/taskgraph/test/test_target_tasks.py
+++ b/taskcluster/taskgraph/test/test_target_tasks.py
@@ -1,14 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
+import contextlib
import unittest
from taskgraph import target_tasks
from taskgraph import try_option_syntax
from taskgraph.graph import Graph
from taskgraph.taskgraph import TaskGraph
from taskgraph.task import Task
from mozunit import main
@@ -60,48 +61,61 @@ class TestTargetTasks(unittest.TestCase)
self.assertFalse(self.default_matches(['release'], 'baobab'))
def test_default_nothing(self):
"""run_on_projects=[] includes nothing"""
self.assertFalse(self.default_matches([], 'mozilla-central'))
self.assertFalse(self.default_matches([], 'mozilla-inbound'))
self.assertFalse(self.default_matches([], 'baobab'))
- def test_try_tasks(self):
+ def make_task_graph(self):
tasks = {
'a': Task(kind=None, label='a', attributes={}, task={}),
'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}),
- 'c': Task(kind=None, label='c', attributes={}, task={}),
+ 'c': Task(kind=None, label='c', attributes={'run_on_projects': ['try']}, task={}),
}
graph = Graph(nodes=set('abc'), edges=set())
- tg = TaskGraph(tasks, graph)
+ return TaskGraph(tasks, graph)
- method = target_tasks.get_method('try_tasks')
- params = {
- 'message': '',
- 'target_task_labels': [],
- }
-
+ @contextlib.contextmanager
+ def fake_TryOptionSyntax(self):
orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
try:
try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
+ yield
+ finally:
+ try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
- # no try specifier
- self.assertEqual(method(tg, params), ['b'])
-
- # try syntax only
- params['message'] = 'try: me'
+ def test_just_try_it(self):
+ "try_mode = None runs try optoin syntax with no options"
+ tg = self.make_task_graph()
+ method = target_tasks.get_method('try_tasks')
+ with self.fake_TryOptionSyntax():
+ params = {
+ 'try_mode': None,
+ 'message': '',
+ }
self.assertEqual(method(tg, params), ['b'])
- # try task config only
- params['message'] = ''
- params['target_task_labels'] = ['c']
- self.assertEqual(method(tg, params), ['c'])
+ def test_try_option_syntax(self):
+ "try_mode = try_option_syntax uses TryOptionSyntax"
+ tg = self.make_task_graph()
+ method = target_tasks.get_method('try_tasks')
+ with self.fake_TryOptionSyntax():
+ params = {
+ 'try_mode': 'try_option_syntax',
+ 'message': 'try: -p all',
+ }
+ self.assertEqual(method(tg, params), ['b'])
- # both syntax and config
- params['message'] = 'try: me'
- self.assertEqual(set(method(tg, params)), set(['b', 'c']))
- finally:
- try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
+ def test_try_task_config(self):
+ "try_mode = try_task_config uses the try config"
+ tg = self.make_task_graph()
+ method = target_tasks.get_method('try_tasks')
+ params = {
+ 'try_mode': 'try_task_config',
+ 'try_task_config': {'tasks': ['a']},
+ }
+ self.assertEqual(method(tg, params), ['a'])
if __name__ == '__main__':
main()
--- a/taskcluster/taskgraph/test/test_try_option_syntax.py
+++ b/taskcluster/taskgraph/test/test_try_option_syntax.py
@@ -1,17 +1,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import unittest
-from taskgraph.try_option_syntax import TryOptionSyntax
+from taskgraph.try_option_syntax import TryOptionSyntax, parse_message
from taskgraph.try_option_syntax import RIDEALONG_BUILDS
from taskgraph.graph import Graph
from taskgraph.taskgraph import TaskGraph
from taskgraph.task import Task
from mozunit import main
def unittest_task(n, tp, bt='opt'):
@@ -58,274 +58,286 @@ unittest_tasks = {k: v for k, v in tasks
if 'unittest_try_name' in v.attributes}
talos_tasks = {k: v for k, v in tasks.iteritems()
if 'talos_try_name' in v.attributes}
graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
class TestTryOptionSyntax(unittest.TestCase):
- def test_empty_message(self):
- "Given an empty message, it should return an empty value"
- tos = TryOptionSyntax('', graph_with_jobs)
- self.assertEqual(tos.build_types, [])
- self.assertEqual(tos.jobs, [])
- self.assertEqual(tos.unittests, [])
- self.assertEqual(tos.talos, [])
- self.assertEqual(tos.platforms, [])
- self.assertEqual(tos.trigger_tests, 0)
- self.assertEqual(tos.talos_trigger_tests, 0)
- self.assertEqual(tos.env, [])
- self.assertFalse(tos.profile)
- self.assertIsNone(tos.tag)
- self.assertFalse(tos.no_retry)
-
- def test_message_without_try(self):
- "Given a non-try message, it should return an empty value"
- tos = TryOptionSyntax('Bug 1234: frobnicte the foo', graph_with_jobs)
- self.assertEqual(tos.build_types, [])
- self.assertEqual(tos.jobs, [])
- self.assertEqual(tos.unittests, [])
- self.assertEqual(tos.talos, [])
- self.assertEqual(tos.platforms, [])
- self.assertEqual(tos.trigger_tests, 0)
- self.assertEqual(tos.talos_trigger_tests, 0)
- self.assertEqual(tos.env, [])
- self.assertFalse(tos.profile)
- self.assertIsNone(tos.tag)
- self.assertFalse(tos.no_retry)
-
def test_unknown_args(self):
"unknown arguments are ignored"
- tos = TryOptionSyntax('try: --doubledash -z extra', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --doubledash -z extra')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
# equilvant to "try:"..
self.assertEqual(tos.build_types, [])
self.assertEqual(tos.jobs, None)
def test_apostrophe_in_message(self):
"apostrophe does not break parsing"
- tos = TryOptionSyntax('Increase spammy log\'s log level. try: -b do', graph_with_jobs)
+ parameters = {'try_options': parse_message('Increase spammy log\'s log level. try: -b do')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.build_types), ['debug', 'opt'])
def test_b_do(self):
"-b do should produce both build_types"
- tos = TryOptionSyntax('try: -b do', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -b do')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.build_types), ['debug', 'opt'])
def test_b_d(self):
"-b d should produce build_types=['debug']"
- tos = TryOptionSyntax('try: -b d', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -b d')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.build_types), ['debug'])
def test_b_o(self):
"-b o should produce build_types=['opt']"
- tos = TryOptionSyntax('try: -b o', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -b o')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.build_types), ['opt'])
def test_build_o(self):
"--build o should produce build_types=['opt']"
- tos = TryOptionSyntax('try: --build o', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --build o')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.build_types), ['opt'])
def test_b_dx(self):
"-b dx should produce build_types=['debug'], silently ignoring the x"
- tos = TryOptionSyntax('try: -b dx', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -b dx')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.build_types), ['debug'])
def test_j_job(self):
"-j somejob sets jobs=['somejob']"
- tos = TryOptionSyntax('try: -j somejob', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -j somejob')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.jobs), ['somejob'])
def test_j_jobs(self):
"-j job1,job2 sets jobs=['job1', 'job2']"
- tos = TryOptionSyntax('try: -j job1,job2', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -j job1,job2')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.jobs), ['job1', 'job2'])
def test_j_all(self):
"-j all sets jobs=None"
- tos = TryOptionSyntax('try: -j all', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -j all')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.jobs, None)
def test_j_twice(self):
"-j job1 -j job2 sets jobs=job1, job2"
- tos = TryOptionSyntax('try: -j job1 -j job2', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -j job1 -j job2')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.jobs), sorted(['job1', 'job2']))
def test_p_all(self):
"-p all sets platforms=None"
- tos = TryOptionSyntax('try: -p all', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -p all')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.platforms, None)
def test_p_linux(self):
"-p linux sets platforms=['linux', 'linux-l10n']"
- tos = TryOptionSyntax('try: -p linux', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -p linux')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.platforms, ['linux', 'linux-l10n'])
def test_p_linux_win32(self):
"-p linux,win32 sets platforms=['linux', 'linux-l10n', 'win32']"
- tos = TryOptionSyntax('try: -p linux,win32', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -p linux,win32')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.platforms), ['linux', 'linux-l10n', 'win32'])
def test_p_expands_ridealongs(self):
"-p linux,linux64 includes the RIDEALONG_BUILDS"
- tos = TryOptionSyntax('try: -p linux,linux64', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -p linux,linux64')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
platforms = set(['linux'] + RIDEALONG_BUILDS['linux'])
platforms |= set(['linux64'] + RIDEALONG_BUILDS['linux64'])
self.assertEqual(sorted(tos.platforms), sorted(platforms))
def test_u_none(self):
"-u none sets unittests=[]"
- tos = TryOptionSyntax('try: -u none', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u none')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), [])
def test_u_all(self):
"-u all sets unittests=[..whole list..]"
- tos = TryOptionSyntax('try: -u all', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u all')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([{'test': t} for t in unittest_tasks]))
def test_u_single(self):
"-u mochitest-webgl sets unittests=[mochitest-webgl]"
- tos = TryOptionSyntax('try: -u mochitest-webgl', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u mochitest-webgl')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
def test_u_alias(self):
"-u mochitest-gl sets unittests=[mochitest-webgl]"
- tos = TryOptionSyntax('try: -u mochitest-gl', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u mochitest-gl')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
def test_u_multi_alias(self):
"-u e10s sets unittests=[all e10s unittests]"
- tos = TryOptionSyntax('try: -u e10s', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u e10s')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': t} for t in unittest_tasks if 'e10s' in t
]))
def test_u_commas(self):
"-u mochitest-webgl,gtest sets unittests=both"
- tos = TryOptionSyntax('try: -u mochitest-webgl,gtest', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u mochitest-webgl,gtest')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'mochitest-webgl'},
{'test': 'gtest'},
]))
def test_u_chunks(self):
"-u gtest-3,gtest-4 selects the third and fourth chunk of gtest"
- tos = TryOptionSyntax('try: -u gtest-3,gtest-4', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u gtest-3,gtest-4')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'only_chunks': set('34')},
]))
def test_u_platform(self):
"-u gtest[linux] selects the linux platform for gtest"
- tos = TryOptionSyntax('try: -u gtest[linux]', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u gtest[linux]')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux']},
]))
def test_u_platforms(self):
"-u gtest[linux,win32] selects the linux and win32 platforms for gtest"
- tos = TryOptionSyntax('try: -u gtest[linux,win32]', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u gtest[linux,win32]')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux', 'win32']},
]))
def test_u_platforms_pretty(self):
"""-u gtest[Ubuntu] selects the linux, linux64, linux64-asan, linux64-stylo-disabled,
and linux64-stylo-sequential platforms for gtest"""
- tos = TryOptionSyntax('try: -u gtest[Ubuntu]', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u gtest[Ubuntu]')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux32', 'linux64', 'linux64-asan',
'linux64-stylo-disabled', 'linux64-stylo-sequential']},
]))
def test_u_platforms_negated(self):
"-u gtest[-linux] selects all platforms but linux for gtest"
- tos = TryOptionSyntax('try: -u gtest[-linux]', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u gtest[-linux]')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
all_platforms = set([x.attributes['test_platform'] for x in unittest_tasks.values()])
self.assertEqual(sorted(tos.unittests[0]['platforms']), sorted(
[x for x in all_platforms if x != 'linux']
))
def test_u_platforms_negated_pretty(self):
"-u gtest[Ubuntu,-x64] selects just linux for gtest"
- tos = TryOptionSyntax('try: -u gtest[Ubuntu,-x64]', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u gtest[Ubuntu,-x64]')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux32']},
]))
def test_u_chunks_platforms(self):
"-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest"
- tos = TryOptionSyntax('try: -u gtest-1[linux,win32]', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -u gtest-1[linux,win32]')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.unittests), sorted([
{'test': 'gtest', 'platforms': ['linux', 'win32'], 'only_chunks': set('1')},
]))
def test_t_none(self):
"-t none sets talos=[]"
- tos = TryOptionSyntax('try: -t none', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -t none')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.talos), [])
def test_t_all(self):
"-t all sets talos=[..whole list..]"
- tos = TryOptionSyntax('try: -t all', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -t all')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.talos), sorted([{'test': t} for t in talos_tasks]))
def test_t_single(self):
"-t mochitest-webgl sets talos=[mochitest-webgl]"
- tos = TryOptionSyntax('try: -t mochitest-webgl', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: -t mochitest-webgl')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(sorted(tos.talos), sorted([{'test': 'mochitest-webgl'}]))
# -t shares an implementation with -u, so it's not tested heavily
def test_trigger_tests(self):
"--rebuild 10 sets trigger_tests"
- tos = TryOptionSyntax('try: --rebuild 10', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --rebuild 10')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.trigger_tests, 10)
def test_talos_trigger_tests(self):
"--rebuild-talos 10 sets talos_trigger_tests"
- tos = TryOptionSyntax('try: --rebuild-talos 10', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --rebuild-talos 10')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.talos_trigger_tests, 10)
def test_interactive(self):
"--interactive sets interactive"
- tos = TryOptionSyntax('try: --interactive', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --interactive')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.interactive, True)
def test_all_email(self):
"--all-emails sets notifications"
- tos = TryOptionSyntax('try: --all-emails', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --all-emails')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.notifications, 'all')
def test_fail_email(self):
"--failure-emails sets notifications"
- tos = TryOptionSyntax('try: --failure-emails', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --failure-emails')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.notifications, 'failure')
def test_no_email(self):
"no email settings don't set notifications"
- tos = TryOptionSyntax('try:', graph_with_jobs)
+ parameters = {'try_options': parse_message('try:')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.notifications, None)
def test_setenv(self):
"--setenv VAR=value adds a environment variables setting to env"
- tos = TryOptionSyntax('try: --setenv VAR1=value1 --setenv VAR2=value2', graph_with_jobs)
+ parameters = {'try_options': parse_message(
+ 'try: --setenv VAR1=value1 --setenv VAR2=value2')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.env, ['VAR1=value1', 'VAR2=value2'])
def test_profile(self):
"--geckoProfile sets profile to true"
- tos = TryOptionSyntax('try: --geckoProfile', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --geckoProfile')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertTrue(tos.profile)
def test_tag(self):
"--tag TAG sets tag to TAG value"
- tos = TryOptionSyntax('try: --tag tagName', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --tag tagName')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertEqual(tos.tag, 'tagName')
def test_no_retry(self):
"--no-retry sets no_retry to true"
- tos = TryOptionSyntax('try: --no-retry', graph_with_jobs)
+ parameters = {'try_options': parse_message('try: --no-retry')}
+ tos = TryOptionSyntax(parameters, graph_with_jobs)
self.assertTrue(tos.no_retry)
if __name__ == '__main__':
main()
--- a/taskcluster/taskgraph/transforms/build.py
+++ b/taskcluster/taskgraph/transforms/build.py
@@ -39,16 +39,18 @@ def set_defaults(config, jobs):
worker.setdefault('env', {})
yield job
@transforms.add
def set_env(config, jobs):
"""Set extra environment variables from try command line."""
+ env = {}
+ if config.params['try_mode'] == 'try_option_syntax':
+ env = config.params['try_options']['env'] or {}
for job in jobs:
- env = config.config['args'].env
if env:
job_env = {}
if 'worker' in job:
job_env = job['worker']['env']
job_env.update(dict(x.split('=') for x in env))
yield job
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -754,27 +754,32 @@ def set_retry_exit_status(config, tests)
for test in tests:
test['retry-exit-status'] = 4
yield test
@transforms.add
def set_profile(config, tests):
"""Set profiling mode for tests."""
+ profile = None
+ if config.params['try_mode'] == 'try_option_syntax':
+ profile = config.params['try_options']['profile']
for test in tests:
- if config.config['args'].profile and test['suite'] == 'talos':
+ if profile and test['suite'] == 'talos':
test['mozharness']['extra-options'].append('--geckoProfile')
yield test
@transforms.add
def set_tag(config, tests):
"""Set test for a specific tag."""
+ tag = None
+ if config.params['try_mode'] == 'try_option_syntax':
+ tag = config.params['try_options']['tag']
for test in tests:
- tag = config.config['args'].tag
if tag:
test['mozharness']['extra-options'].extend(['--tag', tag])
yield test
@transforms.add
def set_test_type(config, tests):
for test in tests:
@@ -816,31 +821,32 @@ def single_stylo_traversal_tests(config,
@transforms.add
def set_worker_type(config, tests):
"""Set the worker type based on the test platform."""
for test in tests:
# during the taskcluster migration, this is a bit tortured, but it
# will get simpler eventually!
test_platform = test['test-platform']
+ try_options = config.params['try_options'] if config.params['try_options'] else {}
if test.get('worker-type'):
# This test already has its worker type defined, so just use that (yields below)
pass
elif test_platform.startswith('macosx'):
test['worker-type'] = MACOSX_WORKER_TYPES['macosx64']
elif test_platform.startswith('win'):
if test.get('suite', '') == 'talos' and \
not any('taskcluster' in cfg for cfg in test['mozharness']['config']):
test['worker-type'] = 'buildbot-bridge/buildbot-bridge'
else:
test['worker-type'] = \
WINDOWS_WORKER_TYPES[test_platform.split('/')[0]][test['virtualization']]
elif test_platform.startswith('linux') or test_platform.startswith('android'):
if test.get('suite', '') == 'talos' and test['build-platform'] != 'linux64-ccov/opt':
- if config.config['args'].taskcluster_worker:
+ if try_options.get('taskcluster_worker'):
test['worker-type'] = 'releng-hardware/gecko-t-linux-talos'
else:
test['worker-type'] = 'buildbot-bridge/buildbot-bridge'
else:
test['worker-type'] = LINUX_WORKER_TYPES[test['instance-size']]
else:
raise Exception("unknown test_platform {}".format(test_platform))
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -252,26 +252,24 @@ def parse_message(message):
# machines but not overload machines with every try push. Therefore, we add
# this temporary option to be able to push jobs to tc-worker.
parser.add_argument('-w', '--taskcluster-worker',
dest='taskcluster_worker', action='store_true', default=False)
# In order to run test jobs multiple times
parser.add_argument('--rebuild', dest='trigger_tests', type=int, default=1)
args, _ = parser.parse_known_args(parts)
- return args
+ return vars(args)
class TryOptionSyntax(object):
- def __init__(self, message, full_task_graph):
+ def __init__(self, parameters, full_task_graph):
"""
- Parse a "try syntax" formatted commit message. This is the old "-b do -p
- win32 -u all" format. Aliases are applied to map short names to full
- names.
+ Apply the try options in parameters.
The resulting object has attributes:
- build_types: a list containing zero or more of 'opt' and 'debug'
- platforms: a list of selected platform names, or None for all
- unittests: a list of tests, of the form given below, or None for all
- jobs: a list of requested job names, or None for all
- trigger_tests: the number of times tests should be triggered (--rebuild)
@@ -300,38 +298,32 @@ class TryOptionSyntax(object):
self.interactive = False
self.notifications = None
self.talos_trigger_tests = 0
self.env = []
self.profile = False
self.tag = None
self.no_retry = False
- parts = split_try_msg(message)
- if not parts:
- return None
-
- args = parse_message(message)
- assert args is not None
-
- self.jobs = self.parse_jobs(args.jobs)
- self.build_types = self.parse_build_types(args.build_types, full_task_graph)
- self.platforms = self.parse_platforms(args.platforms, full_task_graph)
+ options = parameters['try_options']
+ self.jobs = self.parse_jobs(options['jobs'])
+ self.build_types = self.parse_build_types(options['build_types'], full_task_graph)
+ self.platforms = self.parse_platforms(options['platforms'], full_task_graph)
self.unittests = self.parse_test_option(
- "unittest_try_name", args.unittests, full_task_graph)
- self.talos = self.parse_test_option("talos_try_name", args.talos, full_task_graph)
- self.trigger_tests = args.trigger_tests
- self.interactive = args.interactive
- self.notifications = args.notifications
- self.talos_trigger_tests = args.talos_trigger_tests
- self.env = args.env
- self.profile = args.profile
- self.tag = args.tag
- self.no_retry = args.no_retry
- self.include_nightly = args.include_nightly
+ "unittest_try_name", options['unittests'], full_task_graph)
+ self.talos = self.parse_test_option("talos_try_name", options['talos'], full_task_graph)
+ self.trigger_tests = options['trigger_tests']
+ self.interactive = options['interactive']
+ self.notifications = options['notifications']
+ self.talos_trigger_tests = options['talos_trigger_tests']
+ self.env = options['env']
+ self.profile = options['profile']
+ self.tag = options['tag']
+ self.no_retry = options['no_retry']
+ self.include_nightly = options['include_nightly']
def parse_jobs(self, jobs_arg):
if not jobs_arg or jobs_arg == ['all']:
return None
expanded = []
for job in jobs_arg:
expanded.extend(j.strip() for j in job.split(','))
return expanded