Bug 1400295 - Make try syntax use try_task_config.json
This is a prototype. Before this is ready for review, it needs to:
1. Re-implement all the stuff like --rebuild, --setenv, --geckoProfile, etc. (via templates)
2. Move the try_option_syntax test over (requires standing up a new python-test suite for tryselect)
3. Update docs
MozReview-Commit-ID: 4hKTErWMXtt
--- a/tools/tryselect/cli.py
+++ b/tools/tryselect/cli.py
@@ -37,16 +37,21 @@ class BaseTryParser(ArgumentParser):
{'default': None,
'help': 'Load a saved selection.',
}],
[['--list-presets'],
{'action': 'store_true',
'default': False,
'help': 'List available preset selections.',
}],
+ [['--parameters'],
+ {'default': None,
+ 'help': "Use the given parameters.yml to generate tasks, "
+ "defaults to latest parameters.yml from mozilla-central",
+ }],
]
arguments = []
templates = []
def __init__(self, *args, **kwargs):
ArgumentParser.__init__(self, *args, **kwargs)
group = self.add_argument_group("{} arguments".format(self.name))
--- a/tools/tryselect/selectors/fuzzy.py
+++ b/tools/tryselect/selectors/fuzzy.py
@@ -97,21 +97,16 @@ class FuzzyParser(BaseTryParser):
'help': "Update fzf before running.",
}],
[['--full'],
{'action': 'store_true',
'default': False,
'help': "Use the full set of tasks as input to fzf (instead of "
"target tasks).",
}],
- [['-p', '--parameters'],
- {'default': None,
- 'help': "Use the given parameters.yml to generate tasks, "
- "defaults to latest parameters.yml from mozilla-central",
- }],
]
templates = ['artifact', 'env']
def run(cmd, cwd=None):
is_win = platform.system() == 'Windows'
return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
@@ -207,17 +202,18 @@ def run_fuzzy_try(update=False, query=No
if not fzf:
print(FZF_NOT_FOUND)
return
vcs = VCSHelper.create()
vcs.check_working_directory(push)
- all_tasks = generate_tasks(parameters, full)
+ tg = generate_tasks(parameters, full)
+ all_tasks = [label for label in tg.graph.visit_postorder()]
key_shortcuts = [k + ':' + v for k, v in fzf_shortcuts.iteritems()]
cmd = [
fzf, '-m',
'--bind', ','.join(key_shortcuts),
'--header', format_header(),
# Using python to split the preview string is a bit convoluted,
# but is guaranteed to be available on all platforms.
--- a/tools/tryselect/selectors/syntax.py
+++ b/tools/tryselect/selectors/syntax.py
@@ -5,17 +5,19 @@
from __future__ import absolute_import, print_function, unicode_literals
import os
import re
import sys
from collections import defaultdict
import mozpack.path as mozpath
-from .. import preset
+from taskgraph.try_option_syntax import TryOptionSyntax, parse_message
+
+from .. import preset, tasks
from ..cli import BaseTryParser
from ..vcs import VCSHelper
class SyntaxParser(BaseTryParser):
name = 'syntax'
arguments = [
[['paths'],
@@ -633,14 +635,18 @@ class AutoTry(object):
if kwargs["verbose"] and paths_by_flavor:
print('The following tests will be selected: ')
for flavor, paths in paths_by_flavor.iteritems():
print("%s: %s" % (flavor, ",".join(paths)))
if kwargs["verbose"]:
print('The following try syntax was calculated:\n%s' % msg)
- self.vcs.push_to_try('syntax', kwargs["message"].format(msg=msg), push=kwargs['push'])
+ tg = tasks.generate_tasks(params=kwargs["parameters"], full=True)
+ options = parse_message(msg)
+ options = TryOptionSyntax({'try_options': options}, tg)
+ labels = [t.label for t in tg.tasks.itervalues() if options.task_matches(t)]
+ self.vcs.push_to_try('syntax', msg, labels, push=kwargs['push'])
if kwargs["save"]:
assert msg.startswith("try: ")
msg = msg[len("try: "):]
preset.save('try', kwargs["save"], msg)
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -1,26 +1,28 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
+import json
import os
import sys
from mozboot.util import get_state_dir
from mozbuild.base import MozbuildObject
from mozpack.files import FileFinder
from taskgraph.generator import TaskGraphGenerator
from taskgraph.parameters import (
ParameterMismatch,
load_parameters_file,
)
+from taskgraph.taskgraph import TaskGraph
here = os.path.abspath(os.path.dirname(__file__))
build = MozbuildObject.from_environment(cwd=here)
PARAMETER_MISMATCH = """
ERROR - The parameters being used to generate tasks differ from those defined
in your working copy:
@@ -40,27 +42,25 @@ def invalidate(cache):
tc_dir = os.path.join(build.topsrcdir, 'taskcluster')
tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
cmod = os.path.getmtime(cache)
if tmod > cmod:
os.remove(cache)
-def generate_tasks(params=None, full=False):
- params = params or "project=mozilla-central"
-
+def generate_tasks(params="project=mozilla-central", full=False):
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
- attr = 'full_task_set' if full else 'target_task_set'
+ attr = 'full_task_graph' if full else 'target_task_graph'
cache = os.path.join(cache_dir, attr)
invalidate(cache)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
- return fh.read().splitlines()
+ return TaskGraph.from_json(json.load(fh))[1]
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
try:
params = load_parameters_file(params, strict=False)
params.check()
@@ -68,15 +68,14 @@ def generate_tasks(params=None, full=Fal
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
- labels = [label for label in tg.graph.visit_postorder()]
os.chdir(cwd)
with open(cache, 'w') as fh:
- fh.write('\n'.join(labels))
- return labels
+ json.dump(tg.to_json(), fh)
+ return tg
--- a/tools/tryselect/vcs.py
+++ b/tools/tryselect/vcs.py
@@ -92,38 +92,34 @@ class VCSHelper(object):
def check_working_directory(self, push=True):
if not push:
return
if self.has_uncommitted_changes:
print(UNCOMMITTED_CHANGES)
sys.exit(1)
- def push_to_try(self, method, msg, labels=None, templates=None, push=True):
+ def push_to_try(self, method, msg, labels, templates=None, push=True):
commit_message = '%s\n\nPushed via `mach try %s`' % (msg, method)
self.check_working_directory(push)
-
- config = None
- if labels or labels == []:
- config = self.write_task_config(labels, templates)
+ config = self.write_task_config(labels, templates)
try:
if not push:
print("Commit message:")
print(commit_message)
- if config:
- print("Calculated try_task_config.json:")
- with open(config) as fh:
- print(fh.read())
+ print("Calculated try_task_config.json:")
+ with open(config) as fh:
+ print(fh.read())
return
self._push_to_try(commit_message, config)
finally:
- if config and os.path.isfile(config):
+ if os.path.isfile(config):
os.remove(config)
@abstractmethod
def _push_to_try(self, msg, config):
pass
@abstractproperty
def files_changed(self):
@@ -133,18 +129,17 @@ class VCSHelper(object):
def has_uncommitted_changes(self):
pass
class HgHelper(VCSHelper):
def _push_to_try(self, msg, config):
try:
- if config:
- self.run(['hg', 'add', config])
+ self.run(['hg', 'add', config])
return subprocess.check_call(['hg', 'push-to-try', '-m', msg])
except subprocess.CalledProcessError:
try:
self.run(['hg', 'showconfig', 'extensions.push-to-try'])
except subprocess.CalledProcessError:
print(HG_PUSH_TO_TRY_NOT_FOUND)
return 1
finally:
@@ -165,18 +160,17 @@ class GitHelper(VCSHelper):
def _push_to_try(self, msg, config):
try:
subprocess.check_output(['git', 'cinnabar', '--version'], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
print(GIT_CINNABAR_NOT_FOUND)
return 1
- if config:
- self.run(['git', 'add', config])
+ self.run(['git', 'add', config])
subprocess.check_call(['git', 'commit', '--allow-empty', '-m', msg])
try:
return subprocess.call(['git', 'push', 'hg::ssh://hg.mozilla.org/try',
'+HEAD:refs/heads/branches/default/tip'])
finally:
self.run(['git', 'reset', 'HEAD~'])
@property