Bug 1387135 - Add ability to use 'artifact' template to |mach try fuzzy|, r?maja_zf
This uses the new templating mechanism in taskgraph to schedule artifact builds when using
|mach try fuzzy|. Like |mach try syntax|, this will automatically be enabled if
--enable-artifact-builds is set in the mozconfig. The --artifact/--no-artifact arguments
can be used to override the default.
MozReview-Commit-ID: J8HVZzOt4mX
--- a/tools/tryselect/mach_commands.py
+++ b/tools/tryselect/mach_commands.py
@@ -38,16 +38,21 @@ def syntax_parser():
parser.add_argument('--no-artifact',
dest='no_artifact',
action='store_true',
help='Force compiled (non-artifact) builds even when '
'--enable-artifact-builds is set.')
return parser
+def fuzzy_parser():
+ from tryselect.selectors.fuzzy import FuzzyParser
+ return FuzzyParser()
+
+
@CommandProvider
class TrySelect(MachCommandBase):
@Command('try',
category='ci',
description='Push selected tasks to the try server')
@CommandArgument('args', nargs=argparse.REMAINDER)
def try_default(self, args):
@@ -64,29 +69,18 @@ class TrySelect(MachCommandBase):
"""
parser = syntax_parser()
kwargs = vars(parser.parse_args(args))
return self._mach_context.commands.dispatch(
'try', subcommand='syntax', context=self._mach_context, **kwargs)
@SubCommand('try',
'fuzzy',
- description='Select tasks on try using a fuzzy finder')
- @CommandArgument('-q', '--query', metavar='STR',
- help="Use the given query instead of entering the selection "
- "interface. Equivalent to typing <query><ctrl-a><enter> "
- "from the interface.")
- @CommandArgument('-u', '--update', action='store_true', default=False,
- help="Update fzf before running.")
- @CommandArgument('--full', action='store_true', default=False,
- help="Use the full set of tasks as input to fzf (instead of "
- "target tasks).")
- @CommandArgument('-p', '--parameters', default=None,
- help="Use the given parameters.yml to generate tasks, "
- "defaults to latest parameters.yml from mozilla-central")
+ description='Select tasks on try using a fuzzy finder',
+ parser=fuzzy_parser)
def try_fuzzy(self, *args, **kwargs):
"""Select which tasks to use with fzf.
This selector runs all task labels through a fuzzy finding interface.
All selected task labels and their dependencies will be scheduled on
try.
Keyboard Shortcuts
--- a/tools/tryselect/selectors/fuzzy.py
+++ b/tools/tryselect/selectors/fuzzy.py
@@ -8,16 +8,17 @@ import os
import platform
import subprocess
import sys
from distutils.spawn import find_executable
from mozboot.util import get_state_dir
from ..tasks import generate_tasks
+from ..templates import TemplateParser
from ..vcs import VCSHelper
try:
import blessings
terminal = blessings.Terminal()
except ImportError:
from mozlint.formatters.stylish import NullTerminal
terminal = NullTerminal()
@@ -75,16 +76,45 @@ fzf_header_shortcuts = {
'cursor-down': 'ctrl-j',
'toggle-select': 'tab',
'select-all': 'ctrl-a',
'accept': 'enter',
'cancel': 'ctrl-c',
}
+class FuzzyParser(TemplateParser):
+ arguments = [
+ [['-q', '--query'],
+ {'metavar': 'STR',
+ 'help': "Use the given query instead of entering the selection "
+ "interface. Equivalent to typing <query><ctrl-a><enter> "
+ "from the interface.",
+ }],
+ [['-u', '--update'],
+ {'action': 'store_true',
+ 'default': False,
+ 'help': "Update fzf before running.",
+ }],
+ [['--full'],
+ {'action': 'store_true',
+ 'default': False,
+ 'help': "Use the full set of tasks as input to fzf (instead of "
+ "target tasks).",
+ }],
+ [['-p', '--parameters'],
+ {'default': None,
+ 'help': "Use the given parameters.yml to generate tasks, "
+ "defaults to latest parameters.yml from mozilla-central",
+ }],
+ ]
+
+ templates = ['artifact']
+
+
def run(cmd, cwd=None):
is_win = platform.system() == 'Windows'
return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
def run_fzf_install_script(fzf_path, bin_only=False):
# We could run this without installing the shell integrations on all
# platforms, but those integrations are actually really useful so give user
@@ -161,17 +191,17 @@ def fzf_bootstrap(update=False):
def format_header():
shortcuts = []
for action, key in sorted(fzf_header_shortcuts.iteritems()):
shortcuts.append('{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}'.format(
t=terminal, action=action, key=key))
return FZF_HEADER.format(shortcuts=', '.join(shortcuts), t=terminal)
-def run_fuzzy_try(update=False, query=None, full=False, parameters=None):
+def run_fuzzy_try(update=False, query=None, templates=None, full=False, parameters=None, **kwargs):
fzf = fzf_bootstrap(update)
if not fzf:
print(FZF_NOT_FOUND)
return
vcs = VCSHelper.create()
vcs.check_working_directory()
@@ -194,9 +224,10 @@ def run_fuzzy_try(update=False, query=No
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
selected = proc.communicate('\n'.join(all_tasks))[0].splitlines()
if not selected:
print("no tasks selected")
return
- return vcs.push_to_try("Pushed via 'mach try fuzzy', see diff for scheduled tasks", selected)
+ msg = "Pushed via 'mach try fuzzy', see diff for scheduled tasks"
+ return vcs.push_to_try(msg, selected, templates)
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/templates.py
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Templates provide a way of modifying the task definition of selected
+tasks. They live under taskcluster/taskgraph/templates.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+from abc import ABCMeta, abstractmethod, abstractproperty
+from argparse import ArgumentParser
+
+from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+class TemplateParser(ArgumentParser):
+ __metaclass__ = ABCMeta
+
+ @abstractproperty
+ def arguments(self):
+ pass
+
+ @abstractproperty
+ def templates(self):
+ pass
+
+ def __init__(self, *args, **kwargs):
+ ArgumentParser.__init__(self, *args, **kwargs)
+
+ for cli, kwargs in self.arguments:
+ self.add_argument(*cli, **kwargs)
+
+ self.templates = {t: all_templates[t]() for t in self.templates}
+
+ group = self.add_argument_group("template arguments")
+ for template in self.templates.values():
+ template.add_arguments(group)
+
+ def parse_known_args(self, *args, **kwargs):
+ args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs)
+
+ if self.templates:
+ args.templates = {}
+ for name, cls in self.templates.iteritems():
+ context = cls.context(**vars(args))
+ if context is not None:
+ args.templates[name] = context
+
+ return args, remainder
+
+
+class Template(object):
+ __metaclass__ = ABCMeta
+
+ @abstractmethod
+ def add_arguments(self, parser):
+ pass
+
+ @abstractmethod
+ def context(self, **kwargs):
+ pass
+
+
+class Artifact(Template):
+
+ def add_arguments(self, parser):
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('--artifact', action='store_true',
+ help='Force artifact builds where possible.')
+ group.add_argument('--no-artifact', action='store_true',
+ help='Disable artifact builds even if being used locally.')
+
+ def context(self, artifact, no_artifact, **kwargs):
+ if artifact:
+ return {'enabled': 1}
+
+ if no_artifact:
+ return
+
+ build = MozbuildObject.from_environment(cwd=here)
+ try:
+ if build.substs.get("MOZ_ARTIFACT_BUILDS"):
+ print("Artifact builds enabled, pass --no-artifact to disable")
+ return {'enabled': 1}
+ except BuildEnvironmentNotFoundException:
+ pass
+
+
+all_templates = {
+ 'artifact': Artifact,
+}
--- a/tools/tryselect/vcs.py
+++ b/tools/tryselect/vcs.py
@@ -71,20 +71,24 @@ class VCSHelper(object):
def run(self, cmd):
try:
return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print("Error running `{}`:".format(' '.join(cmd)))
print(e.output)
raise
- def write_task_config(self, labels):
+ def write_task_config(self, labels, templates=None):
config = os.path.join(self.root, 'try_task_config.json')
with open(config, 'w') as fh:
- json.dump(sorted(labels), fh, indent=2)
+ try_task_config = {'tasks': sorted(labels)}
+ if templates:
+ try_task_config['templates'] = templates
+
+ json.dump(try_task_config, fh, indent=2)
return config
def check_working_directory(self):
if self.has_uncommitted_changes:
print(UNCOMMITTED_CHANGES)
sys.exit(1)
@abstractmethod
@@ -97,21 +101,21 @@ class VCSHelper(object):
@abstractproperty
def has_uncommitted_changes(self):
pass
class HgHelper(VCSHelper):
- def push_to_try(self, msg, labels=None):
+ def push_to_try(self, msg, labels=None, templates=None):
self.check_working_directory()
if labels:
- config = self.write_task_config(labels)
+ config = self.write_task_config(labels, templates)
self.run(['hg', 'add', config])
try:
return subprocess.check_call(['hg', 'push-to-try', '-m', msg])
except subprocess.CalledProcessError:
try:
self.run(['hg', 'showconfig', 'extensions.push-to-try'])
except subprocess.CalledProcessError:
@@ -131,27 +135,27 @@ class HgHelper(VCSHelper):
@property
def has_uncommitted_changes(self):
stat = [s for s in self.run(['hg', 'status', '-amrn']).split() if s]
return len(stat) > 0
class GitHelper(VCSHelper):
- def push_to_try(self, msg, labels=None):
+ def push_to_try(self, msg, labels=None, templates=None):
self.check_working_directory()
try:
subprocess.check_output(['git', 'cinnabar', '--version'], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
print(GIT_CINNABAR_NOT_FOUND)
return 1
if labels:
- config = self.write_task_config(labels)
+ config = self.write_task_config(labels, templates)
self.run(['git', 'add', config])
subprocess.check_call(['git', 'commit', '--allow-empty', '-m', msg])
try:
return subprocess.call(['git', 'push', 'hg::ssh://hg.mozilla.org/try',
'+HEAD:refs/heads/branches/default/tip'])
finally:
self.run(['git', 'reset', 'HEAD~'])