Bug 1387135 - Add ability to use 'artifact' template to |mach try fuzzy|, r?maja_zf draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 15 Aug 2017 11:39:46 -0400
changeset 648490 34b5892c28c3037278db1c5edc73ba3166d52174
parent 648489 4fe3fd7cf401d2c73b7d26e2e6382fce82e4257e
child 726837 45676dea3b8a1eb97d112b74d74f5e09d505e23a
push id74770
push userahalberstadt@mozilla.com
push dateThu, 17 Aug 2017 20:33:01 +0000
reviewersmaja_zf
bugs1387135
milestone57.0a1
Bug 1387135 - Add ability to use 'artifact' template to |mach try fuzzy|, r?maja_zf This uses the new templating mechanism in taskgraph to schedule artifact builds when using |mach try fuzzy|. Like |mach try syntax|, this will automatically be enabled if --enable-artifact-builds is set in the mozconfig. The --artifact/--no-artifact arguments can be used to override the default. MozReview-Commit-ID: J8HVZzOt4mX
tools/tryselect/mach_commands.py
tools/tryselect/selectors/fuzzy.py
tools/tryselect/templates.py
tools/tryselect/vcs.py
--- a/tools/tryselect/mach_commands.py
+++ b/tools/tryselect/mach_commands.py
@@ -38,16 +38,21 @@ def syntax_parser():
     parser.add_argument('--no-artifact',
                         dest='no_artifact',
                         action='store_true',
                         help='Force compiled (non-artifact) builds even when '
                              '--enable-artifact-builds is set.')
     return parser
 
 
+def fuzzy_parser():
+    from tryselect.selectors.fuzzy import FuzzyParser
+    return FuzzyParser()
+
+
 @CommandProvider
 class TrySelect(MachCommandBase):
 
     @Command('try',
              category='ci',
              description='Push selected tasks to the try server')
     @CommandArgument('args', nargs=argparse.REMAINDER)
     def try_default(self, args):
@@ -64,29 +69,18 @@ class TrySelect(MachCommandBase):
         """
         parser = syntax_parser()
         kwargs = vars(parser.parse_args(args))
         return self._mach_context.commands.dispatch(
             'try', subcommand='syntax', context=self._mach_context, **kwargs)
 
     @SubCommand('try',
                 'fuzzy',
-                description='Select tasks on try using a fuzzy finder')
-    @CommandArgument('-q', '--query', metavar='STR',
-                     help="Use the given query instead of entering the selection "
-                          "interface. Equivalent to typing <query><ctrl-a><enter> "
-                          "from the interface.")
-    @CommandArgument('-u', '--update', action='store_true', default=False,
-                     help="Update fzf before running.")
-    @CommandArgument('--full', action='store_true', default=False,
-                     help="Use the full set of tasks as input to fzf (instead of "
-                          "target tasks).")
-    @CommandArgument('-p', '--parameters', default=None,
-                     help="Use the given parameters.yml to generate tasks, "
-                          "defaults to latest parameters.yml from mozilla-central")
+                description='Select tasks on try using a fuzzy finder',
+                parser=fuzzy_parser)
     def try_fuzzy(self, *args, **kwargs):
         """Select which tasks to use with fzf.
 
         This selector runs all task labels through a fuzzy finding interface.
         All selected task labels and their dependencies will be scheduled on
         try.
 
         Keyboard Shortcuts
--- a/tools/tryselect/selectors/fuzzy.py
+++ b/tools/tryselect/selectors/fuzzy.py
@@ -8,16 +8,17 @@ import os
 import platform
 import subprocess
 import sys
 from distutils.spawn import find_executable
 
 from mozboot.util import get_state_dir
 
 from ..tasks import generate_tasks
+from ..templates import TemplateParser
 from ..vcs import VCSHelper
 
 try:
     import blessings
     terminal = blessings.Terminal()
 except ImportError:
     from mozlint.formatters.stylish import NullTerminal
     terminal = NullTerminal()
@@ -75,16 +76,45 @@ fzf_header_shortcuts = {
     'cursor-down': 'ctrl-j',
     'toggle-select': 'tab',
     'select-all': 'ctrl-a',
     'accept': 'enter',
     'cancel': 'ctrl-c',
 }
 
 
+class FuzzyParser(TemplateParser):
+    arguments = [
+        [['-q', '--query'],
+         {'metavar': 'STR',
+          'help': "Use the given query instead of entering the selection "
+                  "interface. Equivalent to typing <query><ctrl-a><enter> "
+                  "from the interface.",
+          }],
+        [['-u', '--update'],
+         {'action': 'store_true',
+          'default': False,
+          'help': "Update fzf before running.",
+          }],
+        [['--full'],
+         {'action': 'store_true',
+          'default': False,
+          'help': "Use the full set of tasks as input to fzf (instead of "
+                  "target tasks).",
+          }],
+        [['-p', '--parameters'],
+         {'default': None,
+          'help': "Use the given parameters.yml to generate tasks, "
+                  "defaults to latest parameters.yml from mozilla-central",
+          }],
+    ]
+
+    templates = ['artifact']
+
+
 def run(cmd, cwd=None):
     is_win = platform.system() == 'Windows'
     return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
 
 
 def run_fzf_install_script(fzf_path, bin_only=False):
     # We could run this without installing the shell integrations on all
     # platforms, but those integrations are actually really useful so give user
@@ -161,17 +191,17 @@ def fzf_bootstrap(update=False):
 def format_header():
     shortcuts = []
     for action, key in sorted(fzf_header_shortcuts.iteritems()):
         shortcuts.append('{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}'.format(
                          t=terminal, action=action, key=key))
     return FZF_HEADER.format(shortcuts=', '.join(shortcuts), t=terminal)
 
 
-def run_fuzzy_try(update=False, query=None, full=False, parameters=None):
+def run_fuzzy_try(update=False, query=None, templates=None, full=False, parameters=None, **kwargs):
     fzf = fzf_bootstrap(update)
 
     if not fzf:
         print(FZF_NOT_FOUND)
         return
 
     vcs = VCSHelper.create()
     vcs.check_working_directory()
@@ -194,9 +224,10 @@ def run_fuzzy_try(update=False, query=No
 
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
     selected = proc.communicate('\n'.join(all_tasks))[0].splitlines()
 
     if not selected:
         print("no tasks selected")
         return
 
-    return vcs.push_to_try("Pushed via 'mach try fuzzy', see diff for scheduled tasks", selected)
+    msg = "Pushed via 'mach try fuzzy', see diff for scheduled tasks"
+    return vcs.push_to_try(msg, selected, templates)
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/templates.py
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Templates provide a way of modifying the task definition of selected
+tasks. They live under taskcluster/taskgraph/templates.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+from abc import ABCMeta, abstractmethod, abstractproperty
+from argparse import ArgumentParser
+
+from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+class TemplateParser(ArgumentParser):
+    __metaclass__ = ABCMeta
+
+    @abstractproperty
+    def arguments(self):
+        pass
+
+    @abstractproperty
+    def templates(self):
+        pass
+
+    def __init__(self, *args, **kwargs):
+        ArgumentParser.__init__(self, *args, **kwargs)
+
+        for cli, kwargs in self.arguments:
+            self.add_argument(*cli, **kwargs)
+
+        self.templates = {t: all_templates[t]() for t in self.templates}
+
+        group = self.add_argument_group("template arguments")
+        for template in self.templates.values():
+            template.add_arguments(group)
+
+    def parse_known_args(self, *args, **kwargs):
+        args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs)
+
+        if self.templates:
+            args.templates = {}
+            for name, cls in self.templates.iteritems():
+                context = cls.context(**vars(args))
+                if context is not None:
+                    args.templates[name] = context
+
+        return args, remainder
+
+
+class Template(object):
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def add_arguments(self, parser):
+        pass
+
+    @abstractmethod
+    def context(self, **kwargs):
+        pass
+
+
+class Artifact(Template):
+
+    def add_arguments(self, parser):
+        group = parser.add_mutually_exclusive_group()
+        group.add_argument('--artifact', action='store_true',
+                           help='Force artifact builds where possible.')
+        group.add_argument('--no-artifact', action='store_true',
+                           help='Disable artifact builds even if being used locally.')
+
+    def context(self, artifact, no_artifact, **kwargs):
+        if artifact:
+            return {'enabled': 1}
+
+        if no_artifact:
+            return
+
+        build = MozbuildObject.from_environment(cwd=here)
+        try:
+            if build.substs.get("MOZ_ARTIFACT_BUILDS"):
+                print("Artifact builds enabled, pass --no-artifact to disable")
+                return {'enabled': 1}
+        except BuildEnvironmentNotFoundException:
+            pass
+
+
+all_templates = {
+    'artifact': Artifact,
+}
--- a/tools/tryselect/vcs.py
+++ b/tools/tryselect/vcs.py
@@ -71,20 +71,24 @@ class VCSHelper(object):
     def run(self, cmd):
         try:
             return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         except subprocess.CalledProcessError as e:
             print("Error running `{}`:".format(' '.join(cmd)))
             print(e.output)
             raise
 
-    def write_task_config(self, labels):
+    def write_task_config(self, labels, templates=None):
         config = os.path.join(self.root, 'try_task_config.json')
         with open(config, 'w') as fh:
-            json.dump(sorted(labels), fh, indent=2)
+            try_task_config = {'tasks': sorted(labels)}
+            if templates:
+                try_task_config['templates'] = templates
+
+            json.dump(try_task_config, fh, indent=2)
         return config
 
     def check_working_directory(self):
         if self.has_uncommitted_changes:
             print(UNCOMMITTED_CHANGES)
             sys.exit(1)
 
     @abstractmethod
@@ -97,21 +101,21 @@ class VCSHelper(object):
 
     @abstractproperty
     def has_uncommitted_changes(self):
         pass
 
 
 class HgHelper(VCSHelper):
 
-    def push_to_try(self, msg, labels=None):
+    def push_to_try(self, msg, labels=None, templates=None):
         self.check_working_directory()
 
         if labels:
-            config = self.write_task_config(labels)
+            config = self.write_task_config(labels, templates)
             self.run(['hg', 'add', config])
 
         try:
             return subprocess.check_call(['hg', 'push-to-try', '-m', msg])
         except subprocess.CalledProcessError:
             try:
                 self.run(['hg', 'showconfig', 'extensions.push-to-try'])
             except subprocess.CalledProcessError:
@@ -131,27 +135,27 @@ class HgHelper(VCSHelper):
     @property
     def has_uncommitted_changes(self):
         stat = [s for s in self.run(['hg', 'status', '-amrn']).split() if s]
         return len(stat) > 0
 
 
 class GitHelper(VCSHelper):
 
-    def push_to_try(self, msg, labels=None):
+    def push_to_try(self, msg, labels=None, templates=None):
         self.check_working_directory()
 
         try:
             subprocess.check_output(['git', 'cinnabar', '--version'], stderr=subprocess.STDOUT)
         except subprocess.CalledProcessError:
             print(GIT_CINNABAR_NOT_FOUND)
             return 1
 
         if labels:
-            config = self.write_task_config(labels)
+            config = self.write_task_config(labels, templates)
             self.run(['git', 'add', config])
 
         subprocess.check_call(['git', 'commit', '--allow-empty', '-m', msg])
         try:
             return subprocess.call(['git', 'push', 'hg::ssh://hg.mozilla.org/try',
                                     '+HEAD:refs/heads/branches/default/tip'])
         finally:
             self.run(['git', 'reset', 'HEAD~'])