--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -47,16 +47,17 @@ mozilla.pth:testing/firefox-ui/harness
mozilla.pth:testing/marionette/client
mozilla.pth:testing/marionette/harness
mozilla.pth:testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
mozilla.pth:testing/marionette/puppeteer/firefox
mozilla.pth:testing/raptor
mozilla.pth:testing/talos
packages.txt:testing/mozbase/packages.txt
mozilla.pth:tools
+mozilla.pth:tools/tryselect
mozilla.pth:testing/web-platform
mozilla.pth:testing/web-platform/tests/tools/wptrunner
mozilla.pth:testing/web-platform/tests/tools/wptserve
mozilla.pth:testing/web-platform/tests/tools/six
mozilla.pth:testing/xpcshell
mozilla.pth:third_party/python/mock-1.0.0
mozilla.pth:xpcom/typelib/xpt/tools
mozilla.pth:tools/docs
--- a/python/Pipfile
+++ b/python/Pipfile
@@ -1,15 +1,16 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
"d5b4a14" = {path = "./mach"}
+"56126ec" = {path = "./mozboot"}
"8ddb376" = {path = "./mozbuild"}
"8b3b8fb" = {path = "./mozlint"}
"b3ddbcf" = {path = "./mozterm"}
"38a4a9a" = {path = "./mozversioncontrol"}
"26d92fb" = {path = "./../config/mozunit"}
"5942084" = {path = "./../taskcluster"}
"99247d1" = {path = "./../testing/marionette/client"}
"c6b7cc8" = {path = "./../testing/marionette/harness"}
@@ -42,10 +43,11 @@ name = "pypi"
"b1adf4f" = {path = "./../third_party/python/pyyaml"}
"09d375d" = {path = "./../third_party/python/redo"}
"053111f" = {path = "./../third_party/python/requests"}
"d250320" = {path = "./../third_party/python/six"}
"16bd0c6" = {path = "./../third_party/python/requests-unixsocket"}
"8a4f01a" = {path = "./../third_party/python/slugid"}
"bddbd22" = {path = "./../third_party/python/voluptuous"}
"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"}
+"e219a89" = {path = "./../tools/tryselect"}
[dev-packages]
--- a/python/Pipfile.lock
+++ b/python/Pipfile.lock
@@ -1,12 +1,12 @@
{
"_meta": {
"hash": {
- "sha256": "2b21ee005096806e2e1021a34297aab0af3b5c81ae81c73efeb316d7adc4145b"
+ "sha256": "ae3c5a6736890f6bf7e6f10f5bf07c0a7897a2da9c561c7557d1c2e3d6eefc63"
},
"pipfile-spec": 6,
"requires": {},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
@@ -49,16 +49,19 @@
},
"47200d8": {
"markers": "python_version < '3'",
"path": "./../third_party/python/futures"
},
"501835d": {
"path": "./../testing/mozbase/mozinstall"
},
+ "56126ec": {
+ "path": "./mozboot"
+ },
"58d0848": {
"path": "./../testing/mozbase/mozfile"
},
"5942084": {
"path": "./../taskcluster"
},
"605c6a6": {
"path": "./../third_party/python/browsermob-proxy"
@@ -125,16 +128,19 @@
"path": "./../testing/mozbase/mozprofile"
},
"dcaaf6a": {
"path": "./../third_party/python/json-e"
},
"e09e103": {
"path": "./../testing/mozbase/moznetwork"
},
+ "e219a89": {
+ "path": "./../tools/tryselect"
+ },
"f1d74ca": {
"path": "./../testing/mozbase/mozversion"
},
"f1de77a": {
"markers": "python_version < '3.3'",
"path": "./../third_party/python/which"
},
"f4b00e9": {
deleted file mode 100644
--- a/tools/tryselect/cli.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import os
-import subprocess
-import tempfile
-from argparse import ArgumentParser
-
-from .templates import all_templates
-
-
-COMMON_ARGUMENT_GROUPS = {
- 'push': [
- [['-m', '--message'],
- {'const': 'editor',
- 'default': '{msg}',
- 'nargs': '?',
- 'help': 'Use the specified commit message, or create it in your '
- '$EDITOR if blank. Defaults to computed message.',
- }],
- [['--no-push'],
- {'dest': 'push',
- 'action': 'store_false',
- 'help': 'Do not push to try as a result of running this command (if '
- 'specified this command will only print calculated try '
- 'syntax and selection info).',
- }],
- [['--closed-tree'],
- {'action': 'store_true',
- 'default': False,
- 'help': 'Push despite a closed try tree',
- }],
- ],
- 'preset': [
- [['--save'],
- {'default': None,
- 'help': 'Save selection for future use with --preset.',
- }],
- [['--preset'],
- {'default': None,
- 'help': 'Load a saved selection.',
- }],
- [['--list-presets'],
- {'action': 'store_const',
- 'const': 'list_presets',
- 'dest': 'mod_presets',
- 'default': None,
- 'help': 'List available preset selections.',
- }],
- [['--edit-presets'],
- {'action': 'store_const',
- 'const': 'edit_presets',
- 'dest': 'mod_presets',
- 'default': None,
- 'help': 'Edit the preset file.',
- }],
- ],
- 'task': [
- [['--full'],
- {'action': 'store_true',
- 'default': False,
- 'help': "Use the full set of tasks as input to fzf (instead of "
- "target tasks).",
- }],
- [['-p', '--parameters'],
- {'default': None,
- 'help': "Use the given parameters.yml to generate tasks, "
- "defaults to latest parameters.yml from mozilla-central",
- }],
- ],
-}
-
-
-class BaseTryParser(ArgumentParser):
- name = 'try'
- common_groups = ['push', 'preset']
- arguments = []
- templates = []
-
- def __init__(self, *args, **kwargs):
- ArgumentParser.__init__(self, *args, **kwargs)
-
- group = self.add_argument_group("{} arguments".format(self.name))
- for cli, kwargs in self.arguments:
- group.add_argument(*cli, **kwargs)
-
- for name in self.common_groups:
- group = self.add_argument_group("{} arguments".format(name))
- arguments = COMMON_ARGUMENT_GROUPS[name]
- for cli, kwargs in arguments:
- group.add_argument(*cli, **kwargs)
-
- group = self.add_argument_group("template arguments")
- self.templates = {t: all_templates[t]() for t in self.templates}
- for template in self.templates.values():
- template.add_arguments(group)
-
- def validate(self, args):
- if hasattr(args, 'message'):
- if args.message == 'editor':
- if 'EDITOR' not in os.environ:
- self.error("must set the $EDITOR environment variable to use blank --message")
-
- with tempfile.NamedTemporaryFile(mode='r') as fh:
- subprocess.call([os.environ['EDITOR'], fh.name])
- args.message = fh.read().strip()
-
- if '{msg}' not in args.message:
- args.message = '{}\n\n{}'.format(args.message, '{msg}')
-
- def parse_known_args(self, *args, **kwargs):
- args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs)
- self.validate(args)
-
- if self.templates:
- args.templates = {}
- for cls in self.templates.itervalues():
- context = cls.context(**vars(args))
- if context is not None:
- args.templates.update(context)
-
- return args, remainder
deleted file mode 100644
--- a/tools/tryselect/preset.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import ConfigParser
-import os
-import subprocess
-
-from mozboot.util import get_state_dir
-
-
-CONFIG_PATH = os.path.join(get_state_dir()[0], "autotry.ini")
-
-
-def list_presets(section=None):
- config = ConfigParser.RawConfigParser()
-
- data = []
- if config.read([CONFIG_PATH]):
- sections = [section] if section else config.sections()
- for s in sections:
- try:
- data.extend(config.items(s))
- except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
- pass
-
- if not data:
- print("No presets found")
-
- for name, value in data:
- print("%s: %s" % (name, value))
-
-
-def edit_presets(section=None):
- if 'EDITOR' not in os.environ:
- print("error: must set the $EDITOR environment variable to use --edit-presets")
- return
- subprocess.call([os.environ['EDITOR'], CONFIG_PATH])
-
-
-def load(name, section=None):
- config = ConfigParser.RawConfigParser()
- if not config.read([CONFIG_PATH]):
- return
-
- sections = [section] if section else config.sections()
- for s in sections:
- try:
- return config.get(s, name), s
- except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
- pass
- return None, None
-
-
-def save(section, name, data):
- config = ConfigParser.RawConfigParser()
- config.read([CONFIG_PATH])
-
- if not config.has_section(section):
- config.add_section(section)
-
- config.set(section, name, data)
-
- with open(CONFIG_PATH, "w") as f:
- config.write(f)
-
- print('preset saved, run with: --preset={}'.format(name))
deleted file mode 100644
--- a/tools/tryselect/selectors/empty.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-from ..cli import BaseTryParser
-from ..vcs import VCSHelper
-
-
-class EmptyParser(BaseTryParser):
- name = 'empty'
- common_groups = ['push']
-
-
-def run_empty_try(message='{msg}', push=True, **kwargs):
- vcs = VCSHelper.create()
- msg = 'No try selector specified, use "Add New Jobs" to select tasks.'
- return vcs.push_to_try('empty', message.format(msg=msg), [], push=push,
- closed_tree=kwargs["closed_tree"])
deleted file mode 100644
--- a/tools/tryselect/selectors/fuzzy.py
+++ /dev/null
@@ -1,262 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import os
-import platform
-import re
-import subprocess
-import sys
-from distutils.spawn import find_executable
-
-from mozboot.util import get_state_dir
-from mozterm import Terminal
-from moztest.resolve import TestResolver, get_suite_definition
-
-from .. import preset as pset
-from ..cli import BaseTryParser
-from ..tasks import generate_tasks
-from ..vcs import VCSHelper
-
-terminal = Terminal()
-
-here = os.path.abspath(os.path.dirname(__file__))
-
-FZF_NOT_FOUND = """
-Could not find the `fzf` binary.
-
-The `mach try fuzzy` command depends on fzf. Please install it following the
-appropriate instructions for your platform:
-
- https://github.com/junegunn/fzf#installation
-
-Only the binary is required, if you do not wish to install the shell and
-editor integrations, download the appropriate binary and put it on your $PATH:
-
- https://github.com/junegunn/fzf-bin/releases
-""".lstrip()
-
-FZF_INSTALL_FAILED = """
-Failed to install fzf.
-
-Please install fzf manually following the appropriate instructions for your
-platform:
-
- https://github.com/junegunn/fzf#installation
-
-Only the binary is required, if you do not wish to install the shell and
-editor integrations, download the appropriate binary and put it on your $PATH:
-
- https://github.com/junegunn/fzf-bin/releases
-""".lstrip()
-
-FZF_HEADER = """
-For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf
-{shortcuts}
-""".strip()
-
-fzf_shortcuts = {
- 'ctrl-a': 'select-all',
- 'ctrl-d': 'deselect-all',
- 'ctrl-t': 'toggle-all',
- 'alt-bspace': 'beginning-of-line+kill-line',
- '?': 'toggle-preview',
-}
-
-fzf_header_shortcuts = {
- 'cursor-up': 'ctrl-k',
- 'cursor-down': 'ctrl-j',
- 'toggle-select': 'tab',
- 'select-all': 'ctrl-a',
- 'accept': 'enter',
- 'cancel': 'ctrl-c',
-}
-
-
-class FuzzyParser(BaseTryParser):
- name = 'fuzzy'
- arguments = [
- [['-q', '--query'],
- {'metavar': 'STR',
- 'help': "Use the given query instead of entering the selection "
- "interface. Equivalent to typing <query><ctrl-a><enter> "
- "from the interface.",
- }],
- [['-u', '--update'],
- {'action': 'store_true',
- 'default': False,
- 'help': "Update fzf before running.",
- }],
- ]
- common_groups = ['push', 'task', 'preset']
- templates = ['artifact', 'path', 'env', 'rebuild', 'chemspill-prio', 'talos-profile']
-
-
-def run(cmd, cwd=None):
- is_win = platform.system() == 'Windows'
- return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
-
-
-def run_fzf_install_script(fzf_path):
- if platform.system() == 'Windows':
- cmd = ['bash', '-c', './install --bin']
- else:
- cmd = ['./install', '--bin']
-
- if run(cmd, cwd=fzf_path):
- print(FZF_INSTALL_FAILED)
- sys.exit(1)
-
-
-def fzf_bootstrap(update=False):
- """Bootstrap fzf if necessary and return path to the executable.
-
- The bootstrap works by cloning the fzf repository and running the included
- `install` script. If update is True, we will pull the repository and re-run
- the install script.
- """
- fzf_bin = find_executable('fzf')
- if fzf_bin and not update:
- return fzf_bin
-
- fzf_path = os.path.join(get_state_dir()[0], 'fzf')
- if update and not os.path.isdir(fzf_path):
- print("fzf installed somewhere other than {}, please update manually".format(fzf_path))
- sys.exit(1)
-
- def get_fzf():
- return find_executable('fzf', os.path.join(fzf_path, 'bin'))
-
- if update:
- ret = run(['git', 'pull'], cwd=fzf_path)
- if ret:
- print("Update fzf failed.")
- sys.exit(1)
-
- run_fzf_install_script(fzf_path)
- return get_fzf()
-
- if os.path.isdir(fzf_path):
- fzf_bin = get_fzf()
- if fzf_bin:
- return fzf_bin
- # Fzf is cloned, but binary doesn't exist. Try running the install script
- return fzf_bootstrap(update=True)
-
- install = raw_input("Could not detect fzf, install it now? [y/n]: ")
- if install.lower() != 'y':
- return
-
- if not find_executable('git'):
- print("Git not found.")
- print(FZF_INSTALL_FAILED)
- sys.exit(1)
-
- cmd = ['git', 'clone', '--depth', '1', 'https://github.com/junegunn/fzf.git']
- if subprocess.call(cmd, cwd=os.path.dirname(fzf_path)):
- print(FZF_INSTALL_FAILED)
- sys.exit(1)
-
- run_fzf_install_script(fzf_path)
-
- print("Installed fzf to {}".format(fzf_path))
- return get_fzf()
-
-
-def format_header():
- shortcuts = []
- for action, key in sorted(fzf_header_shortcuts.iteritems()):
- shortcuts.append('{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}'.format(
- t=terminal, action=action, key=key))
- return FZF_HEADER.format(shortcuts=', '.join(shortcuts), t=terminal)
-
-
-def filter_by_paths(tasks, paths):
- resolver = TestResolver.from_environment(cwd=here)
- run_suites, run_tests = resolver.resolve_metadata(paths)
- flavors = set([(t['flavor'], t.get('subsuite')) for t in run_tests])
-
- task_regexes = set()
- for flavor, subsuite in flavors:
- suite = get_suite_definition(flavor, subsuite, strict=True)
- if 'task_regex' not in suite:
- print("warning: no tasks could be resolved from flavor '{}'{}".format(
- flavor, " and subsuite '{}'".format(subsuite) if subsuite else ""))
- continue
-
- task_regexes.update(suite['task_regex'])
-
- def match_task(task):
- return any(re.search(pattern, task) for pattern in task_regexes)
-
- return filter(match_task, tasks)
-
-
-def run_fuzzy_try(update=False, query=None, templates=None, full=False, parameters=None,
- save=False, preset=None, mod_presets=False, push=True, message='{msg}',
- paths=None, **kwargs):
- if mod_presets:
- return getattr(pset, mod_presets)(section='fuzzy')
-
- fzf = fzf_bootstrap(update)
-
- if not fzf:
- print(FZF_NOT_FOUND)
- return 1
-
- vcs = VCSHelper.create()
- vcs.check_working_directory(push)
-
- all_tasks = generate_tasks(parameters, full, root=vcs.root)
-
- if paths:
- all_tasks = filter_by_paths(all_tasks, paths)
- if not all_tasks:
- return 1
-
- key_shortcuts = [k + ':' + v for k, v in fzf_shortcuts.iteritems()]
- cmd = [
- fzf, '-m',
- '--bind', ','.join(key_shortcuts),
- '--header', format_header(),
- # Using python to split the preview string is a bit convoluted,
- # but is guaranteed to be available on all platforms.
- '--preview', 'python -c "print(\\"\\n\\".join(sorted([s.strip(\\"\'\\") for s in \\"{+}\\".split()])))"', # noqa
- '--preview-window=right:20%',
- '--print-query',
- ]
-
- if query:
- cmd.extend(['-f', query])
- elif preset:
- value = pset.load(preset, section='fuzzy')[0]
- cmd.extend(['-f', value])
-
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
- out = proc.communicate('\n'.join(all_tasks))[0].splitlines()
-
- selected = []
- if out:
- query = out[0]
- selected = out[1:]
-
- if not selected:
- print("no tasks selected")
- return
-
- if save:
- pset.save('fuzzy', save, query)
-
- # build commit message
- msg = "Fuzzy"
- args = []
- if paths:
- args.append("paths={}".format(':'.join(paths)))
- if query:
- args.append("query={}".format(query))
- if args:
- msg = "{} {}".format(msg, '&'.join(args))
- return vcs.push_to_try('fuzzy', message.format(msg=msg), selected, templates, push=push,
- closed_tree=kwargs["closed_tree"])
deleted file mode 100644
--- a/tools/tryselect/selectors/syntax.py
+++ /dev/null
@@ -1,629 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import os
-import re
-import sys
-from collections import defaultdict
-
-import mozpack.path as mozpath
-from moztest.resolve import TestResolver
-
-from .. import preset
-from ..cli import BaseTryParser
-from ..vcs import VCSHelper
-
-here = os.path.abspath(os.path.dirname(__file__))
-
-
-class SyntaxParser(BaseTryParser):
- name = 'syntax'
- arguments = [
- [['paths'],
- {'nargs': '*',
- 'help': 'Paths to search for tests to run on try.',
- }],
- [['-b', '--build'],
- {'dest': 'builds',
- 'default': 'do',
- 'help': 'Build types to run (d for debug, o for optimized).',
- }],
- [['-p', '--platform'],
- {'dest': 'platforms',
- 'action': 'append',
- 'help': 'Platforms to run (required if not found in the environment as '
- 'AUTOTRY_PLATFORM_HINT).',
- }],
- [['-u', '--unittests'],
- {'dest': 'tests',
- 'action': 'append',
- 'help': 'Test suites to run in their entirety.',
- }],
- [['-t', '--talos'],
- {'action': 'append',
- 'help': 'Talos suites to run.',
- }],
- [['-j', '--jobs'],
- {'action': 'append',
- 'help': 'Job tasks to run.',
- }],
- [['--tag'],
- {'dest': 'tags',
- 'action': 'append',
- 'help': 'Restrict tests to the given tag (may be specified multiple times).',
- }],
- [['--and'],
- {'action': 'store_true',
- 'dest': 'intersection',
- 'help': 'When -u and paths are supplied run only the intersection of the '
- 'tests specified by the two arguments.',
- }],
- [['--no-artifact'],
- {'action': 'store_true',
- 'help': 'Disable artifact builds even if --enable-artifact-builds is set '
- 'in the mozconfig.',
- }],
- [['-v', '--verbose'],
- {'dest': 'verbose',
- 'action': 'store_true',
- 'default': False,
- 'help': 'Print detailed information about the resulting test selection '
- 'and commands performed.',
- }],
- [['--detect-paths'],
- {'dest': 'detect_paths',
- 'action': 'store_true',
- 'default': False,
- 'help': 'Provide test paths based on files changed in the working copy.',
- }],
- ]
-
- # Arguments we will accept on the command line and pass through to try
- # syntax with no further intervention. The set is taken from
- # http://trychooser.pub.build.mozilla.org with a few additions.
- #
- # Note that the meaning of store_false and store_true arguments is
- # not preserved here, as we're only using these to echo the literal
- # arguments to another consumer. Specifying either store_false or
- # store_true here will have an equivalent effect.
- pass_through_arguments = {
- '--rebuild': {
- 'action': 'store',
- 'dest': 'rebuild',
- 'help': 'Re-trigger all test jobs (up to 20 times)',
- },
- '--rebuild-talos': {
- 'action': 'store',
- 'dest': 'rebuild_talos',
- 'help': 'Re-trigger all talos jobs',
- },
- '--interactive': {
- 'action': 'store_true',
- 'dest': 'interactive',
- 'help': 'Allow ssh-like access to running test containers',
- },
- '--no-retry': {
- 'action': 'store_true',
- 'dest': 'no_retry',
- 'help': 'Do not retrigger failed tests',
- },
- '--setenv': {
- 'action': 'append',
- 'dest': 'setenv',
- 'help': 'Set the corresponding variable in the test environment for'
- 'applicable harnesses.',
- },
- '-f': {
- 'action': 'store_true',
- 'dest': 'failure_emails',
- 'help': 'Request failure emails only',
- },
- '--failure-emails': {
- 'action': 'store_true',
- 'dest': 'failure_emails',
- 'help': 'Request failure emails only',
- },
- '-e': {
- 'action': 'store_true',
- 'dest': 'all_emails',
- 'help': 'Request all emails',
- },
- '--all-emails': {
- 'action': 'store_true',
- 'dest': 'all_emails',
- 'help': 'Request all emails',
- },
- '--artifact': {
- 'action': 'store_true',
- 'dest': 'artifact',
- 'help': 'Force artifact builds where possible.',
- },
- '--upload-xdbs': {
- 'action': 'store_true',
- 'dest': 'upload_xdbs',
- 'help': 'Upload XDB compilation db files generated by hazard build',
- },
- }
- templates = ['chemspill-prio']
-
- def __init__(self, *args, **kwargs):
- BaseTryParser.__init__(self, *args, **kwargs)
-
- group = self.add_argument_group("pass-through arguments")
- for arg, opts in self.pass_through_arguments.items():
- group.add_argument(arg, **opts)
-
-
-class TryArgumentTokenizer(object):
- symbols = [("seperator", ","),
- ("list_start", "\["),
- ("list_end", "\]"),
- ("item", "([^,\[\]\s][^,\[\]]+)"),
- ("space", "\s+")]
- token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
-
- def tokenize(self, data):
- for match in self.token_re.finditer(data):
- symbol = match.lastgroup
- data = match.group(symbol)
- if symbol == "space":
- pass
- else:
- yield symbol, data
-
-
-class TryArgumentParser(object):
- """Simple three-state parser for handling expressions
- of the from "foo[sub item, another], bar,baz". This takes
- input from the TryArgumentTokenizer and runs through a small
- state machine, returning a dictionary of {top-level-item:[sub_items]}
- i.e. the above would result in
- {"foo":["sub item", "another"], "bar": [], "baz": []}
- In the case of invalid input a ValueError is raised."""
-
- EOF = object()
-
- def __init__(self):
- self.reset()
-
- def reset(self):
- self.tokens = None
- self.current_item = None
- self.data = {}
- self.token = None
- self.state = None
-
- def parse(self, tokens):
- self.reset()
- self.tokens = tokens
- self.consume()
- self.state = self.item_state
- while self.token[0] != self.EOF:
- self.state()
- return self.data
-
- def consume(self):
- try:
- self.token = self.tokens.next()
- except StopIteration:
- self.token = (self.EOF, None)
-
- def expect(self, *types):
- if self.token[0] not in types:
- raise ValueError("Error parsing try string, unexpected %s" % (self.token[0]))
-
- def item_state(self):
- self.expect("item")
- value = self.token[1].strip()
- if value not in self.data:
- self.data[value] = []
- self.current_item = value
- self.consume()
- if self.token[0] == "seperator":
- self.consume()
- elif self.token[0] == "list_start":
- self.consume()
- self.state = self.subitem_state
- elif self.token[0] == self.EOF:
- pass
- else:
- raise ValueError
-
- def subitem_state(self):
- self.expect("item")
- value = self.token[1].strip()
- self.data[self.current_item].append(value)
- self.consume()
- if self.token[0] == "seperator":
- self.consume()
- elif self.token[0] == "list_end":
- self.consume()
- self.state = self.after_list_end_state
- else:
- raise ValueError
-
- def after_list_end_state(self):
- self.expect("seperator")
- self.consume()
- self.state = self.item_state
-
-
-def parse_arg(arg):
- tokenizer = TryArgumentTokenizer()
- parser = TryArgumentParser()
- return parser.parse(tokenizer.tokenize(arg))
-
-
-class AutoTry(object):
-
- # Maps from flavors to the job names needed to run that flavour
- flavor_jobs = {
- 'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
- 'xpcshell': ['xpcshell'],
- 'chrome': ['mochitest-o'],
- 'browser-chrome': ['mochitest-browser-chrome-1',
- 'mochitest-e10s-browser-chrome-1',
- 'mochitest-browser-chrome-e10s-1'],
- 'devtools-chrome': ['mochitest-devtools-chrome-1',
- 'mochitest-e10s-devtools-chrome-1',
- 'mochitest-devtools-chrome-e10s-1'],
- 'crashtest': ['crashtest', 'crashtest-e10s'],
- 'reftest': ['reftest', 'reftest-e10s'],
- 'web-platform-tests': ['web-platform-tests-1'],
- }
-
- flavor_suites = {
- "mochitest": "mochitests",
- "xpcshell": "xpcshell",
- "chrome": "mochitest-o",
- "browser-chrome": "mochitest-bc",
- "devtools-chrome": "mochitest-dt",
- "crashtest": "crashtest",
- "reftest": "reftest",
- "web-platform-tests": "web-platform-tests",
- }
-
- compiled_suites = [
- "cppunit",
- "gtest",
- "jittest",
- ]
-
- common_suites = [
- "cppunit",
- "crashtest",
- "firefox-ui-functional",
- "geckoview",
- "geckoview-junit",
- "gtest",
- "jittest",
- "jsreftest",
- "marionette",
- "marionette-e10s",
- "mochitests",
- "reftest",
- "robocop",
- "web-platform-tests",
- "xpcshell",
- ]
-
- def __init__(self, topsrcdir, mach_context):
- self.topsrcdir = topsrcdir
- self._resolver = None
- self.mach_context = mach_context
- self.vcs = VCSHelper.create()
-
- @property
- def resolver(self):
- if self._resolver is None:
- self._resolver = TestResolver.from_environment(cwd=here)
- return self._resolver
-
- def split_try_string(self, data):
- return re.findall(r'(?:\[.*?\]|\S)+', data)
-
- def paths_by_flavor(self, paths=None, tags=None):
- paths_by_flavor = defaultdict(set)
-
- if not (paths or tags):
- return dict(paths_by_flavor)
-
- tests = list(self.resolver.resolve_tests(paths=paths,
- tags=tags))
-
- for t in tests:
- if t['flavor'] in self.flavor_suites:
- flavor = t['flavor']
- if 'subsuite' in t and t['subsuite'] == 'devtools':
- flavor = 'devtools-chrome'
-
- if flavor in ['crashtest', 'reftest']:
- manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
- paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath))
- elif 'dir_relpath' in t:
- paths_by_flavor[flavor].add(t['dir_relpath'])
- else:
- file_relpath = os.path.relpath(t['path'], self.topsrcdir)
- dir_relpath = os.path.dirname(file_relpath)
- paths_by_flavor[flavor].add(dir_relpath)
-
- for flavor, path_set in paths_by_flavor.items():
- paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths)
-
- return dict(paths_by_flavor)
-
- def deduplicate_prefixes(self, path_set, input_paths):
- # Removes paths redundant to test selection in the given path set.
- # If a path was passed on the commandline that is the prefix of a
- # path in our set, we only need to include the specified prefix to
- # run the intended tests (every test in "layout/base" will run if
- # "layout" is passed to the reftest harness).
- removals = set()
- additions = set()
-
- for path in path_set:
- full_path = path
- while path:
- path, _ = os.path.split(path)
- if path in input_paths:
- removals.add(full_path)
- additions.add(path)
-
- return additions | (path_set - removals)
-
- def remove_duplicates(self, paths_by_flavor, tests):
- rv = {}
- for item in paths_by_flavor:
- if self.flavor_suites[item] not in tests:
- rv[item] = paths_by_flavor[item].copy()
- return rv
-
- def calc_try_syntax(self, platforms, tests, talos, jobs, builds, paths_by_flavor, tags,
- extras, intersection):
- parts = ["try:"]
-
- if platforms:
- parts.extend(["-b", builds, "-p", ",".join(platforms)])
-
- suites = tests if not intersection else {}
- paths = set()
- for flavor, flavor_tests in paths_by_flavor.iteritems():
- suite = self.flavor_suites[flavor]
- if suite not in suites and (not intersection or suite in tests):
- for job_name in self.flavor_jobs[flavor]:
- for test in flavor_tests:
- paths.add("%s:%s" % (flavor, test))
- suites[job_name] = tests.get(suite, [])
-
- # intersection implies tests are expected
- if intersection and not suites:
- raise ValueError("No tests found matching filters")
-
- if extras.get('artifact') and any([p.endswith("-nightly") for p in platforms]):
- print('You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. '
- 'Running without |--artifact| instead.')
- del extras['artifact']
-
- if extras.get('artifact'):
- rejected = []
- for suite in suites.keys():
- if any([suite.startswith(c) for c in self.compiled_suites]):
- rejected.append(suite)
- if rejected:
- raise ValueError("You can't run {} with "
- "--artifact option.".format(', '.join(rejected)))
-
- if extras.get('artifact') and 'all' in suites.keys():
- non_compiled_suites = set(self.common_suites) - set(self.compiled_suites)
- message = ('You asked for |-u all| with |--artifact| but compiled-code tests ({tests})'
- ' can\'t run against an artifact build. Running (-u {non_compiled_suites}) '
- 'instead.')
- string_format = {
- 'tests': ','.join(self.compiled_suites),
- 'non_compiled_suites': ','.join(non_compiled_suites),
- }
- print(message.format(**string_format))
- del suites['all']
- suites.update({suite_name: None for suite_name in non_compiled_suites})
-
- if suites:
- parts.append("-u")
- parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
- for k, v in sorted(suites.items())))
-
- if talos:
- parts.append("-t")
- parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
- for k, v in sorted(talos.items())))
-
- if jobs:
- parts.append("-j")
- parts.append(",".join(jobs))
-
- if tags:
- parts.append(' '.join('--tag %s' % t for t in tags))
-
- if paths:
- parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
-
- args_by_dest = {v['dest']: k for k, v in SyntaxParser.pass_through_arguments.items()}
- for dest, value in extras.iteritems():
- assert dest in args_by_dest
- arg = args_by_dest[dest]
- action = SyntaxParser.pass_through_arguments[arg]['action']
- if action == 'store':
- parts.append(arg)
- parts.append(value)
- if action == 'append':
- for e in value:
- parts.append(arg)
- parts.append(e)
- if action in ('store_true', 'store_false'):
- parts.append(arg)
-
- return " ".join(parts)
-
- def normalise_list(self, items, allow_subitems=False):
- rv = defaultdict(list)
- for item in items:
- parsed = parse_arg(item)
- for key, values in parsed.iteritems():
- rv[key].extend(values)
-
- if not allow_subitems:
- if not all(item == [] for item in rv.itervalues()):
- raise ValueError("Unexpected subitems in argument")
- return rv.keys()
- else:
- return rv
-
- def validate_args(self, **kwargs):
- tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"]
- if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected):
- if 'AUTOTRY_PLATFORM_HINT' in os.environ:
- kwargs["platforms"] = [os.environ['AUTOTRY_PLATFORM_HINT']]
- elif tests_selected:
- print("Must specify platform when selecting tests.")
- sys.exit(1)
- else:
- print("Either platforms or jobs must be specified as an argument to autotry.")
- sys.exit(1)
-
- try:
- platforms = (self.normalise_list(kwargs["platforms"])
- if kwargs["platforms"] else {})
- except ValueError as e:
- print("Error parsing -p argument:\n%s" % e.message)
- sys.exit(1)
-
- try:
- tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
- if kwargs["tests"] else {})
- except ValueError as e:
- print("Error parsing -u argument (%s):\n%s" % (kwargs["tests"], e.message))
- sys.exit(1)
-
- try:
- talos = (self.normalise_list(kwargs["talos"], allow_subitems=True)
- if kwargs["talos"] else [])
- except ValueError as e:
- print("Error parsing -t argument:\n%s" % e.message)
- sys.exit(1)
-
- try:
- jobs = (self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {})
- except ValueError as e:
- print("Error parsing -j argument:\n%s" % e.message)
- sys.exit(1)
-
- paths = []
- for p in kwargs["paths"]:
- p = mozpath.normpath(os.path.abspath(p))
- if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
- print('Specified path "%s" is not a directory under the srcdir,'
- ' unable to specify tests outside of the srcdir' % p)
- sys.exit(1)
- if len(p) <= len(self.topsrcdir):
- print('Specified path "%s" is at the top of the srcdir and would'
- ' select all tests.' % p)
- sys.exit(1)
- paths.append(os.path.relpath(p, self.topsrcdir))
-
- try:
- tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
- except ValueError as e:
- print("Error parsing --tags argument:\n%s" % e.message)
- sys.exit(1)
-
- extra_values = {k['dest'] for k in SyntaxParser.pass_through_arguments.values()}
- extra_args = {k: v for k, v in kwargs.items()
- if k in extra_values and v}
-
- return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args
-
- def run(self, **kwargs):
- if kwargs["mod_presets"]:
- getattr(preset, kwargs["mod_presets"])(section='try')
- sys.exit()
-
- if kwargs["preset"]:
- value = preset.load(kwargs["preset"], section='try')[0]
- defaults = vars(SyntaxParser().parse_args(self.split_try_string(value)))
-
- if defaults is None:
- print("No saved configuration called %s found in autotry.ini" % kwargs["preset"],
- file=sys.stderr)
-
- for key, value in kwargs.iteritems():
- if value in (None, []) and key in defaults:
- kwargs[key] = defaults[key]
-
- if not any(kwargs[item] for item in ("paths", "tests", "tags")):
- if kwargs['detect_paths']:
- res = self.resolver.get_outgoing_metadata()
- kwargs['paths'] = res['paths']
- kwargs['tags'] = res['tags']
- else:
- kwargs['paths'] = set()
- kwargs['tags'] = set()
-
- builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args(**kwargs)
-
- if paths or tags:
- paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
- for item in paths]
- paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags)
-
- if not paths_by_flavor and not tests:
- print("No tests were found when attempting to resolve paths:\n\n\t%s" %
- paths)
- sys.exit(1)
-
- if not kwargs["intersection"]:
- paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests)
- else:
- paths_by_flavor = {}
-
- # No point in dealing with artifacts if we aren't running any builds
- local_artifact_build = False
- if platforms:
- local_artifact_build = kwargs.get('local_artifact_build', False)
-
- # Add --artifact if --enable-artifact-builds is set ...
- if local_artifact_build:
- extra["artifact"] = True
- # ... unless --no-artifact is explicitly given.
- if kwargs["no_artifact"]:
- if "artifact" in extra:
- del extra["artifact"]
-
- try:
- msg = self.calc_try_syntax(platforms, tests, talos, jobs, builds,
- paths_by_flavor, tags, extra, kwargs["intersection"])
- except ValueError as e:
- print(e.message)
- sys.exit(1)
-
- if local_artifact_build and not kwargs["no_artifact"]:
- print('mozconfig has --enable-artifact-builds; including '
- '--artifact flag in try syntax (use --no-artifact '
- 'to override)')
-
- if kwargs["verbose"] and paths_by_flavor:
- print('The following tests will be selected: ')
- for flavor, paths in paths_by_flavor.iteritems():
- print("%s: %s" % (flavor, ",".join(paths)))
-
- if kwargs["verbose"]:
- print('The following try syntax was calculated:\n%s' % msg)
-
- self.vcs.push_to_try('syntax', kwargs["message"].format(msg=msg), push=kwargs['push'],
- closed_tree=kwargs["closed_tree"])
-
- if kwargs["save"]:
- assert msg.startswith("try: ")
- msg = msg[len("try: "):]
- preset.save('try', kwargs["save"], msg)
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/setup.py
@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from setuptools import setup
+
+setup(
+ author='Mozilla Foundation',
+ author_email='dev-builds@lists.mozilla.org',
+ name='tryselect',
+ description='',
+ license='MPL 2.0',
+ packages=['tryselect', 'tryselect.selectors'],
+ version='1.0.0',
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Topic :: Software Development :: Build Tools',
+ 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: Implementation :: CPython',
+ ],
+ keywords='mozilla try tryselect',
+)
deleted file mode 100644
--- a/tools/tryselect/tasks.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import os
-import sys
-
-from mozboot.util import get_state_dir
-from mozbuild.base import MozbuildObject
-from mozpack.files import FileFinder
-
-import taskgraph
-from taskgraph.generator import TaskGraphGenerator
-from taskgraph.parameters import (
- ParameterMismatch,
- load_parameters_file,
-)
-
-here = os.path.abspath(os.path.dirname(__file__))
-build = MozbuildObject.from_environment(cwd=here)
-
-
-PARAMETER_MISMATCH = """
-ERROR - The parameters being used to generate tasks differ from those defined
-in your working copy:
-
- {}
-
-To fix this, either rebase onto the latest mozilla-central or pass in
--p/--parameters. For more information on how to define parameters, see:
-https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters
-"""
-
-
-def invalidate(cache, root):
- if not os.path.isfile(cache):
- return
-
- tc_dir = os.path.join(root, 'taskcluster')
- tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
- cmod = os.path.getmtime(cache)
-
- if tmod > cmod:
- os.remove(cache)
-
-
-def generate_tasks(params, full, root):
- params = params or "project=mozilla-central"
-
- cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
- attr = 'full_task_set' if full else 'target_task_set'
- cache = os.path.join(cache_dir, attr)
-
- invalidate(cache, root)
- if os.path.isfile(cache):
- with open(cache, 'r') as fh:
- return fh.read().splitlines()
-
- if not os.path.isdir(cache_dir):
- os.makedirs(cache_dir)
-
- print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
- try:
- params = load_parameters_file(params, strict=False)
- params.check()
- except ParameterMismatch as e:
- print(PARAMETER_MISMATCH.format(e.args[0]))
- sys.exit(1)
-
- taskgraph.fast = True
- cwd = os.getcwd()
- os.chdir(build.topsrcdir)
-
- root = os.path.join(root, 'taskcluster', 'ci')
- tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
- labels = [label for label in tg.graph.visit_postorder()]
-
- os.chdir(cwd)
-
- with open(cache, 'w') as fh:
- fh.write('\n'.join(labels))
- return labels
deleted file mode 100644
--- a/tools/tryselect/templates.py
+++ /dev/null
@@ -1,169 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-Templates provide a way of modifying the task definition of selected
-tasks. They live under taskcluster/taskgraph/templates.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import os
-import sys
-from abc import ABCMeta, abstractmethod
-from argparse import Action, SUPPRESS
-
-import mozpack.path as mozpath
-from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
-
-here = os.path.abspath(os.path.dirname(__file__))
-build = MozbuildObject.from_environment(cwd=here)
-
-
-class Template(object):
- __metaclass__ = ABCMeta
-
- @abstractmethod
- def add_arguments(self, parser):
- pass
-
- @abstractmethod
- def context(self, **kwargs):
- pass
-
-
-class Artifact(Template):
-
- def add_arguments(self, parser):
- group = parser.add_mutually_exclusive_group()
- group.add_argument('--artifact', action='store_true',
- help='Force artifact builds where possible.')
- group.add_argument('--no-artifact', action='store_true',
- help='Disable artifact builds even if being used locally.')
-
- def context(self, artifact, no_artifact, **kwargs):
- if artifact:
- return {
- 'artifact': {'enabled': '1'}
- }
-
- if no_artifact:
- return
-
- try:
- if build.substs.get("MOZ_ARTIFACT_BUILDS"):
- print("Artifact builds enabled, pass --no-artifact to disable")
- return {
- 'artifact': {'enabled': '1'}
- }
- except BuildEnvironmentNotFoundException:
- pass
-
-
-class Path(Template):
-
- def add_arguments(self, parser):
- parser.add_argument('paths', nargs='*',
- help='Run tasks containing tests under the specified path(s).')
-
- def context(self, paths, **kwargs):
- if not paths:
- return
-
- for p in paths:
- if not os.path.exists(p):
- print("error: '{}' is not a valid path.".format(p), file=sys.stderr)
- sys.exit(1)
-
- paths = [mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir) for p in paths]
- return {
- 'env': {
- # can't use os.pathsep as machine splitting could be a different platform
- 'MOZHARNESS_TEST_PATHS': ':'.join(paths),
- }
- }
-
-
-class Environment(Template):
-
- def add_arguments(self, parser):
- parser.add_argument('--env', action='append', default=None,
- help='Set an environment variable, of the form FOO=BAR. '
- 'Can be passed in multiple times.')
-
- def context(self, env, **kwargs):
- if not env:
- return
- return {
- 'env': dict(e.split('=', 1) for e in env),
- }
-
-
-class RangeAction(Action):
- def __init__(self, min, max, *args, **kwargs):
- self.min = min
- self.max = max
- kwargs['metavar'] = '[{}-{}]'.format(self.min, self.max)
- super(RangeAction, self).__init__(*args, **kwargs)
-
- def __call__(self, parser, namespace, values, option_string=None):
- name = option_string or self.dest
- if values < self.min:
- parser.error('{} can not be less than {}'.format(name, self.min))
- if values > self.max:
- parser.error('{} can not be more than {}'.format(name, self.max))
- setattr(namespace, self.dest, values)
-
-
-class Rebuild(Template):
-
- def add_arguments(self, parser):
- parser.add_argument('--rebuild', action=RangeAction, min=2, max=20, default=None, type=int,
- help='Rebuild all selected tasks the specified number of times.')
-
- def context(self, rebuild, **kwargs):
- if not rebuild:
- return
-
- return {
- 'rebuild': rebuild,
- }
-
-
-class ChemspillPrio(Template):
-
- def add_arguments(self, parser):
- parser.add_argument('--chemspill-prio', action='store_true',
- help='Run at a higher priority than most try jobs (chemspills only).')
-
- def context(self, chemspill_prio, **kwargs):
- if chemspill_prio:
- return {
- 'chemspill-prio': {}
- }
-
-
-class TalosProfile(Template):
-
- def add_arguments(self, parser):
- parser.add_argument('--talos-profile', dest='profile', action='store_true', default=False,
- help='Create and upload a gecko profile during talos tasks.')
- # This is added for consistency with the 'syntax' selector
- parser.add_argument('--geckoProfile', dest='profile', action='store_true', default=False,
- help=SUPPRESS)
-
- def context(self, profile, **kwargs):
- if not profile:
- return
- return {'talos-profile': profile}
-
-
-all_templates = {
- 'artifact': Artifact,
- 'path': Path,
- 'env': Environment,
- 'rebuild': Rebuild,
- 'chemspill-prio': ChemspillPrio,
- 'talos-profile': TalosProfile,
-}
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/tryselect/cli.py
@@ -0,0 +1,125 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import subprocess
+import tempfile
+from argparse import ArgumentParser
+
+from .templates import all_templates
+
+
+COMMON_ARGUMENT_GROUPS = {
+ 'push': [
+ [['-m', '--message'],
+ {'const': 'editor',
+ 'default': '{msg}',
+ 'nargs': '?',
+ 'help': 'Use the specified commit message, or create it in your '
+ '$EDITOR if blank. Defaults to computed message.',
+ }],
+ [['--no-push'],
+ {'dest': 'push',
+ 'action': 'store_false',
+ 'help': 'Do not push to try as a result of running this command (if '
+ 'specified this command will only print calculated try '
+ 'syntax and selection info).',
+ }],
+ [['--closed-tree'],
+ {'action': 'store_true',
+ 'default': False,
+ 'help': 'Push despite a closed try tree',
+ }],
+ ],
+ 'preset': [
+ [['--save'],
+ {'default': None,
+ 'help': 'Save selection for future use with --preset.',
+ }],
+ [['--preset'],
+ {'default': None,
+ 'help': 'Load a saved selection.',
+ }],
+ [['--list-presets'],
+ {'action': 'store_const',
+ 'const': 'list_presets',
+ 'dest': 'mod_presets',
+ 'default': None,
+ 'help': 'List available preset selections.',
+ }],
+ [['--edit-presets'],
+ {'action': 'store_const',
+ 'const': 'edit_presets',
+ 'dest': 'mod_presets',
+ 'default': None,
+ 'help': 'Edit the preset file.',
+ }],
+ ],
+ 'task': [
+ [['--full'],
+ {'action': 'store_true',
+ 'default': False,
+ 'help': "Use the full set of tasks as input to fzf (instead of "
+ "target tasks).",
+ }],
+ [['-p', '--parameters'],
+ {'default': None,
+ 'help': "Use the given parameters.yml to generate tasks, "
+ "defaults to latest parameters.yml from mozilla-central",
+ }],
+ ],
+}
+
+
+class BaseTryParser(ArgumentParser):
+ name = 'try'
+ common_groups = ['push', 'preset']
+ arguments = []
+ templates = []
+
+ def __init__(self, *args, **kwargs):
+ ArgumentParser.__init__(self, *args, **kwargs)
+
+ group = self.add_argument_group("{} arguments".format(self.name))
+ for cli, kwargs in self.arguments:
+ group.add_argument(*cli, **kwargs)
+
+ for name in self.common_groups:
+ group = self.add_argument_group("{} arguments".format(name))
+ arguments = COMMON_ARGUMENT_GROUPS[name]
+ for cli, kwargs in arguments:
+ group.add_argument(*cli, **kwargs)
+
+ group = self.add_argument_group("template arguments")
+ self.templates = {t: all_templates[t]() for t in self.templates}
+ for template in self.templates.values():
+ template.add_arguments(group)
+
+ def validate(self, args):
+ if hasattr(args, 'message'):
+ if args.message == 'editor':
+ if 'EDITOR' not in os.environ:
+ self.error("must set the $EDITOR environment variable to use blank --message")
+
+ with tempfile.NamedTemporaryFile(mode='r') as fh:
+ subprocess.call([os.environ['EDITOR'], fh.name])
+ args.message = fh.read().strip()
+
+ if '{msg}' not in args.message:
+ args.message = '{}\n\n{}'.format(args.message, '{msg}')
+
+ def parse_known_args(self, *args, **kwargs):
+ args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs)
+ self.validate(args)
+
+ if self.templates:
+ args.templates = {}
+ for cls in self.templates.itervalues():
+ context = cls.context(**vars(args))
+ if context is not None:
+ args.templates.update(context)
+
+ return args, remainder
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/tryselect/preset.py
@@ -0,0 +1,69 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import ConfigParser
+import os
+import subprocess
+
+from mozboot.util import get_state_dir
+
+
+CONFIG_PATH = os.path.join(get_state_dir()[0], "autotry.ini")
+
+
+def list_presets(section=None):
+ config = ConfigParser.RawConfigParser()
+
+ data = []
+ if config.read([CONFIG_PATH]):
+ sections = [section] if section else config.sections()
+ for s in sections:
+ try:
+ data.extend(config.items(s))
+ except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+ pass
+
+ if not data:
+ print("No presets found")
+
+ for name, value in data:
+ print("%s: %s" % (name, value))
+
+
+def edit_presets(section=None):
+ if 'EDITOR' not in os.environ:
+ print("error: must set the $EDITOR environment variable to use --edit-presets")
+ return
+ subprocess.call([os.environ['EDITOR'], CONFIG_PATH])
+
+
+def load(name, section=None):
+ config = ConfigParser.RawConfigParser()
+ if not config.read([CONFIG_PATH]):
+ return
+
+ sections = [section] if section else config.sections()
+ for s in sections:
+ try:
+ return config.get(s, name), s
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ pass
+ return None, None
+
+
+def save(section, name, data):
+ config = ConfigParser.RawConfigParser()
+ config.read([CONFIG_PATH])
+
+ if not config.has_section(section):
+ config.add_section(section)
+
+ config.set(section, name, data)
+
+ with open(CONFIG_PATH, "w") as f:
+ config.write(f)
+
+ print('preset saved, run with: --preset={}'.format(name))
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/tryselect/selectors/empty.py
@@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from ..cli import BaseTryParser
+from ..vcs import VCSHelper
+
+
+class EmptyParser(BaseTryParser):
+ name = 'empty'
+ common_groups = ['push']
+
+
+def run_empty_try(message='{msg}', push=True, **kwargs):
+ vcs = VCSHelper.create()
+ msg = 'No try selector specified, use "Add New Jobs" to select tasks.'
+ return vcs.push_to_try('empty', message.format(msg=msg), [], push=push,
+ closed_tree=kwargs["closed_tree"])
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/tryselect/selectors/fuzzy.py
@@ -0,0 +1,262 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import platform
+import re
+import subprocess
+import sys
+from distutils.spawn import find_executable
+
+from mozboot.util import get_state_dir
+from mozterm import Terminal
+from moztest.resolve import TestResolver, get_suite_definition
+
+from .. import preset as pset
+from ..cli import BaseTryParser
+from ..tasks import generate_tasks
+from ..vcs import VCSHelper
+
+terminal = Terminal()
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+FZF_NOT_FOUND = """
+Could not find the `fzf` binary.
+
+The `mach try fuzzy` command depends on fzf. Please install it following the
+appropriate instructions for your platform:
+
+ https://github.com/junegunn/fzf#installation
+
+Only the binary is required, if you do not wish to install the shell and
+editor integrations, download the appropriate binary and put it on your $PATH:
+
+ https://github.com/junegunn/fzf-bin/releases
+""".lstrip()
+
+FZF_INSTALL_FAILED = """
+Failed to install fzf.
+
+Please install fzf manually following the appropriate instructions for your
+platform:
+
+ https://github.com/junegunn/fzf#installation
+
+Only the binary is required, if you do not wish to install the shell and
+editor integrations, download the appropriate binary and put it on your $PATH:
+
+ https://github.com/junegunn/fzf-bin/releases
+""".lstrip()
+
+FZF_HEADER = """
+For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf
+{shortcuts}
+""".strip()
+
+fzf_shortcuts = {
+ 'ctrl-a': 'select-all',
+ 'ctrl-d': 'deselect-all',
+ 'ctrl-t': 'toggle-all',
+ 'alt-bspace': 'beginning-of-line+kill-line',
+ '?': 'toggle-preview',
+}
+
+fzf_header_shortcuts = {
+ 'cursor-up': 'ctrl-k',
+ 'cursor-down': 'ctrl-j',
+ 'toggle-select': 'tab',
+ 'select-all': 'ctrl-a',
+ 'accept': 'enter',
+ 'cancel': 'ctrl-c',
+}
+
+
+class FuzzyParser(BaseTryParser):
+ name = 'fuzzy'
+ arguments = [
+ [['-q', '--query'],
+ {'metavar': 'STR',
+ 'help': "Use the given query instead of entering the selection "
+ "interface. Equivalent to typing <query><ctrl-a><enter> "
+ "from the interface.",
+ }],
+ [['-u', '--update'],
+ {'action': 'store_true',
+ 'default': False,
+ 'help': "Update fzf before running.",
+ }],
+ ]
+ common_groups = ['push', 'task', 'preset']
+ templates = ['artifact', 'path', 'env', 'rebuild', 'chemspill-prio', 'talos-profile']
+
+
+def run(cmd, cwd=None):
+ is_win = platform.system() == 'Windows'
+ return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
+
+
+def run_fzf_install_script(fzf_path):
+ if platform.system() == 'Windows':
+ cmd = ['bash', '-c', './install --bin']
+ else:
+ cmd = ['./install', '--bin']
+
+ if run(cmd, cwd=fzf_path):
+ print(FZF_INSTALL_FAILED)
+ sys.exit(1)
+
+
+def fzf_bootstrap(update=False):
+ """Bootstrap fzf if necessary and return path to the executable.
+
+ The bootstrap works by cloning the fzf repository and running the included
+ `install` script. If update is True, we will pull the repository and re-run
+ the install script.
+ """
+ fzf_bin = find_executable('fzf')
+ if fzf_bin and not update:
+ return fzf_bin
+
+ fzf_path = os.path.join(get_state_dir()[0], 'fzf')
+ if update and not os.path.isdir(fzf_path):
+ print("fzf installed somewhere other than {}, please update manually".format(fzf_path))
+ sys.exit(1)
+
+ def get_fzf():
+ return find_executable('fzf', os.path.join(fzf_path, 'bin'))
+
+ if update:
+ ret = run(['git', 'pull'], cwd=fzf_path)
+ if ret:
+ print("Update fzf failed.")
+ sys.exit(1)
+
+ run_fzf_install_script(fzf_path)
+ return get_fzf()
+
+ if os.path.isdir(fzf_path):
+ fzf_bin = get_fzf()
+ if fzf_bin:
+ return fzf_bin
+ # Fzf is cloned, but binary doesn't exist. Try running the install script
+ return fzf_bootstrap(update=True)
+
+ install = raw_input("Could not detect fzf, install it now? [y/n]: ")
+ if install.lower() != 'y':
+ return
+
+ if not find_executable('git'):
+ print("Git not found.")
+ print(FZF_INSTALL_FAILED)
+ sys.exit(1)
+
+ cmd = ['git', 'clone', '--depth', '1', 'https://github.com/junegunn/fzf.git']
+ if subprocess.call(cmd, cwd=os.path.dirname(fzf_path)):
+ print(FZF_INSTALL_FAILED)
+ sys.exit(1)
+
+ run_fzf_install_script(fzf_path)
+
+ print("Installed fzf to {}".format(fzf_path))
+ return get_fzf()
+
+
+def format_header():
+ shortcuts = []
+ for action, key in sorted(fzf_header_shortcuts.iteritems()):
+ shortcuts.append('{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}'.format(
+ t=terminal, action=action, key=key))
+ return FZF_HEADER.format(shortcuts=', '.join(shortcuts), t=terminal)
+
+
+def filter_by_paths(tasks, paths):
+ resolver = TestResolver.from_environment(cwd=here)
+ run_suites, run_tests = resolver.resolve_metadata(paths)
+ flavors = set([(t['flavor'], t.get('subsuite')) for t in run_tests])
+
+ task_regexes = set()
+ for flavor, subsuite in flavors:
+ suite = get_suite_definition(flavor, subsuite, strict=True)
+ if 'task_regex' not in suite:
+ print("warning: no tasks could be resolved from flavor '{}'{}".format(
+ flavor, " and subsuite '{}'".format(subsuite) if subsuite else ""))
+ continue
+
+ task_regexes.update(suite['task_regex'])
+
+ def match_task(task):
+ return any(re.search(pattern, task) for pattern in task_regexes)
+
+ return filter(match_task, tasks)
+
+
+def run_fuzzy_try(update=False, query=None, templates=None, full=False, parameters=None,
+ save=False, preset=None, mod_presets=False, push=True, message='{msg}',
+ paths=None, **kwargs):
+ if mod_presets:
+ return getattr(pset, mod_presets)(section='fuzzy')
+
+ fzf = fzf_bootstrap(update)
+
+ if not fzf:
+ print(FZF_NOT_FOUND)
+ return 1
+
+ vcs = VCSHelper.create()
+ vcs.check_working_directory(push)
+
+ all_tasks = generate_tasks(parameters, full, root=vcs.root)
+
+ if paths:
+ all_tasks = filter_by_paths(all_tasks, paths)
+ if not all_tasks:
+ return 1
+
+ key_shortcuts = [k + ':' + v for k, v in fzf_shortcuts.iteritems()]
+ cmd = [
+ fzf, '-m',
+ '--bind', ','.join(key_shortcuts),
+ '--header', format_header(),
+ # Using python to split the preview string is a bit convoluted,
+ # but is guaranteed to be available on all platforms.
+ '--preview', 'python -c "print(\\"\\n\\".join(sorted([s.strip(\\"\'\\") for s in \\"{+}\\".split()])))"', # noqa
+ '--preview-window=right:20%',
+ '--print-query',
+ ]
+
+ if query:
+ cmd.extend(['-f', query])
+ elif preset:
+ value = pset.load(preset, section='fuzzy')[0]
+ cmd.extend(['-f', value])
+
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+ out = proc.communicate('\n'.join(all_tasks))[0].splitlines()
+
+ selected = []
+ if out:
+ query = out[0]
+ selected = out[1:]
+
+ if not selected:
+ print("no tasks selected")
+ return
+
+ if save:
+ pset.save('fuzzy', save, query)
+
+ # build commit message
+ msg = "Fuzzy"
+ args = []
+ if paths:
+ args.append("paths={}".format(':'.join(paths)))
+ if query:
+ args.append("query={}".format(query))
+ if args:
+ msg = "{} {}".format(msg, '&'.join(args))
+ return vcs.push_to_try('fuzzy', message.format(msg=msg), selected, templates, push=push,
+ closed_tree=kwargs["closed_tree"])
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/tryselect/selectors/syntax.py
@@ -0,0 +1,629 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import re
+import sys
+from collections import defaultdict
+
+import mozpack.path as mozpath
+from moztest.resolve import TestResolver
+
+from .. import preset
+from ..cli import BaseTryParser
+from ..vcs import VCSHelper
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+class SyntaxParser(BaseTryParser):
+ name = 'syntax'
+ arguments = [
+ [['paths'],
+ {'nargs': '*',
+ 'help': 'Paths to search for tests to run on try.',
+ }],
+ [['-b', '--build'],
+ {'dest': 'builds',
+ 'default': 'do',
+ 'help': 'Build types to run (d for debug, o for optimized).',
+ }],
+ [['-p', '--platform'],
+ {'dest': 'platforms',
+ 'action': 'append',
+ 'help': 'Platforms to run (required if not found in the environment as '
+ 'AUTOTRY_PLATFORM_HINT).',
+ }],
+ [['-u', '--unittests'],
+ {'dest': 'tests',
+ 'action': 'append',
+ 'help': 'Test suites to run in their entirety.',
+ }],
+ [['-t', '--talos'],
+ {'action': 'append',
+ 'help': 'Talos suites to run.',
+ }],
+ [['-j', '--jobs'],
+ {'action': 'append',
+ 'help': 'Job tasks to run.',
+ }],
+ [['--tag'],
+ {'dest': 'tags',
+ 'action': 'append',
+ 'help': 'Restrict tests to the given tag (may be specified multiple times).',
+ }],
+ [['--and'],
+ {'action': 'store_true',
+ 'dest': 'intersection',
+ 'help': 'When -u and paths are supplied run only the intersection of the '
+ 'tests specified by the two arguments.',
+ }],
+ [['--no-artifact'],
+ {'action': 'store_true',
+ 'help': 'Disable artifact builds even if --enable-artifact-builds is set '
+ 'in the mozconfig.',
+ }],
+ [['-v', '--verbose'],
+ {'dest': 'verbose',
+ 'action': 'store_true',
+ 'default': False,
+ 'help': 'Print detailed information about the resulting test selection '
+ 'and commands performed.',
+ }],
+ [['--detect-paths'],
+ {'dest': 'detect_paths',
+ 'action': 'store_true',
+ 'default': False,
+ 'help': 'Provide test paths based on files changed in the working copy.',
+ }],
+ ]
+
+ # Arguments we will accept on the command line and pass through to try
+ # syntax with no further intervention. The set is taken from
+ # http://trychooser.pub.build.mozilla.org with a few additions.
+ #
+ # Note that the meaning of store_false and store_true arguments is
+ # not preserved here, as we're only using these to echo the literal
+ # arguments to another consumer. Specifying either store_false or
+ # store_true here will have an equivalent effect.
+ pass_through_arguments = {
+ '--rebuild': {
+ 'action': 'store',
+ 'dest': 'rebuild',
+ 'help': 'Re-trigger all test jobs (up to 20 times)',
+ },
+ '--rebuild-talos': {
+ 'action': 'store',
+ 'dest': 'rebuild_talos',
+ 'help': 'Re-trigger all talos jobs',
+ },
+ '--interactive': {
+ 'action': 'store_true',
+ 'dest': 'interactive',
+ 'help': 'Allow ssh-like access to running test containers',
+ },
+ '--no-retry': {
+ 'action': 'store_true',
+ 'dest': 'no_retry',
+ 'help': 'Do not retrigger failed tests',
+ },
+ '--setenv': {
+ 'action': 'append',
+ 'dest': 'setenv',
+ 'help': 'Set the corresponding variable in the test environment for'
+ 'applicable harnesses.',
+ },
+ '-f': {
+ 'action': 'store_true',
+ 'dest': 'failure_emails',
+ 'help': 'Request failure emails only',
+ },
+ '--failure-emails': {
+ 'action': 'store_true',
+ 'dest': 'failure_emails',
+ 'help': 'Request failure emails only',
+ },
+ '-e': {
+ 'action': 'store_true',
+ 'dest': 'all_emails',
+ 'help': 'Request all emails',
+ },
+ '--all-emails': {
+ 'action': 'store_true',
+ 'dest': 'all_emails',
+ 'help': 'Request all emails',
+ },
+ '--artifact': {
+ 'action': 'store_true',
+ 'dest': 'artifact',
+ 'help': 'Force artifact builds where possible.',
+ },
+ '--upload-xdbs': {
+ 'action': 'store_true',
+ 'dest': 'upload_xdbs',
+ 'help': 'Upload XDB compilation db files generated by hazard build',
+ },
+ }
+ templates = ['chemspill-prio']
+
+ def __init__(self, *args, **kwargs):
+ BaseTryParser.__init__(self, *args, **kwargs)
+
+ group = self.add_argument_group("pass-through arguments")
+ for arg, opts in self.pass_through_arguments.items():
+ group.add_argument(arg, **opts)
+
+
+class TryArgumentTokenizer(object):
+ symbols = [("seperator", ","),
+ ("list_start", "\["),
+ ("list_end", "\]"),
+ ("item", "([^,\[\]\s][^,\[\]]+)"),
+ ("space", "\s+")]
+ token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
+
+ def tokenize(self, data):
+ for match in self.token_re.finditer(data):
+ symbol = match.lastgroup
+ data = match.group(symbol)
+ if symbol == "space":
+ pass
+ else:
+ yield symbol, data
+
+
+class TryArgumentParser(object):
+ """Simple three-state parser for handling expressions
+ of the from "foo[sub item, another], bar,baz". This takes
+ input from the TryArgumentTokenizer and runs through a small
+ state machine, returning a dictionary of {top-level-item:[sub_items]}
+ i.e. the above would result in
+ {"foo":["sub item", "another"], "bar": [], "baz": []}
+ In the case of invalid input a ValueError is raised."""
+
+ EOF = object()
+
+ def __init__(self):
+ self.reset()
+
+ def reset(self):
+ self.tokens = None
+ self.current_item = None
+ self.data = {}
+ self.token = None
+ self.state = None
+
+ def parse(self, tokens):
+ self.reset()
+ self.tokens = tokens
+ self.consume()
+ self.state = self.item_state
+ while self.token[0] != self.EOF:
+ self.state()
+ return self.data
+
+ def consume(self):
+ try:
+ self.token = self.tokens.next()
+ except StopIteration:
+ self.token = (self.EOF, None)
+
+ def expect(self, *types):
+ if self.token[0] not in types:
+ raise ValueError("Error parsing try string, unexpected %s" % (self.token[0]))
+
+ def item_state(self):
+ self.expect("item")
+ value = self.token[1].strip()
+ if value not in self.data:
+ self.data[value] = []
+ self.current_item = value
+ self.consume()
+ if self.token[0] == "seperator":
+ self.consume()
+ elif self.token[0] == "list_start":
+ self.consume()
+ self.state = self.subitem_state
+ elif self.token[0] == self.EOF:
+ pass
+ else:
+ raise ValueError
+
+ def subitem_state(self):
+ self.expect("item")
+ value = self.token[1].strip()
+ self.data[self.current_item].append(value)
+ self.consume()
+ if self.token[0] == "seperator":
+ self.consume()
+ elif self.token[0] == "list_end":
+ self.consume()
+ self.state = self.after_list_end_state
+ else:
+ raise ValueError
+
+ def after_list_end_state(self):
+ self.expect("seperator")
+ self.consume()
+ self.state = self.item_state
+
+
+def parse_arg(arg):
+ tokenizer = TryArgumentTokenizer()
+ parser = TryArgumentParser()
+ return parser.parse(tokenizer.tokenize(arg))
+
+
+class AutoTry(object):
+
+ # Maps from flavors to the job names needed to run that flavour
+ flavor_jobs = {
+ 'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
+ 'xpcshell': ['xpcshell'],
+ 'chrome': ['mochitest-o'],
+ 'browser-chrome': ['mochitest-browser-chrome-1',
+ 'mochitest-e10s-browser-chrome-1',
+ 'mochitest-browser-chrome-e10s-1'],
+ 'devtools-chrome': ['mochitest-devtools-chrome-1',
+ 'mochitest-e10s-devtools-chrome-1',
+ 'mochitest-devtools-chrome-e10s-1'],
+ 'crashtest': ['crashtest', 'crashtest-e10s'],
+ 'reftest': ['reftest', 'reftest-e10s'],
+ 'web-platform-tests': ['web-platform-tests-1'],
+ }
+
+ flavor_suites = {
+ "mochitest": "mochitests",
+ "xpcshell": "xpcshell",
+ "chrome": "mochitest-o",
+ "browser-chrome": "mochitest-bc",
+ "devtools-chrome": "mochitest-dt",
+ "crashtest": "crashtest",
+ "reftest": "reftest",
+ "web-platform-tests": "web-platform-tests",
+ }
+
+ compiled_suites = [
+ "cppunit",
+ "gtest",
+ "jittest",
+ ]
+
+ common_suites = [
+ "cppunit",
+ "crashtest",
+ "firefox-ui-functional",
+ "geckoview",
+ "geckoview-junit",
+ "gtest",
+ "jittest",
+ "jsreftest",
+ "marionette",
+ "marionette-e10s",
+ "mochitests",
+ "reftest",
+ "robocop",
+ "web-platform-tests",
+ "xpcshell",
+ ]
+
+ def __init__(self, topsrcdir, mach_context):
+ self.topsrcdir = topsrcdir
+ self._resolver = None
+ self.mach_context = mach_context
+ self.vcs = VCSHelper.create()
+
+ @property
+ def resolver(self):
+ if self._resolver is None:
+ self._resolver = TestResolver.from_environment(cwd=here)
+ return self._resolver
+
+ def split_try_string(self, data):
+ return re.findall(r'(?:\[.*?\]|\S)+', data)
+
+ def paths_by_flavor(self, paths=None, tags=None):
+ paths_by_flavor = defaultdict(set)
+
+ if not (paths or tags):
+ return dict(paths_by_flavor)
+
+ tests = list(self.resolver.resolve_tests(paths=paths,
+ tags=tags))
+
+ for t in tests:
+ if t['flavor'] in self.flavor_suites:
+ flavor = t['flavor']
+ if 'subsuite' in t and t['subsuite'] == 'devtools':
+ flavor = 'devtools-chrome'
+
+ if flavor in ['crashtest', 'reftest']:
+ manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
+ paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath))
+ elif 'dir_relpath' in t:
+ paths_by_flavor[flavor].add(t['dir_relpath'])
+ else:
+ file_relpath = os.path.relpath(t['path'], self.topsrcdir)
+ dir_relpath = os.path.dirname(file_relpath)
+ paths_by_flavor[flavor].add(dir_relpath)
+
+ for flavor, path_set in paths_by_flavor.items():
+ paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths)
+
+ return dict(paths_by_flavor)
+
+ def deduplicate_prefixes(self, path_set, input_paths):
+ # Removes paths redundant to test selection in the given path set.
+ # If a path was passed on the commandline that is the prefix of a
+ # path in our set, we only need to include the specified prefix to
+ # run the intended tests (every test in "layout/base" will run if
+ # "layout" is passed to the reftest harness).
+ removals = set()
+ additions = set()
+
+ for path in path_set:
+ full_path = path
+ while path:
+ path, _ = os.path.split(path)
+ if path in input_paths:
+ removals.add(full_path)
+ additions.add(path)
+
+ return additions | (path_set - removals)
+
+ def remove_duplicates(self, paths_by_flavor, tests):
+ rv = {}
+ for item in paths_by_flavor:
+ if self.flavor_suites[item] not in tests:
+ rv[item] = paths_by_flavor[item].copy()
+ return rv
+
+ def calc_try_syntax(self, platforms, tests, talos, jobs, builds, paths_by_flavor, tags,
+ extras, intersection):
+ parts = ["try:"]
+
+ if platforms:
+ parts.extend(["-b", builds, "-p", ",".join(platforms)])
+
+ suites = tests if not intersection else {}
+ paths = set()
+ for flavor, flavor_tests in paths_by_flavor.iteritems():
+ suite = self.flavor_suites[flavor]
+ if suite not in suites and (not intersection or suite in tests):
+ for job_name in self.flavor_jobs[flavor]:
+ for test in flavor_tests:
+ paths.add("%s:%s" % (flavor, test))
+ suites[job_name] = tests.get(suite, [])
+
+ # intersection implies tests are expected
+ if intersection and not suites:
+ raise ValueError("No tests found matching filters")
+
+ if extras.get('artifact') and any([p.endswith("-nightly") for p in platforms]):
+ print('You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. '
+ 'Running without |--artifact| instead.')
+ del extras['artifact']
+
+ if extras.get('artifact'):
+ rejected = []
+ for suite in suites.keys():
+ if any([suite.startswith(c) for c in self.compiled_suites]):
+ rejected.append(suite)
+ if rejected:
+ raise ValueError("You can't run {} with "
+ "--artifact option.".format(', '.join(rejected)))
+
+ if extras.get('artifact') and 'all' in suites.keys():
+ non_compiled_suites = set(self.common_suites) - set(self.compiled_suites)
+ message = ('You asked for |-u all| with |--artifact| but compiled-code tests ({tests})'
+ ' can\'t run against an artifact build. Running (-u {non_compiled_suites}) '
+ 'instead.')
+ string_format = {
+ 'tests': ','.join(self.compiled_suites),
+ 'non_compiled_suites': ','.join(non_compiled_suites),
+ }
+ print(message.format(**string_format))
+ del suites['all']
+ suites.update({suite_name: None for suite_name in non_compiled_suites})
+
+ if suites:
+ parts.append("-u")
+ parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
+ for k, v in sorted(suites.items())))
+
+ if talos:
+ parts.append("-t")
+ parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
+ for k, v in sorted(talos.items())))
+
+ if jobs:
+ parts.append("-j")
+ parts.append(",".join(jobs))
+
+ if tags:
+ parts.append(' '.join('--tag %s' % t for t in tags))
+
+ if paths:
+ parts.append("--try-test-paths %s" % " ".join(sorted(paths)))
+
+ args_by_dest = {v['dest']: k for k, v in SyntaxParser.pass_through_arguments.items()}
+ for dest, value in extras.iteritems():
+ assert dest in args_by_dest
+ arg = args_by_dest[dest]
+ action = SyntaxParser.pass_through_arguments[arg]['action']
+ if action == 'store':
+ parts.append(arg)
+ parts.append(value)
+ if action == 'append':
+ for e in value:
+ parts.append(arg)
+ parts.append(e)
+ if action in ('store_true', 'store_false'):
+ parts.append(arg)
+
+ return " ".join(parts)
+
+ def normalise_list(self, items, allow_subitems=False):
+ rv = defaultdict(list)
+ for item in items:
+ parsed = parse_arg(item)
+ for key, values in parsed.iteritems():
+ rv[key].extend(values)
+
+ if not allow_subitems:
+ if not all(item == [] for item in rv.itervalues()):
+ raise ValueError("Unexpected subitems in argument")
+ return rv.keys()
+ else:
+ return rv
+
+ def validate_args(self, **kwargs):
+ tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"]
+ if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected):
+ if 'AUTOTRY_PLATFORM_HINT' in os.environ:
+ kwargs["platforms"] = [os.environ['AUTOTRY_PLATFORM_HINT']]
+ elif tests_selected:
+ print("Must specify platform when selecting tests.")
+ sys.exit(1)
+ else:
+ print("Either platforms or jobs must be specified as an argument to autotry.")
+ sys.exit(1)
+
+ try:
+ platforms = (self.normalise_list(kwargs["platforms"])
+ if kwargs["platforms"] else {})
+ except ValueError as e:
+ print("Error parsing -p argument:\n%s" % e.message)
+ sys.exit(1)
+
+ try:
+ tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
+ if kwargs["tests"] else {})
+ except ValueError as e:
+ print("Error parsing -u argument (%s):\n%s" % (kwargs["tests"], e.message))
+ sys.exit(1)
+
+ try:
+ talos = (self.normalise_list(kwargs["talos"], allow_subitems=True)
+ if kwargs["talos"] else [])
+ except ValueError as e:
+ print("Error parsing -t argument:\n%s" % e.message)
+ sys.exit(1)
+
+ try:
+ jobs = (self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {})
+ except ValueError as e:
+ print("Error parsing -j argument:\n%s" % e.message)
+ sys.exit(1)
+
+ paths = []
+ for p in kwargs["paths"]:
+ p = mozpath.normpath(os.path.abspath(p))
+ if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
+ print('Specified path "%s" is not a directory under the srcdir,'
+ ' unable to specify tests outside of the srcdir' % p)
+ sys.exit(1)
+ if len(p) <= len(self.topsrcdir):
+ print('Specified path "%s" is at the top of the srcdir and would'
+ ' select all tests.' % p)
+ sys.exit(1)
+ paths.append(os.path.relpath(p, self.topsrcdir))
+
+ try:
+ tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
+ except ValueError as e:
+ print("Error parsing --tags argument:\n%s" % e.message)
+ sys.exit(1)
+
+ extra_values = {k['dest'] for k in SyntaxParser.pass_through_arguments.values()}
+ extra_args = {k: v for k, v in kwargs.items()
+ if k in extra_values and v}
+
+ return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args
+
+ def run(self, **kwargs):
+ if kwargs["mod_presets"]:
+ getattr(preset, kwargs["mod_presets"])(section='try')
+ sys.exit()
+
+ if kwargs["preset"]:
+ value = preset.load(kwargs["preset"], section='try')[0]
+ defaults = vars(SyntaxParser().parse_args(self.split_try_string(value)))
+
+ if defaults is None:
+ print("No saved configuration called %s found in autotry.ini" % kwargs["preset"],
+ file=sys.stderr)
+
+ for key, value in kwargs.iteritems():
+ if value in (None, []) and key in defaults:
+ kwargs[key] = defaults[key]
+
+ if not any(kwargs[item] for item in ("paths", "tests", "tags")):
+ if kwargs['detect_paths']:
+ res = self.resolver.get_outgoing_metadata()
+ kwargs['paths'] = res['paths']
+ kwargs['tags'] = res['tags']
+ else:
+ kwargs['paths'] = set()
+ kwargs['tags'] = set()
+
+ builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args(**kwargs)
+
+ if paths or tags:
+ paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
+ for item in paths]
+ paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags)
+
+ if not paths_by_flavor and not tests:
+ print("No tests were found when attempting to resolve paths:\n\n\t%s" %
+ paths)
+ sys.exit(1)
+
+ if not kwargs["intersection"]:
+ paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests)
+ else:
+ paths_by_flavor = {}
+
+ # No point in dealing with artifacts if we aren't running any builds
+ local_artifact_build = False
+ if platforms:
+ local_artifact_build = kwargs.get('local_artifact_build', False)
+
+ # Add --artifact if --enable-artifact-builds is set ...
+ if local_artifact_build:
+ extra["artifact"] = True
+ # ... unless --no-artifact is explicitly given.
+ if kwargs["no_artifact"]:
+ if "artifact" in extra:
+ del extra["artifact"]
+
+ try:
+ msg = self.calc_try_syntax(platforms, tests, talos, jobs, builds,
+ paths_by_flavor, tags, extra, kwargs["intersection"])
+ except ValueError as e:
+ print(e.message)
+ sys.exit(1)
+
+ if local_artifact_build and not kwargs["no_artifact"]:
+ print('mozconfig has --enable-artifact-builds; including '
+ '--artifact flag in try syntax (use --no-artifact '
+ 'to override)')
+
+ if kwargs["verbose"] and paths_by_flavor:
+ print('The following tests will be selected: ')
+ for flavor, paths in paths_by_flavor.iteritems():
+ print("%s: %s" % (flavor, ",".join(paths)))
+
+ if kwargs["verbose"]:
+ print('The following try syntax was calculated:\n%s' % msg)
+
+ self.vcs.push_to_try('syntax', kwargs["message"].format(msg=msg), push=kwargs['push'],
+ closed_tree=kwargs["closed_tree"])
+
+ if kwargs["save"]:
+ assert msg.startswith("try: ")
+ msg = msg[len("try: "):]
+ preset.save('try', kwargs["save"], msg)
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/tryselect/tasks.py
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import sys
+
+from mozboot.util import get_state_dir
+from mozbuild.base import MozbuildObject
+from mozpack.files import FileFinder
+
+import taskgraph
+from taskgraph.generator import TaskGraphGenerator
+from taskgraph.parameters import (
+ ParameterMismatch,
+ load_parameters_file,
+)
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+
+PARAMETER_MISMATCH = """
+ERROR - The parameters being used to generate tasks differ from those defined
+in your working copy:
+
+ {}
+
+To fix this, either rebase onto the latest mozilla-central or pass in
+-p/--parameters. For more information on how to define parameters, see:
+https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters
+"""
+
+
+def invalidate(cache, root):
+ if not os.path.isfile(cache):
+ return
+
+ tc_dir = os.path.join(root, 'taskcluster')
+ tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
+ cmod = os.path.getmtime(cache)
+
+ if tmod > cmod:
+ os.remove(cache)
+
+
+def generate_tasks(params, full, root):
+ params = params or "project=mozilla-central"
+
+ cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
+ attr = 'full_task_set' if full else 'target_task_set'
+ cache = os.path.join(cache_dir, attr)
+
+ invalidate(cache, root)
+ if os.path.isfile(cache):
+ with open(cache, 'r') as fh:
+ return fh.read().splitlines()
+
+ if not os.path.isdir(cache_dir):
+ os.makedirs(cache_dir)
+
+ print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
+ try:
+ params = load_parameters_file(params, strict=False)
+ params.check()
+ except ParameterMismatch as e:
+ print(PARAMETER_MISMATCH.format(e.args[0]))
+ sys.exit(1)
+
+ taskgraph.fast = True
+ cwd = os.getcwd()
+ os.chdir(build.topsrcdir)
+
+ root = os.path.join(root, 'taskcluster', 'ci')
+ tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
+ labels = [label for label in tg.graph.visit_postorder()]
+
+ os.chdir(cwd)
+
+ with open(cache, 'w') as fh:
+ fh.write('\n'.join(labels))
+ return labels
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/tryselect/templates.py
@@ -0,0 +1,169 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Templates provide a way of modifying the task definition of selected
+tasks. They live under taskcluster/taskgraph/templates.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import sys
+from abc import ABCMeta, abstractmethod
+from argparse import Action, SUPPRESS
+
+import mozpack.path as mozpath
+from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+
+class Template(object):
+ __metaclass__ = ABCMeta
+
+ @abstractmethod
+ def add_arguments(self, parser):
+ pass
+
+ @abstractmethod
+ def context(self, **kwargs):
+ pass
+
+
+class Artifact(Template):
+
+ def add_arguments(self, parser):
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('--artifact', action='store_true',
+ help='Force artifact builds where possible.')
+ group.add_argument('--no-artifact', action='store_true',
+ help='Disable artifact builds even if being used locally.')
+
+ def context(self, artifact, no_artifact, **kwargs):
+ if artifact:
+ return {
+ 'artifact': {'enabled': '1'}
+ }
+
+ if no_artifact:
+ return
+
+ try:
+ if build.substs.get("MOZ_ARTIFACT_BUILDS"):
+ print("Artifact builds enabled, pass --no-artifact to disable")
+ return {
+ 'artifact': {'enabled': '1'}
+ }
+ except BuildEnvironmentNotFoundException:
+ pass
+
+
+class Path(Template):
+
+ def add_arguments(self, parser):
+ parser.add_argument('paths', nargs='*',
+ help='Run tasks containing tests under the specified path(s).')
+
+ def context(self, paths, **kwargs):
+ if not paths:
+ return
+
+ for p in paths:
+ if not os.path.exists(p):
+ print("error: '{}' is not a valid path.".format(p), file=sys.stderr)
+ sys.exit(1)
+
+ paths = [mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir) for p in paths]
+ return {
+ 'env': {
+ # can't use os.pathsep as machine splitting could be a different platform
+ 'MOZHARNESS_TEST_PATHS': ':'.join(paths),
+ }
+ }
+
+
+class Environment(Template):
+
+ def add_arguments(self, parser):
+ parser.add_argument('--env', action='append', default=None,
+ help='Set an environment variable, of the form FOO=BAR. '
+ 'Can be passed in multiple times.')
+
+ def context(self, env, **kwargs):
+ if not env:
+ return
+ return {
+ 'env': dict(e.split('=', 1) for e in env),
+ }
+
+
+class RangeAction(Action):
+ def __init__(self, min, max, *args, **kwargs):
+ self.min = min
+ self.max = max
+ kwargs['metavar'] = '[{}-{}]'.format(self.min, self.max)
+ super(RangeAction, self).__init__(*args, **kwargs)
+
+ def __call__(self, parser, namespace, values, option_string=None):
+ name = option_string or self.dest
+ if values < self.min:
+ parser.error('{} can not be less than {}'.format(name, self.min))
+ if values > self.max:
+ parser.error('{} can not be more than {}'.format(name, self.max))
+ setattr(namespace, self.dest, values)
+
+
+class Rebuild(Template):
+
+ def add_arguments(self, parser):
+ parser.add_argument('--rebuild', action=RangeAction, min=2, max=20, default=None, type=int,
+ help='Rebuild all selected tasks the specified number of times.')
+
+ def context(self, rebuild, **kwargs):
+ if not rebuild:
+ return
+
+ return {
+ 'rebuild': rebuild,
+ }
+
+
+class ChemspillPrio(Template):
+
+ def add_arguments(self, parser):
+ parser.add_argument('--chemspill-prio', action='store_true',
+ help='Run at a higher priority than most try jobs (chemspills only).')
+
+ def context(self, chemspill_prio, **kwargs):
+ if chemspill_prio:
+ return {
+ 'chemspill-prio': {}
+ }
+
+
+class TalosProfile(Template):
+
+ def add_arguments(self, parser):
+ parser.add_argument('--talos-profile', dest='profile', action='store_true', default=False,
+ help='Create and upload a gecko profile during talos tasks.')
+ # This is added for consistency with the 'syntax' selector
+ parser.add_argument('--geckoProfile', dest='profile', action='store_true', default=False,
+ help=SUPPRESS)
+
+ def context(self, profile, **kwargs):
+ if not profile:
+ return
+ return {'talos-profile': profile}
+
+
+all_templates = {
+ 'artifact': Artifact,
+ 'path': Path,
+ 'env': Environment,
+ 'rebuild': Rebuild,
+ 'chemspill-prio': ChemspillPrio,
+ 'talos-profile': TalosProfile,
+}
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/tryselect/vcs.py
@@ -0,0 +1,212 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import json
+import os
+import subprocess
+import sys
+from abc import ABCMeta, abstractmethod, abstractproperty
+
+GIT_CINNABAR_NOT_FOUND = """
+Could not detect `git-cinnabar`.
+
+The `mach try` command requires git-cinnabar to be installed when
+pushing from git. For more information and installation instruction,
+please see:
+
+ https://github.com/glandium/git-cinnabar
+""".lstrip()
+
+HG_PUSH_TO_TRY_NOT_FOUND = """
+Could not detect `push-to-try`.
+
+The `mach try` command requires the push-to-try extension enabled
+when pushing from hg. Please install it by running:
+
+ $ ./mach mercurial-setup
+""".lstrip()
+
+VCS_NOT_FOUND = """
+Could not detect version control. Only `hg` or `git` are supported.
+""".strip()
+
+UNCOMMITTED_CHANGES = """
+ERROR please commit changes before continuing
+""".strip()
+
+
+class VCSHelper(object):
+ """A abstract base VCS helper that detects hg or git"""
+ __metaclass__ = ABCMeta
+
+ def __init__(self, root):
+ self.root = root
+
+ @classmethod
+ def find_vcs(cls):
+ # First check if we're in an hg repo, if not try git
+ commands = (
+ ['hg', 'root'],
+ ['git', 'rev-parse', '--show-toplevel'],
+ )
+
+ for cmd in commands:
+ try:
+ output = subprocess.check_output(cmd, stderr=open(os.devnull, 'w')).strip()
+ except (subprocess.CalledProcessError, OSError):
+ continue
+
+ return cmd[0], output
+ return None, ''
+
+ @classmethod
+ def create(cls):
+ vcs, root = cls.find_vcs()
+ if not vcs:
+ print(VCS_NOT_FOUND)
+ sys.exit(1)
+ return vcs_class[vcs](root)
+
+ def run(self, cmd):
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate()
+
+ if proc.returncode:
+ print("Error running `{}`:".format(' '.join(cmd)))
+ if out:
+ print("stdout:\n{}".format(out))
+ if err:
+ print("stderr:\n{}".format(err))
+ raise subprocess.CalledProcessError(proc.returncode, cmd, out)
+ return out
+
+ def write_task_config(self, labels, templates=None):
+ config = os.path.join(self.root, 'try_task_config.json')
+ with open(config, 'w') as fh:
+ try_task_config = {'tasks': sorted(labels)}
+ if templates:
+ try_task_config['templates'] = templates
+
+ json.dump(try_task_config, fh, indent=2, separators=(',', ':'))
+ fh.write('\n')
+ return config
+
+ def check_working_directory(self, push=True):
+ if not push:
+ return
+
+ if self.has_uncommitted_changes:
+ print(UNCOMMITTED_CHANGES)
+ sys.exit(1)
+
+ def push_to_try(self, method, msg, labels=None, templates=None, push=True,
+ closed_tree=False):
+ closed_tree_string = " ON A CLOSED TREE" if closed_tree else ""
+ commit_message = ('%s%s\n\nPushed via `mach try %s`' %
+ (msg, closed_tree_string, method))
+
+ self.check_working_directory(push)
+
+ config = None
+ if labels or labels == []:
+ config = self.write_task_config(labels, templates)
+
+ try:
+ if not push:
+ print("Commit message:")
+ print(commit_message)
+ if config:
+ print("Calculated try_task_config.json:")
+ with open(config) as fh:
+ print(fh.read())
+ return
+
+ self._push_to_try(commit_message, config)
+ finally:
+ if config and os.path.isfile(config):
+ os.remove(config)
+
+ @abstractmethod
+ def _push_to_try(self, msg, config):
+ pass
+
+ @abstractproperty
+ def files_changed(self):
+ pass
+
+ @abstractproperty
+ def has_uncommitted_changes(self):
+ pass
+
+
+class HgHelper(VCSHelper):
+
+ def _push_to_try(self, msg, config):
+ try:
+ if config:
+ self.run(['hg', 'add', config])
+ return subprocess.check_call(['hg', 'push-to-try', '-m', msg])
+ except subprocess.CalledProcessError:
+ try:
+ self.run(['hg', 'showconfig', 'extensions.push-to-try'])
+ except subprocess.CalledProcessError:
+ print(HG_PUSH_TO_TRY_NOT_FOUND)
+ return 1
+ finally:
+ self.run(['hg', 'revert', '-a'])
+
+ @property
+ def files_changed(self):
+ return self.run(['hg', 'log', '-r', '::. and not public()',
+ '--template', '{join(files, "\n")}\n']).splitlines()
+
+ @property
+ def has_uncommitted_changes(self):
+ stat = [s for s in self.run(['hg', 'status', '-amrn']).split() if s]
+ return len(stat) > 0
+
+
+class GitHelper(VCSHelper):
+
+ def _push_to_try(self, msg, config):
+ try:
+ subprocess.check_output(['git', 'cinnabar', '--version'], stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError:
+ print(GIT_CINNABAR_NOT_FOUND)
+ return 1
+
+ if config:
+ self.run(['git', 'add', config])
+ subprocess.check_call(['git', 'commit', '--allow-empty', '-m', msg])
+ try:
+ return subprocess.call(['git', 'push', 'hg::ssh://hg.mozilla.org/try',
+ '+HEAD:refs/heads/branches/default/tip'])
+ finally:
+ self.run(['git', 'reset', 'HEAD~'])
+
+ @property
+ def files_changed(self):
+ # This finds the files changed on the current branch based on the
+ # diff of the current branch its merge-base base with other branches.
+ current_branch = self.run(['git', 'rev-parse', 'HEAD']).strip()
+ all_branches = self.run(['git', 'for-each-ref', 'refs/heads', 'refs/remotes',
+ '--format=%(objectname)']).splitlines()
+ other_branches = set(all_branches) - set([current_branch])
+ base_commit = self.run(['git', 'merge-base', 'HEAD'] + list(other_branches)).strip()
+ return self.run(['git', 'diff', '--name-only', '-z', 'HEAD',
+ base_commit]).strip('\0').split('\0')
+
+ @property
+ def has_uncommitted_changes(self):
+ stat = [s for s in self.run(['git', 'diff', '--cached', '--name-only',
+ '--diff-filter=AMD']).split() if s]
+ return len(stat) > 0
+
+
+vcs_class = {
+ 'git': GitHelper,
+ 'hg': HgHelper,
+}
deleted file mode 100644
--- a/tools/tryselect/vcs.py
+++ /dev/null
@@ -1,212 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function
-
-import json
-import os
-import subprocess
-import sys
-from abc import ABCMeta, abstractmethod, abstractproperty
-
-GIT_CINNABAR_NOT_FOUND = """
-Could not detect `git-cinnabar`.
-
-The `mach try` command requires git-cinnabar to be installed when
-pushing from git. For more information and installation instruction,
-please see:
-
- https://github.com/glandium/git-cinnabar
-""".lstrip()
-
-HG_PUSH_TO_TRY_NOT_FOUND = """
-Could not detect `push-to-try`.
-
-The `mach try` command requires the push-to-try extension enabled
-when pushing from hg. Please install it by running:
-
- $ ./mach mercurial-setup
-""".lstrip()
-
-VCS_NOT_FOUND = """
-Could not detect version control. Only `hg` or `git` are supported.
-""".strip()
-
-UNCOMMITTED_CHANGES = """
-ERROR please commit changes before continuing
-""".strip()
-
-
-class VCSHelper(object):
- """A abstract base VCS helper that detects hg or git"""
- __metaclass__ = ABCMeta
-
- def __init__(self, root):
- self.root = root
-
- @classmethod
- def find_vcs(cls):
- # First check if we're in an hg repo, if not try git
- commands = (
- ['hg', 'root'],
- ['git', 'rev-parse', '--show-toplevel'],
- )
-
- for cmd in commands:
- try:
- output = subprocess.check_output(cmd, stderr=open(os.devnull, 'w')).strip()
- except (subprocess.CalledProcessError, OSError):
- continue
-
- return cmd[0], output
- return None, ''
-
- @classmethod
- def create(cls):
- vcs, root = cls.find_vcs()
- if not vcs:
- print(VCS_NOT_FOUND)
- sys.exit(1)
- return vcs_class[vcs](root)
-
- def run(self, cmd):
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = proc.communicate()
-
- if proc.returncode:
- print("Error running `{}`:".format(' '.join(cmd)))
- if out:
- print("stdout:\n{}".format(out))
- if err:
- print("stderr:\n{}".format(err))
- raise subprocess.CalledProcessError(proc.returncode, cmd, out)
- return out
-
- def write_task_config(self, labels, templates=None):
- config = os.path.join(self.root, 'try_task_config.json')
- with open(config, 'w') as fh:
- try_task_config = {'tasks': sorted(labels)}
- if templates:
- try_task_config['templates'] = templates
-
- json.dump(try_task_config, fh, indent=2, separators=(',', ':'))
- fh.write('\n')
- return config
-
- def check_working_directory(self, push=True):
- if not push:
- return
-
- if self.has_uncommitted_changes:
- print(UNCOMMITTED_CHANGES)
- sys.exit(1)
-
- def push_to_try(self, method, msg, labels=None, templates=None, push=True,
- closed_tree=False):
- closed_tree_string = " ON A CLOSED TREE" if closed_tree else ""
- commit_message = ('%s%s\n\nPushed via `mach try %s`' %
- (msg, closed_tree_string, method))
-
- self.check_working_directory(push)
-
- config = None
- if labels or labels == []:
- config = self.write_task_config(labels, templates)
-
- try:
- if not push:
- print("Commit message:")
- print(commit_message)
- if config:
- print("Calculated try_task_config.json:")
- with open(config) as fh:
- print(fh.read())
- return
-
- self._push_to_try(commit_message, config)
- finally:
- if config and os.path.isfile(config):
- os.remove(config)
-
- @abstractmethod
- def _push_to_try(self, msg, config):
- pass
-
- @abstractproperty
- def files_changed(self):
- pass
-
- @abstractproperty
- def has_uncommitted_changes(self):
- pass
-
-
-class HgHelper(VCSHelper):
-
- def _push_to_try(self, msg, config):
- try:
- if config:
- self.run(['hg', 'add', config])
- return subprocess.check_call(['hg', 'push-to-try', '-m', msg])
- except subprocess.CalledProcessError:
- try:
- self.run(['hg', 'showconfig', 'extensions.push-to-try'])
- except subprocess.CalledProcessError:
- print(HG_PUSH_TO_TRY_NOT_FOUND)
- return 1
- finally:
- self.run(['hg', 'revert', '-a'])
-
- @property
- def files_changed(self):
- return self.run(['hg', 'log', '-r', '::. and not public()',
- '--template', '{join(files, "\n")}\n']).splitlines()
-
- @property
- def has_uncommitted_changes(self):
- stat = [s for s in self.run(['hg', 'status', '-amrn']).split() if s]
- return len(stat) > 0
-
-
-class GitHelper(VCSHelper):
-
- def _push_to_try(self, msg, config):
- try:
- subprocess.check_output(['git', 'cinnabar', '--version'], stderr=subprocess.STDOUT)
- except subprocess.CalledProcessError:
- print(GIT_CINNABAR_NOT_FOUND)
- return 1
-
- if config:
- self.run(['git', 'add', config])
- subprocess.check_call(['git', 'commit', '--allow-empty', '-m', msg])
- try:
- return subprocess.call(['git', 'push', 'hg::ssh://hg.mozilla.org/try',
- '+HEAD:refs/heads/branches/default/tip'])
- finally:
- self.run(['git', 'reset', 'HEAD~'])
-
- @property
- def files_changed(self):
- # This finds the files changed on the current branch based on the
- # diff of the current branch its merge-base base with other branches.
- current_branch = self.run(['git', 'rev-parse', 'HEAD']).strip()
- all_branches = self.run(['git', 'for-each-ref', 'refs/heads', 'refs/remotes',
- '--format=%(objectname)']).splitlines()
- other_branches = set(all_branches) - set([current_branch])
- base_commit = self.run(['git', 'merge-base', 'HEAD'] + list(other_branches)).strip()
- return self.run(['git', 'diff', '--name-only', '-z', 'HEAD',
- base_commit]).strip('\0').split('\0')
-
- @property
- def has_uncommitted_changes(self):
- stat = [s for s in self.run(['git', 'diff', '--cached', '--name-only',
- '--diff-filter=AMD']).split() if s]
- return len(stat) > 0
-
-
-vcs_class = {
- 'git': GitHelper,
- 'hg': HgHelper,
-}