Bug 1413928 - [tryselect] Implement paths for |mach try fuzzy|
This enables the syntax like:
./mach try fuzzy dom/indexedDB
This will open up the fzf interface like normal, except only tasks
that have tests under dom/indexedDB will be selectable (and there
will only be one chunk per configuration).
This can be combined with -q/--query like normal:
./mach try fuzzy dom/indexedDB -q "!pgo !cov !asan"
When the tasks get scheduled, only the tests under the specified
path(s) will run within the harness.
MozReview-Commit-ID: IHRXXi5mB4G
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -102,58 +102,65 @@ TEST_SUITES = {
},
}
# Maps test flavors to metadata on how to run that test.
TEST_FLAVORS = {
'a11y': {
'mach_command': 'mochitest',
'kwargs': {'flavor': 'a11y', 'test_paths': []},
+ 'task_regex': 'mochitest-a11y(?:-1)?$',
},
'browser-chrome': {
'mach_command': 'mochitest',
'kwargs': {'flavor': 'browser-chrome', 'test_paths': []},
+ 'task_regex': 'mochitest-browser-chrome(?:-e10s)?(?:-1)?$',
},
'crashtest': {},
'chrome': {
'mach_command': 'mochitest',
'kwargs': {'flavor': 'chrome', 'test_paths': []},
+ 'task_regex': 'mochitest-chrome(?:-e10s)?(?:-1)?$',
},
'firefox-ui-functional': {
'mach_command': 'firefox-ui-functional',
'kwargs': {'tests': []},
},
'firefox-ui-update': {
'mach_command': 'firefox-ui-update',
'kwargs': {'tests': []},
},
'marionette': {
'mach_command': 'marionette-test',
'kwargs': {'tests': []},
},
'mochitest': {
'mach_command': 'mochitest',
'kwargs': {'flavor': 'mochitest', 'test_paths': []},
+ 'task_regex': 'mochitest(?:-e10s)?(?:-1)?$',
},
'python': {
'mach_command': 'python-test',
'kwargs': {},
},
'reftest': {
'mach_command': 'reftest',
'kwargs': {'tests': []},
+ 'task_regex': '(opt|debug)-reftest(?:-no-accel|-gpu|-stylo)?(?:-e10s)?(?:-1)?$',
},
'steeplechase': {},
'web-platform-tests': {
'mach_command': 'web-platform-tests',
'kwargs': {'include': []},
+ 'task_regex': 'web-platform-tests(?:-reftests|-wdspec)?(?:-e10s)?(?:-1)?$',
},
'xpcshell': {
'mach_command': 'xpcshell-test',
'kwargs': {'test_paths': []},
+ 'task_regex': 'xpcshell(?:-1)?$',
},
}
for i in range(1, MOCHITEST_TOTAL_CHUNKS + 1):
TEST_SUITES['mochitest-%d' % i] = {
'aliases': ('M%d' % i, 'm%d' % i),
'mach_command': 'mochitest',
'kwargs': {
--- a/tools/tryselect/docs/selectors/fuzzy.rst
+++ b/tools/tryselect/docs/selectors/fuzzy.rst
@@ -63,16 +63,68 @@ Would likely match a similar set of task
'a | 'b => OR operator (joins two exact match operators together)
For example:
.. code-block:: text
^start 'exact | !ignore fuzzy end$
+Test Paths
+----------
+
+One or more paths to a file or directory may be specified as positional arguments. When
+specifying paths, the list of available tasks to choose from is filtered down such that
+only suites that have tests in a specified path can be selected. Notably, only the first
+chunk of each suite/platform appears. When the tasks are scheduled, only tests that live
+under one of the specified paths will be run.
+
+.. note::
+
+ When using paths, be aware that all tests under the specified paths will run in the
+ same chunk. This might produce a different ordering from what gets run on production
+ branches, and may yield different results.
+
+ For suites that restart the browser between each manifest (like mochitest), this
+ shouldn't be as big of a concern.
+
+Paths can be used with the interactive fzf window, or using the ``-q/--query`` argument.
+For example, running:
+
+.. code-block:: shell
+
+ $ mach try fuzzy layout/reftests/reftest-sanity -q "!pgo !cov !asan 'linux64"
+
+Would produce the following ``try_task_config.json``:
+
+.. code-block:: json
+
+ {
+ "templates":{
+ "env":{
+ "MOZHARNESS_TEST_PATHS":"layout/reftests/reftest-sanity"
+ }
+ },
+ "tasks":[
+ "test-linux64-qr/debug-reftest-e10s-1",
+ "test-linux64-qr/opt-reftest-e10s-1",
+ "test-linux64-stylo-disabled/debug-reftest-e10s-1",
+ "test-linux64-stylo-disabled/opt-reftest-e10s-1",
+ "test-linux64/debug-reftest-e10s-1",
+ "test-linux64/debug-reftest-no-accel-e10s-1",
+ "test-linux64/debug-reftest-stylo-e10s-1",
+ "test-linux64/opt-reftest-e10s-1",
+ "test-linux64/opt-reftest-no-accel-e10s-1",
+ "test-linux64/opt-reftest-stylo-e10s-1"
+ ]
+ }
+
+Inside of these tasks, the reftest harness will only run tests that live under
+``layout/reftests/reftest-sanity``.
+
Additional Arguments
--------------------
There are a few additional command line arguments you may wish to use:
``-q/--query``
Instead of opening the interactive interface, automatically apply the specified
query. This is equivalent to opening the interface then typing: ``<query><ctrl-a><enter>``.
--- a/tools/tryselect/selectors/fuzzy.py
+++ b/tools/tryselect/selectors/fuzzy.py
@@ -1,30 +1,33 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import os
import platform
+import re
import subprocess
import sys
from distutils.spawn import find_executable
from mozboot.util import get_state_dir
from mozterm import Terminal
+from moztest.resolve import TestResolver, TEST_FLAVORS
from .. import preset as pset
from ..cli import BaseTryParser
from ..tasks import generate_tasks
from ..vcs import VCSHelper
terminal = Terminal()
+here = os.path.abspath(os.path.dirname(__file__))
FZF_NOT_FOUND = """
Could not find the `fzf` binary.
The `mach try fuzzy` command depends on fzf. Please install it following the
appropriate instructions for your platform:
https://github.com/junegunn/fzf#installation
@@ -83,17 +86,17 @@ class FuzzyParser(BaseTryParser):
}],
[['-u', '--update'],
{'action': 'store_true',
'default': False,
'help': "Update fzf before running.",
}],
]
common_groups = ['push', 'task', 'preset']
- templates = ['artifact', 'env', 'rebuild']
+ templates = ['artifact', 'path', 'env', 'rebuild']
def run(cmd, cwd=None):
is_win = platform.system() == 'Windows'
return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
def run_fzf_install_script(fzf_path):
@@ -165,33 +168,49 @@ def fzf_bootstrap(update=False):
def format_header():
shortcuts = []
for action, key in sorted(fzf_header_shortcuts.iteritems()):
shortcuts.append('{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}'.format(
t=terminal, action=action, key=key))
return FZF_HEADER.format(shortcuts=', '.join(shortcuts), t=terminal)
+def filter_by_paths(tasks, paths):
+ resolver = TestResolver.from_environment(cwd=here)
+ run_suites, run_tests = resolver.resolve_metadata(paths)
+ flavors = set([t['flavor'] for t in run_tests])
+ task_regexes = [TEST_FLAVORS[f]['task_regex']
+ for f in flavors if 'task_regex' in TEST_FLAVORS[f]]
+
+ def match_task(task):
+ return any(re.search(pattern, task) for pattern in task_regexes)
+
+ return filter(match_task, tasks)
+
+
def run_fuzzy_try(update=False, query=None, templates=None, full=False, parameters=None,
save=False, preset=None, mod_presets=False, push=True, message='{msg}',
- **kwargs):
+ paths=None, **kwargs):
if mod_presets:
return getattr(pset, mod_presets)(section='fuzzy')
fzf = fzf_bootstrap(update)
if not fzf:
print(FZF_NOT_FOUND)
return
vcs = VCSHelper.create()
vcs.check_working_directory(push)
all_tasks = generate_tasks(parameters, full, root=vcs.root)
+ if paths:
+ all_tasks = filter_by_paths(all_tasks, paths)
+
key_shortcuts = [k + ':' + v for k, v in fzf_shortcuts.iteritems()]
cmd = [
fzf, '-m',
'--bind', ','.join(key_shortcuts),
'--header', format_header(),
# Using python to split the preview string is a bit convoluted,
# but is guaranteed to be available on all platforms.
'--preview', 'python -c "print(\\"\\n\\".join(sorted([s.strip(\\"\'\\") for s in \\"{+}\\".split()])))"', # noqa
@@ -215,12 +234,19 @@ def run_fuzzy_try(update=False, query=No
if not selected:
print("no tasks selected")
return
if save:
pset.save('fuzzy', save, query)
- query = " with query: {}".format(query) if query else ""
- msg = "Fuzzy{}".format(query)
+ # build commit message
+ msg = "Fuzzy"
+ args = []
+ if paths:
+ args.append("paths={}".format(':'.join(paths)))
+ if query:
+ args.append("query={}".format(query))
+ if args:
+ msg = "{} {}".format(msg, '&'.join(args))
return vcs.push_to_try('fuzzy', message.format(msg=msg), selected, templates, push=push,
closed_tree=kwargs["closed_tree"])
--- a/tools/tryselect/test/python.ini
+++ b/tools/tryselect/test/python.ini
@@ -1,4 +1,5 @@
[DEFAULT]
subsuite=try
+[test_fuzzy.py]
[test_templates.py]
new file mode 100644
--- /dev/null
+++ b/tools/tryselect/test/test_fuzzy.py
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import mozunit
+import pytest
+from moztest.resolve import TestResolver
+
+from tryselect.selectors import fuzzy
+
+
+@pytest.fixture
+def patch_resolver(monkeypatch):
+ def inner(suites, tests):
+ def fake_test_metadata(*args, **kwargs):
+ return suites, tests
+ monkeypatch.setattr(TestResolver, 'resolve_metadata', fake_test_metadata)
+ return inner
+
+
+def test_filter_by_paths(patch_resolver):
+ tasks = ['foobar/xpcshell-1', 'foobar/mochitest', 'foobar/xpcshell']
+
+ patch_resolver(['xpcshell'], {})
+ assert fuzzy.filter_by_paths(tasks, 'dummy') == []
+
+ patch_resolver([], [{'flavor': 'xpcshell'}])
+ assert fuzzy.filter_by_paths(tasks, 'dummy') == ['foobar/xpcshell-1', 'foobar/xpcshell']
+
+
+if __name__ == '__main__':
+ mozunit.main()
--- a/tools/tryselect/test/test_fuzzy.t
+++ b/tools/tryselect/test/test_fuzzy.t
@@ -1,47 +1,47 @@
$ . $TESTDIR/setup.sh
$ cd $topsrcdir
Test fuzzy selector
$ ./mach try fuzzy $testargs -q "'foo"
Commit message:
- Fuzzy with query: 'foo
+ Fuzzy query='foo
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"tasks":[
"test/foo-debug",
"test/foo-opt"
]
}
$ ./mach try fuzzy $testargs -q "'bar"
no tasks selected
$ ./mach try fuzzy $testargs --full -q "'bar"
Commit message:
- Fuzzy with query: 'bar
+ Fuzzy query='bar
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"tasks":[
"test/bar-debug",
"test/bar-opt"
]
}
Test templates
$ ./mach try fuzzy --no-push --artifact -q "'foo"
Commit message:
- Fuzzy with query: 'foo
+ Fuzzy query='foo
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"templates":{
"artifact":{
"enabled":"1"
}
@@ -49,17 +49,17 @@ Test templates
"tasks":[
"test/foo-debug",
"test/foo-opt"
]
}
$ ./mach try fuzzy $testargs --env FOO=1 --env BAR=baz -q "'foo"
Commit message:
- Fuzzy with query: 'foo
+ Fuzzy query='foo
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"templates":{
"env":{
"FOO":"1",
"BAR":"baz"
--- a/tools/tryselect/test/test_message.t
+++ b/tools/tryselect/test/test_message.t
@@ -2,30 +2,30 @@
$ cd $topsrcdir
Test custom commit messages with fuzzy selector
$ ./mach try fuzzy $testargs -q foo --message "Foobar"
Commit message:
Foobar
- Fuzzy with query: foo
+ Fuzzy query=foo
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"tasks":[
"test/foo-debug",
"test/foo-opt"
]
}
$ ./mach try fuzzy $testargs -q foo -m "Foobar: {msg}"
Commit message:
- Foobar: Fuzzy with query: foo
+ Foobar: Fuzzy query=foo
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"tasks":[
"test/foo-debug",
"test/foo-opt"
]
--- a/tools/tryselect/test/test_preset.t
+++ b/tools/tryselect/test/test_preset.t
@@ -63,41 +63,41 @@ Test preset with syntax subcommand
bar = -b do -p win32 -u none -t all --tag bar
Test preset with fuzzy subcommand
$ ./mach try fuzzy $testargs --save baz -q "'baz"
preset saved, run with: --preset=baz
Commit message:
- Fuzzy with query: 'baz
+ Fuzzy query='baz
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"tasks":[
"build-baz"
]
}
$ ./mach try fuzzy $testargs --preset baz
Commit message:
- Fuzzy with query: 'baz
+ Fuzzy query='baz
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"tasks":[
"build-baz"
]
}
$ ./mach try $testargs --preset baz
Commit message:
- Fuzzy with query: 'baz
+ Fuzzy query='baz
Pushed via `mach try fuzzy`
Calculated try_task_config.json:
{
"tasks":[
"build-baz"
]
}