Bug 1401199 - Autodetect relevant parameters
MozReview-Commit-ID: GMiGuNApoUF
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -35,16 +35,17 @@ tasks:
$if: 'tasks_for == "hg-push"'
then: {createdForUser: "${ownerEmail}"}
routes:
$if: 'tasks_for == "hg-push"'
then:
- "index.gecko.v2.${repository.project}.latest.firefox.decision"
- "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
+ - "index.gecko.v2.revision.${push.revision}.decision"
- "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
- "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
- "notify.email.${ownerEmail}.on-failed"
- "notify.email.${ownerEmail}.on-exception"
else:
- "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
- "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
- "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
--- a/python/mozversioncontrol/mozversioncontrol/__init__.py
+++ b/python/mozversioncontrol/mozversioncontrol/__init__.py
@@ -175,16 +175,21 @@ class Repository(object):
Returns True if the working directory does not have any file
modifications. False otherwise.
By default, untracked and ignored files are not considered. If
``untracked`` or ``ignored`` are set, they influence the clean check
to factor these file classes into consideration.
"""
+ @abc.abstractmethod
+ def get_first_public_ancestor(self):
+ """Return the revision of the first public ancestor of the working
+ directory."""
+
class HgRepository(Repository):
'''An implementation of `Repository` for Mercurial repositories.'''
def __init__(self, path, hg='hg'):
import hglib.client
super(HgRepository, self).__init__(path, tool=hg)
self._env[b'HGPLAIN'] = b'1'
@@ -298,16 +303,20 @@ class HgRepository(Repository):
args.append(b'--unknown')
if ignored:
args.append(b'--ignored')
# If output is empty, there are no entries of requested status, which
# means we are clean.
return not len(self._run_in_client(args).strip())
+ def get_first_public_ancestor(self):
+ return self._run('log', '-r', 'first(reverse(ancestors(.)) and public())',
+ '-T', '{node}')
+
class GitRepository(Repository):
'''An implementation of `Repository` for Git repositories.'''
def __init__(self, path, git='git'):
super(GitRepository, self).__init__(path, tool=git)
@property
def name(self):
@@ -360,16 +369,19 @@ class GitRepository(Repository):
args = ['status', '--porcelain']
if untracked:
args.append('--untracked-files')
if ignored:
args.append('--ignored')
return not len(self._run(*args).strip())
+ def get_first_public_ancestor(self):
+ return self._run('show', '--format=format:"%H"', self.get_upstream())
+
def get_repository_object(path, hg='hg', git='git'):
'''Get a repository object for the repository at `path`.
If `path` is not a known VCS repository, raise an exception.
'''
if os.path.isdir(os.path.join(path, '.hg')):
return HgRepository(path, hg=hg)
elif os.path.exists(os.path.join(path, '.git')):
--- a/taskcluster/docs/mach.rst
+++ b/taskcluster/docs/mach.rst
@@ -19,29 +19,39 @@ graph-generation process and output the
Get the target task graph
``mach taskgraph optimized``
Get the optimized task graph
``mach taskgraph morphed``
Get the morhped task graph
-Each of these commands takes an optional ``--parameters`` option giving a file
-with parameters to guide the graph generation. The decision task helpfully
-produces such a file on every run, and that is generally the easiest way to get
-a parameter file. The parameter keys and values are described in
+See :doc:`how-tos` for further practical tips on debugging task-graph mechanics
+locally.
+
+Parameters
+----------
+
+Each of these commands takes an optional ``--parameters`` argument giving a
+file with parameters to guide the graph generation. The decision task
+helpfully produces such a file on every run, and that is generally the easiest
+way to get a parameter file. The parameter keys and values are described in
:doc:`parameters`; using that information, you may modify an existing
``parameters.yml`` or create your own. The ``--parameters`` option can also
-take an argument of the form ``project=<project>`` which will fetch the
-parameters from the latest push on that project; or ``task-id=<task-id>`` which
-will fetch the parameters from the given decision task. It defaults to
-``project=mozilla-central``.
+take the following forms:
-See :doc:`how-tos` for further practical tips on debugging task-graph mechanics
-locally.
+``project=<project>``
+ Fetch the parameters from the latest push on that project
+``task-id=<task-id>``
+ Fetch the parameters from the given decision task id
+``revision=<rev>``
+ Fetch the parameters from the decision task associated with the given revision.
+
+If unspecified, the taskgraph module will attempt to automatically fetch a
+relevant parameters.yml based on the state of your local repository.
Taskgraph JSON Format
---------------------
By default, the above commands will only output a list of tasks. Use `-J` flag
to output full task definitions. For example:
.. code-block:: shell
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -37,17 +37,17 @@ class ShowTaskGraphSubCommand(SubCommand
CommandArgument('--verbose', '-v', action="store_true",
help="include debug-level logging output"),
CommandArgument('--json', '-J', action="store_const",
dest="format", const="json",
help="Output task graph as a JSON object"),
CommandArgument('--labels', '-L', action="store_const",
dest="format", const="labels",
help="Output the label for each task in the task graph (default)"),
- CommandArgument('--parameters', '-p', default="project=mozilla-central",
+ CommandArgument('--parameters', '-p', default=None,
help="parameters file (.yml or .json; see "
"`taskcluster/docs/parameters.rst`)`"),
CommandArgument('--no-optimize', dest="optimize", action="store_false",
default="true",
help="do not remove tasks from the graph that are found in the "
"index (a.k.a. optimize the graph)"),
CommandArgument('--tasks-regex', '--tasks', default=None,
help="only return tasks with labels matching this regular "
@@ -314,17 +314,17 @@ class MachCommands(MachCommandBase):
parameters=parameters,
test=False)
except Exception:
traceback.print_exc()
sys.exit(1)
@SubCommand('taskgraph', 'test-action-callback',
description='Run an action callback in a testing mode')
- @CommandArgument('--parameters', '-p', default='project=mozilla-central',
+ @CommandArgument('--parameters', '-p', default=None,
help='parameters file (.yml or .json; see '
'`taskcluster/docs/parameters.rst`)`')
@CommandArgument('--task-id', default=None,
help='TaskId to which the action applies')
@CommandArgument('--task-group-id', default=None,
help='TaskGroupId to which the action applies')
@CommandArgument('--input', default=None,
help='Action input (.yml or .json)')
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -4,16 +4,24 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import json
import yaml
from mozbuild.util import ReadOnlyDict
+from mozversioncontrol import get_repository_object
+
+from . import GECKO
+
+
+class ParameterMismatchException(Exception):
+ """Raised when a parameters.yml has extra or missing parameters."""
+
# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
PARAMETER_NAMES = set([
'base_repository',
'build_date',
'filters',
'head_ref',
'head_repository',
@@ -45,52 +53,72 @@ class Parameters(ReadOnlyDict):
if missing:
msg.append("missing parameters: " + ", ".join(missing))
extra = names - PARAMETER_NAMES
if extra:
msg.append("extra parameters: " + ", ".join(extra))
if msg:
- raise Exception("; ".join(msg))
+ raise ParameterMismatchException("; ".join(msg))
def __getitem__(self, k):
if k not in PARAMETER_NAMES:
raise KeyError("no such parameter {!r}".format(k))
try:
return super(Parameters, self).__getitem__(k)
except KeyError:
raise KeyError("taskgraph parameter {!r} not found".format(k))
-def load_parameters_file(filename):
+def find_matching_parameters():
+ try:
+ # attempt to use mozilla-central first to avoid vcs penalty
+ params = load_parameters_file('project=mozilla-central')
+ params.check()
+ return params
+ except ParameterMismatchException as e:
+ repo = get_repository_object(GECKO)
+ rev = repo.get_first_public_ancestor()
+ try:
+ params = load_parameters_file('revision={}'.format(rev))
+ params.check()
+ except KeyError:
+ raise e
+ return params
+
+
+def load_parameters_file(filename=None):
"""
Load parameters from a path, url, decision task-id or project.
Examples:
task-id=fdtgsD5DQUmAQZEaGMvQ4Q
project=mozilla-central
"""
import urllib
from taskgraph.util.taskcluster import get_artifact_url, find_task_id
if not filename:
- return Parameters()
+ return find_matching_parameters()
try:
# reading parameters from a local parameters.yml file
f = open(filename)
except IOError:
# fetching parameters.yml using task task-id, project or supplied url
task_id = None
if filename.startswith("task-id="):
task_id = filename.split("=")[1]
elif filename.startswith("project="):
index = "gecko.v2.{}.latest.firefox.decision".format(filename.split("=")[1])
task_id = find_task_id(index)
+ elif filename.startswith("revision="):
+ index = "gecko.v2.revision.{}.decision".format(filename.split("=")[1])
+ task_id = find_task_id(index)
if task_id:
filename = get_artifact_url(task_id, 'public/parameters.yml')
f = urllib.urlopen(filename)
if filename.endswith('.yml'):
return Parameters(**yaml.safe_load(f))
elif filename.endswith('.json'):
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -1,57 +1,75 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import os
+import sys
from mozboot.util import get_state_dir
from mozbuild.base import MozbuildObject
from mozpack.files import FileFinder
from taskgraph.generator import TaskGraphGenerator
-from taskgraph.parameters import load_parameters_file
+from taskgraph.parameters import (
+ ParameterMismatchException,
+ load_parameters_file,
+)
here = os.path.abspath(os.path.dirname(__file__))
build = MozbuildObject.from_environment(cwd=here)
+PARAMETER_MISMATCH = """
+ERROR - The parameters being used to generate tasks differ from those defined
+in your working copy:
+
+ {}
+
+To fix this, either rebase onto the latest mozilla-central or pass in
+-p/--parameters. For more information on how to define parameters, see:
+https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters
+"""
+
+
def invalidate(cache):
if not os.path.isfile(cache):
return
tc_dir = os.path.join(build.topsrcdir, 'taskcluster')
tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
cmod = os.path.getmtime(cache)
if tmod > cmod:
os.remove(cache)
def generate_tasks(params=None, full=False):
- params = params or "project=mozilla-central"
-
cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return fh.read().splitlines()
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
- params = load_parameters_file(params)
- params.check()
+ try:
+ load_parameters_file(params)
+ params.check()
+ except ParameterMismatchException as e:
+ print(PARAMETER_MISMATCH.format(e.args[0]))
+ sys.exit(1)
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
labels = [label for label in tg.graph.visit_postorder()]