Bug 1401199 - Autodetect relevant parameters draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 19 Sep 2017 15:32:20 -0400
changeset 672125 7eff4cf44b9e15199c4b3c298f61c0928c0e4f0d
parent 671959 e350889b001a917c8db4df8298c31d27c7c27899
child 733719 0c14e0f19dc3e8b13570eacb6c82461d2b52aaad
push id82157
push userahalberstadt@mozilla.com
push dateThu, 28 Sep 2017 19:19:27 +0000
bugs1401199
milestone58.0a1
Bug 1401199 - Autodetect relevant parameters MozReview-Commit-ID: GMiGuNApoUF
.taskcluster.yml
python/mozversioncontrol/mozversioncontrol/__init__.py
taskcluster/docs/mach.rst
taskcluster/mach_commands.py
taskcluster/taskgraph/parameters.py
tools/tryselect/tasks.py
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -35,16 +35,17 @@ tasks:
       $if: 'tasks_for == "hg-push"'
       then: {createdForUser: "${ownerEmail}"}
 
     routes:
       $if: 'tasks_for == "hg-push"'
       then:
         - "index.gecko.v2.${repository.project}.latest.firefox.decision"
         - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
+        - "index.gecko.v2.revision.${push.revision}.decision"
         - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
         - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
         - "notify.email.${ownerEmail}.on-failed"
         - "notify.email.${ownerEmail}.on-exception"
       else:
         - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
         - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
         - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
--- a/python/mozversioncontrol/mozversioncontrol/__init__.py
+++ b/python/mozversioncontrol/mozversioncontrol/__init__.py
@@ -175,16 +175,21 @@ class Repository(object):
         Returns True if the working directory does not have any file
         modifications. False otherwise.
 
         By default, untracked and ignored files are not considered. If
         ``untracked`` or ``ignored`` are set, they influence the clean check
         to factor these file classes into consideration.
         """
 
+    @abc.abstractmethod
+    def get_first_public_ancestor(self):
+        """Return the revision of the first public ancestor of the working
+        directory."""
+
 
 class HgRepository(Repository):
     '''An implementation of `Repository` for Mercurial repositories.'''
     def __init__(self, path, hg='hg'):
         import hglib.client
 
         super(HgRepository, self).__init__(path, tool=hg)
         self._env[b'HGPLAIN'] = b'1'
@@ -298,16 +303,20 @@ class HgRepository(Repository):
             args.append(b'--unknown')
         if ignored:
             args.append(b'--ignored')
 
         # If output is empty, there are no entries of requested status, which
         # means we are clean.
         return not len(self._run_in_client(args).strip())
 
+    def get_first_public_ancestor(self):
+        return self._run('log', '-r', 'first(reverse(ancestors(.)) and public())',
+                         '-T', '{node}')
+
 
 class GitRepository(Repository):
     '''An implementation of `Repository` for Git repositories.'''
     def __init__(self, path, git='git'):
         super(GitRepository, self).__init__(path, tool=git)
 
     @property
     def name(self):
@@ -360,16 +369,19 @@ class GitRepository(Repository):
         args = ['status', '--porcelain']
         if untracked:
             args.append('--untracked-files')
         if ignored:
             args.append('--ignored')
 
         return not len(self._run(*args).strip())
 
+    def get_first_public_ancestor(self):
+        return self._run('show', '--format=format:"%H"', self.get_upstream())
+
 
 def get_repository_object(path, hg='hg', git='git'):
     '''Get a repository object for the repository at `path`.
     If `path` is not a known VCS repository, raise an exception.
     '''
     if os.path.isdir(os.path.join(path, '.hg')):
         return HgRepository(path, hg=hg)
     elif os.path.exists(os.path.join(path, '.git')):
--- a/taskcluster/docs/mach.rst
+++ b/taskcluster/docs/mach.rst
@@ -19,29 +19,39 @@ graph-generation process and output the 
    Get the target task graph
 
 ``mach taskgraph optimized``
    Get the optimized task graph
 
 ``mach taskgraph morphed``
    Get the morhped task graph
 
-Each of these commands takes an optional ``--parameters`` option giving a file
-with parameters to guide the graph generation.  The decision task helpfully
-produces such a file on every run, and that is generally the easiest way to get
-a parameter file.  The parameter keys and values are described in
+See :doc:`how-tos` for further practical tips on debugging task-graph mechanics
+locally.
+
+Parameters
+----------
+
+Each of these commands takes an optional ``--parameters`` argument giving a
+file with parameters to guide the graph generation.  The decision task
+helpfully produces such a file on every run, and that is generally the easiest
+way to get a parameter file.  The parameter keys and values are described in
 :doc:`parameters`; using that information, you may modify an existing
 ``parameters.yml`` or create your own.  The ``--parameters`` option can also
-take an argument of the form ``project=<project>`` which will fetch the
-parameters from the latest push on that project; or ``task-id=<task-id>`` which
-will fetch the parameters from the given decision task. It defaults to
-``project=mozilla-central``.
+take the following forms:
 
-See :doc:`how-tos` for further practical tips on debugging task-graph mechanics
-locally.
+``project=<project>``
+   Fetch the parameters from the latest push on that project
+``task-id=<task-id>``
+   Fetch the parameters from the given decision task id
+``revision=<rev>``
+   Fetch the parameters from the decision task associated with the given revision.
+
+If unspecified, the taskgraph module will attempt to automatically fetch a
+relevant parameters.yml based on the state of your local repository.
 
 Taskgraph JSON Format
 ---------------------
 By default, the above commands will only output a list of tasks. Use `-J` flag
 to output full task definitions. For example:
 
 .. code-block:: shell
 
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -37,17 +37,17 @@ class ShowTaskGraphSubCommand(SubCommand
             CommandArgument('--verbose', '-v', action="store_true",
                             help="include debug-level logging output"),
             CommandArgument('--json', '-J', action="store_const",
                             dest="format", const="json",
                             help="Output task graph as a JSON object"),
             CommandArgument('--labels', '-L', action="store_const",
                             dest="format", const="labels",
                             help="Output the label for each task in the task graph (default)"),
-            CommandArgument('--parameters', '-p', default="project=mozilla-central",
+            CommandArgument('--parameters', '-p', default=None,
                             help="parameters file (.yml or .json; see "
                                  "`taskcluster/docs/parameters.rst`)`"),
             CommandArgument('--no-optimize', dest="optimize", action="store_false",
                             default="true",
                             help="do not remove tasks from the graph that are found in the "
                             "index (a.k.a. optimize the graph)"),
             CommandArgument('--tasks-regex', '--tasks', default=None,
                             help="only return tasks with labels matching this regular "
@@ -314,17 +314,17 @@ class MachCommands(MachCommandBase):
                     parameters=parameters,
                     test=False)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
 
     @SubCommand('taskgraph', 'test-action-callback',
                 description='Run an action callback in a testing mode')
-    @CommandArgument('--parameters', '-p', default='project=mozilla-central',
+    @CommandArgument('--parameters', '-p', default=None,
                      help='parameters file (.yml or .json; see '
                           '`taskcluster/docs/parameters.rst`)`')
     @CommandArgument('--task-id', default=None,
                      help='TaskId to which the action applies')
     @CommandArgument('--task-group-id', default=None,
                      help='TaskGroupId to which the action applies')
     @CommandArgument('--input', default=None,
                      help='Action input (.yml or .json)')
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -4,16 +4,24 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import yaml
 from mozbuild.util import ReadOnlyDict
+from mozversioncontrol import get_repository_object
+
+from . import GECKO
+
+
+class ParameterMismatchException(Exception):
+    """Raised when a parameters.yml has extra or missing parameters."""
+
 
 # Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
 PARAMETER_NAMES = set([
     'base_repository',
     'build_date',
     'filters',
     'head_ref',
     'head_repository',
@@ -45,52 +53,72 @@ class Parameters(ReadOnlyDict):
         if missing:
             msg.append("missing parameters: " + ", ".join(missing))
 
         extra = names - PARAMETER_NAMES
         if extra:
             msg.append("extra parameters: " + ", ".join(extra))
 
         if msg:
-            raise Exception("; ".join(msg))
+            raise ParameterMismatchException("; ".join(msg))
 
     def __getitem__(self, k):
         if k not in PARAMETER_NAMES:
             raise KeyError("no such parameter {!r}".format(k))
         try:
             return super(Parameters, self).__getitem__(k)
         except KeyError:
             raise KeyError("taskgraph parameter {!r} not found".format(k))
 
 
-def load_parameters_file(filename):
+def find_matching_parameters():
+    try:
+        # attempt to use mozilla-central first to avoid vcs penalty
+        params = load_parameters_file('project=mozilla-central')
+        params.check()
+        return params
+    except ParameterMismatchException as e:
+        repo = get_repository_object(GECKO)
+        rev = repo.get_first_public_ancestor()
+        try:
+            params = load_parameters_file('revision={}'.format(rev))
+            params.check()
+        except KeyError:
+            raise e
+        return params
+
+
+def load_parameters_file(filename=None):
     """
     Load parameters from a path, url, decision task-id or project.
 
     Examples:
         task-id=fdtgsD5DQUmAQZEaGMvQ4Q
         project=mozilla-central
     """
     import urllib
     from taskgraph.util.taskcluster import get_artifact_url, find_task_id
 
     if not filename:
-        return Parameters()
+        return find_matching_parameters()
 
     try:
         # reading parameters from a local parameters.yml file
         f = open(filename)
     except IOError:
         # fetching parameters.yml using task task-id, project or supplied url
         task_id = None
         if filename.startswith("task-id="):
             task_id = filename.split("=")[1]
         elif filename.startswith("project="):
             index = "gecko.v2.{}.latest.firefox.decision".format(filename.split("=")[1])
             task_id = find_task_id(index)
+        elif filename.startswith("revision="):
+            index = "gecko.v2.revision.{}.decision".format(filename.split("=")[1])
+            task_id = find_task_id(index)
 
         if task_id:
             filename = get_artifact_url(task_id, 'public/parameters.yml')
         f = urllib.urlopen(filename)
 
     if filename.endswith('.yml'):
         return Parameters(**yaml.safe_load(f))
     elif filename.endswith('.json'):
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -1,57 +1,75 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
+import sys
 
 from mozboot.util import get_state_dir
 from mozbuild.base import MozbuildObject
 from mozpack.files import FileFinder
 
 from taskgraph.generator import TaskGraphGenerator
-from taskgraph.parameters import load_parameters_file
+from taskgraph.parameters import (
+    ParameterMismatchException,
+    load_parameters_file,
+)
 
 here = os.path.abspath(os.path.dirname(__file__))
 build = MozbuildObject.from_environment(cwd=here)
 
 
+PARAMETER_MISMATCH = """
+ERROR - The parameters being used to generate tasks differ from those defined
+in your working copy:
+
+    {}
+
+To fix this, either rebase onto the latest mozilla-central or pass in
+-p/--parameters. For more information on how to define parameters, see:
+https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters
+"""
+
+
 def invalidate(cache):
     if not os.path.isfile(cache):
         return
 
     tc_dir = os.path.join(build.topsrcdir, 'taskcluster')
     tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
     cmod = os.path.getmtime(cache)
 
     if tmod > cmod:
         os.remove(cache)
 
 
 def generate_tasks(params=None, full=False):
-    params = params or "project=mozilla-central"
-
     cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
     attr = 'full_task_set' if full else 'target_task_set'
     cache = os.path.join(cache_dir, attr)
 
     invalidate(cache)
     if os.path.isfile(cache):
         with open(cache, 'r') as fh:
             return fh.read().splitlines()
 
     if not os.path.isdir(cache_dir):
         os.makedirs(cache_dir)
 
     print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
-    params = load_parameters_file(params)
-    params.check()
+    try:
+        load_parameters_file(params)
+        params.check()
+    except ParameterMismatchException as e:
+        print(PARAMETER_MISMATCH.format(e.args[0]))
+        sys.exit(1)
 
     cwd = os.getcwd()
     os.chdir(build.topsrcdir)
 
     root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
     tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
     labels = [label for label in tg.graph.visit_postorder()]