Bug 1412121 - [tryselect] First attempt to download the taskgraph from CI, r?dustin draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 21 Nov 2017 15:00:35 -0500
changeset 701568 fd6bed0f7a004935a969baafa1d9610f02ab301e
parent 701567 a40d15eb38605cb096d6df39b4bcbffa2cbfd05c
child 741200 f3f1a2e4f20be795d5620b893bb7594ef5d565fc
push id90198
push userahalberstadt@mozilla.com
push dateTue, 21 Nov 2017 20:53:45 +0000
reviewersdustin
bugs1412121
milestone59.0a1
Bug 1412121 - [tryselect] First attempt to download the taskgraph from CI, r?dustin Re-generating taskgraphs can take ~40 seconds on my machine. Downloading the taskgraph from CI only takes around 20. So first attempt to download it based on the parent revision. If something goes wrong or the 'taskcluster' directory was modified locally, we fall back to generating it. MozReview-Commit-ID: 2WRRs4AoXcK
tools/tryselect/tasks.py
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -3,80 +3,125 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
 import sys
 
+import requests
 from mozboot.util import get_state_dir
 from mozbuild.base import MozbuildObject
 from mozpack.files import FileFinder
+from mozversioncontrol import get_repository_object
 
 from taskgraph.generator import TaskGraphGenerator
 from taskgraph.parameters import (
     ParameterMismatch,
     load_parameters_file,
 )
 from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.taskcluster import (
+    get_artifact,
+    find_task_id,
+)
 
 here = os.path.abspath(os.path.dirname(__file__))
 build = MozbuildObject.from_environment(cwd=here)
 
 
 PARAMETER_MISMATCH = """
 ERROR - The parameters being used to generate tasks differ from those defined
 in your working copy:
 
     {}
 
 To fix this, either rebase onto the latest mozilla-central or pass in
 -p/--parameters. For more information on how to define parameters, see:
 https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters
 """
 
+PARAMETER_DEFAULT = "project=mozilla-central"
+
 
 def invalidate(cache):
     if not os.path.isfile(cache):
         return
 
     tc_dir = os.path.join(build.topsrcdir, 'taskcluster')
     tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
     cmod = os.path.getmtime(cache)
 
     if tmod > cmod:
         os.remove(cache)
 
 
-def get_taskgraph(params=None, full=False):
-    params = params or "project=mozilla-central"
-    cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
-    attr = 'full_task_graph' if full else 'target_task_graph'
-    cache = os.path.join(cache_dir, attr)
+def get_taskgraph_from_artifact(attr):
+    vcs = get_repository_object(build.topsrcdir)
+
+    # If files under /taskcluster are modified locally, then any downloaded
+    # taskgraph artifact will be invalid and there's no point in continuing.
+    if any(f.startswith('taskcluster') for f in vcs.get_outgoing_files()):
+        return
+
+    print("downloading {}".format(attr.replace('_', ' ')))
+    index = "gecko.v2.mozilla-central.revision.{}.firefox.decision".format(vcs.base_ref)
+    task_id = find_task_id(index)
 
-    invalidate(cache)
-    if os.path.isfile(cache):
-        with open(cache, 'r') as fh:
-            return TaskGraph.from_json(json.load(fh))[1]
+    step = attr.split('_')[0]
+    try:
+        response = get_artifact(task_id, 'public/{}-task-graph.json'.format(step))
+    except requests.HTTPError as e:
+        print("Bad response: {}.. ".format(e.response.status_code), end="")
+        sys.stdout.flush()
+        return
+    return TaskGraph.from_json(response)[1]
 
-    if not os.path.isdir(cache_dir):
-        os.makedirs(cache_dir)
 
-    print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
+def get_taskgraph_from_generator(attr, params):
+    print("generating {}".format(attr.replace('_', ' ')))
     try:
         params = load_parameters_file(params, strict=False)
         params.check()
     except ParameterMismatch as e:
         print(PARAMETER_MISMATCH.format(e.args[0]))
         sys.exit(1)
 
     cwd = os.getcwd()
     os.chdir(build.topsrcdir)
 
     root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
     tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
 
     os.chdir(cwd)
+    return tg
+
+
+def get_taskgraph(params=None, full=False):
+    params = params or PARAMETER_DEFAULT
+    cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
+    attr = 'full_task_graph' if full else 'target_task_graph'
+    cache = os.path.join(cache_dir, attr)
+
+    invalidate(cache)
+    if os.path.isfile(cache):
+        with open(cache, 'r') as fh:
+            return TaskGraph.from_json(json.load(fh))[1]
+
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+
+    # First attempt to download a taskgraph from a mozilla-central decision
+    # task artifact as it is faster. If that fails or if taskcluster was
+    # modified locally, fallback to generating the taskgraph.
+    print("Task configuration changed.. ", end="")
+    sys.stdout.flush()
+    tg = None
+    if params == PARAMETER_DEFAULT:
+        tg = get_taskgraph_from_artifact(attr)
+    if not tg:
+        tg = get_taskgraph_from_generator(attr, params)
 
     with open(cache, 'w') as fh:
         json.dump(tg.to_json(), fh)
+
     return tg