Bug 1412121 - [tryselect] First attempt to download the taskgraph from CI, r?dustin
Re-generating taskgraphs can take ~40 seconds on my machine. Downloading the
taskgraph from CI only takes around 20. So first attempt to download it based
on the parent revision.
If something goes wrong or the 'taskcluster' directory was modified locally, we
fall back to generating it.
MozReview-Commit-ID: 2WRRs4AoXcK
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -3,80 +3,125 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
import sys
+import requests
from mozboot.util import get_state_dir
from mozbuild.base import MozbuildObject
from mozpack.files import FileFinder
+from mozversioncontrol import get_repository_object
from taskgraph.generator import TaskGraphGenerator
from taskgraph.parameters import (
ParameterMismatch,
load_parameters_file,
)
from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.taskcluster import (
+ get_artifact,
+ find_task_id,
+)
here = os.path.abspath(os.path.dirname(__file__))
build = MozbuildObject.from_environment(cwd=here)
PARAMETER_MISMATCH = """
ERROR - The parameters being used to generate tasks differ from those defined
in your working copy:
{}
To fix this, either rebase onto the latest mozilla-central or pass in
-p/--parameters. For more information on how to define parameters, see:
https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters
"""
+PARAMETER_DEFAULT = "project=mozilla-central"
+
def invalidate(cache):
if not os.path.isfile(cache):
return
tc_dir = os.path.join(build.topsrcdir, 'taskcluster')
tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
cmod = os.path.getmtime(cache)
if tmod > cmod:
os.remove(cache)
-def get_taskgraph(params=None, full=False):
- params = params or "project=mozilla-central"
- cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
- attr = 'full_task_graph' if full else 'target_task_graph'
- cache = os.path.join(cache_dir, attr)
+def get_taskgraph_from_artifact(attr):
+ vcs = get_repository_object(build.topsrcdir)
+
+ # If files under /taskcluster are modified locally, then any downloaded
+ # taskgraph artifact will be invalid and there's no point in continuing.
+ if any(f.startswith('taskcluster') for f in vcs.get_outgoing_files()):
+ return
+
+ print("downloading {}".format(attr.replace('_', ' ')))
+ index = "gecko.v2.mozilla-central.revision.{}.firefox.decision".format(vcs.base_ref)
+ task_id = find_task_id(index)
- invalidate(cache)
- if os.path.isfile(cache):
- with open(cache, 'r') as fh:
- return TaskGraph.from_json(json.load(fh))[1]
+ step = attr.split('_')[0]
+ try:
+ response = get_artifact(task_id, 'public/{}-task-graph.json'.format(step))
+ except requests.HTTPError as e:
+ print("Bad response: {}.. ".format(e.response.status_code), end="")
+ sys.stdout.flush()
+ return
+ return TaskGraph.from_json(response)[1]
- if not os.path.isdir(cache_dir):
- os.makedirs(cache_dir)
- print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
+def get_taskgraph_from_generator(attr, params):
+ print("generating {}".format(attr.replace('_', ' ')))
try:
params = load_parameters_file(params, strict=False)
params.check()
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(build.topsrcdir, 'taskcluster', 'ci')
tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
os.chdir(cwd)
+ return tg
+
+
+def get_taskgraph(params=None, full=False):
+ params = params or PARAMETER_DEFAULT
+ cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
+ attr = 'full_task_graph' if full else 'target_task_graph'
+ cache = os.path.join(cache_dir, attr)
+
+ invalidate(cache)
+ if os.path.isfile(cache):
+ with open(cache, 'r') as fh:
+ return TaskGraph.from_json(json.load(fh))[1]
+
+ if not os.path.isdir(cache_dir):
+ os.makedirs(cache_dir)
+
+ # First attempt to download a taskgraph from a mozilla-central decision
+ # task artifact as it is faster. If that fails or if taskcluster was
+ # modified locally, fallback to generating the taskgraph.
+ print("Task configuration changed.. ", end="")
+ sys.stdout.flush()
+ tg = None
+ if params == PARAMETER_DEFAULT:
+ tg = get_taskgraph_from_artifact(attr)
+ if not tg:
+ tg = get_taskgraph_from_generator(attr, params)
with open(cache, 'w') as fh:
json.dump(tg.to_json(), fh)
+
return tg