Bug 1258497: Factor out some common code; r=gps
This code will also be used by taskgraph's legacy kind.
MozReview-Commit-ID: Gd4zZYEee2W
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -30,160 +30,16 @@ ARTIFACT_URL = 'https://queue.taskcluste
DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
DEFAULT_TRY = 'try: -b do -p all -u all'
DEFAULT_JOB_PATH = os.path.join(
ROOT, 'tasks', 'branches', 'base_jobs.yml'
)
-def merge_dicts(*dicts):
- merged_dict = {}
- for dictionary in dicts:
- merged_dict.update(dictionary)
- return merged_dict
-
-def gaia_info():
- '''
- Fetch details from in tree gaia.json (which links this version of
- gecko->gaia) and construct the usual base/head/ref/rev pairing...
- '''
- gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
-
- if gaia['git'] is None or \
- gaia['git']['remote'] == '' or \
- gaia['git']['git_revision'] == '' or \
- gaia['git']['branch'] == '':
-
- # Just use the hg params...
- return {
- 'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
- 'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
- 'gaia_ref': gaia['revision'],
- 'gaia_rev': gaia['revision']
- }
-
- else:
- # Use git
- return {
- 'gaia_base_repository': gaia['git']['remote'],
- 'gaia_head_repository': gaia['git']['remote'],
- 'gaia_rev': gaia['git']['git_revision'],
- 'gaia_ref': gaia['git']['branch'],
- }
-
-def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
- """
- Configure a build dependent task. This is shared between post-build and test tasks.
-
- :param task_path: location to the task yaml
- :param parameters: parameters to load the template
- :param taskid: taskid of the dependent task
- :param templates: reference to the template builder
- :param build_treeherder_config: parent treeherder config
- :return: the configured task
- """
- task = templates.load(task_path, parameters)
- task['taskId'] = taskid
-
- if 'requires' not in task:
- task['requires'] = []
-
- task['requires'].append(parameters['build_slugid'])
-
- if 'treeherder' not in task['task']['extra']:
- task['task']['extra']['treeherder'] = {}
-
- # Copy over any treeherder configuration from the build so
- # tests show up under the same platform...
- treeherder_config = task['task']['extra']['treeherder']
-
- treeherder_config['collection'] = \
- build_treeherder_config.get('collection', {})
-
- treeherder_config['build'] = \
- build_treeherder_config.get('build', {})
-
- treeherder_config['machine'] = \
- build_treeherder_config.get('machine', {})
-
- if 'routes' not in task['task']:
- task['task']['routes'] = []
-
- if 'scopes' not in task['task']:
- task['task']['scopes'] = []
-
- return task
-
-def set_interactive_task(task, interactive):
- r"""Make the task interactive.
-
- :param task: task definition.
- :param interactive: True if the task should be interactive.
- """
- if not interactive:
- return
-
- payload = task["task"]["payload"]
- if "features" not in payload:
- payload["features"] = {}
- payload["features"]["interactive"] = True
-
-def remove_caches_from_task(task):
- r"""Remove all caches but tc-vcs from the task.
-
- :param task: task definition.
- """
- whitelist = [
- re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
- re.compile("^tooltool-cache$"),
- ]
- try:
- caches = task["task"]["payload"]["cache"]
- for cache in caches.keys():
- if not any(pat.match(cache) for pat in whitelist):
- caches.pop(cache)
- except KeyError:
- pass
-
-def query_vcs_info(repository, revision):
- """Query the pushdate and pushid of a repository/revision.
- This is intended to be used on hg.mozilla.org/mozilla-central and
- similar. It may or may not work for other hg repositories.
- """
- if not repository or not revision:
- sys.stderr.write('cannot query vcs info because vcs info not provided\n')
- return None
-
- VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
-
- try:
- import requests
- url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
- revision)
- sys.stderr.write("Querying version control for metadata: %s\n" % url)
- contents = requests.get(url).json()
-
- changesets = []
- for c in contents['changesets']:
- changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
-
- pushid = contents['changesets'][-1]['pushid']
- pushdate = contents['changesets'][-1]['pushdate'][0]
-
- return VCSInfo(pushid, pushdate, changesets)
-
- except Exception:
- sys.stderr.write(
- "Error querying VCS info for '%s' revision '%s'\n" % (
- repository, revision,
- )
- )
- return None
-
@CommandProvider
class DecisionTask(object):
@Command('taskcluster-decision', category="ci",
description="Build a decision task")
@CommandArgument('--project',
required=True,
help='Treeherder project name')
@CommandArgument('--url',
@@ -197,16 +53,17 @@ class DecisionTask(object):
@CommandArgument('--comment',
required=True,
help='Commit message for this revision')
@CommandArgument('--owner',
required=True,
help='email address of who owns this graph')
@CommandArgument('task', help="Path to decision task to run.")
def run_task(self, **params):
+ from taskcluster_graph.mach_util import gaia_info
from taskcluster_graph.slugidjar import SlugidJar
from taskcluster_graph.from_now import (
json_time_from_now,
current_json_time,
)
from taskcluster_graph.templates import Templates
templates = Templates(ROOT)
@@ -313,16 +170,24 @@ class Graph(object):
help='Run tasks even if their conditions are not met')
def create_graph(self, **params):
from functools import partial
from mozpack.path import match as mozpackmatch
from slugid import nice as slugid
+ from taskcluster_graph.mach_util import (
+ merge_dicts,
+ gaia_info,
+ configure_dependent_task,
+ set_interactive_task,
+ remove_caches_from_task,
+ query_vcs_info
+ )
import taskcluster_graph.transform.routes as routes_transform
import taskcluster_graph.transform.treeherder as treeherder_transform
from taskcluster_graph.commit_parser import parse_commit
from taskcluster_graph.image_builder import (
docker_image,
normalize_image_details,
task_id_for_image
)
@@ -689,16 +554,21 @@ class CIBuild(object):
help='path to build task definition')
@CommandArgument('--interactive',
required=False,
default=False,
action="store_true",
dest="interactive",
help="Run the task with the interactive feature enabled")
def create_ci_build(self, **params):
+ from taskcluster_graph.mach_util import (
+ gaia_info,
+ set_interactive_task,
+ query_vcs_info
+ )
from taskcluster_graph.templates import Templates
from taskcluster_graph.image_builder import docker_image
import taskcluster_graph.build_task
templates = Templates(ROOT)
# TODO handle git repos
head_repository = params['head_repository']
if not head_repository:
new file mode 100644
--- /dev/null
+++ b/testing/taskcluster/taskcluster_graph/mach_util.py
@@ -0,0 +1,162 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+from collections import defaultdict
+import os
+import json
+import copy
+import re
+import sys
+import time
+from collections import namedtuple
+
+ROOT = os.path.dirname(os.path.realpath(__file__))
+GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..'))
+
+def merge_dicts(*dicts):
+ merged_dict = {}
+ for dictionary in dicts:
+ merged_dict.update(dictionary)
+ return merged_dict
+
+def gaia_info():
+ '''
+ Fetch details from in tree gaia.json (which links this version of
+ gecko->gaia) and construct the usual base/head/ref/rev pairing...
+ '''
+ gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
+
+ if gaia['git'] is None or \
+ gaia['git']['remote'] == '' or \
+ gaia['git']['git_revision'] == '' or \
+ gaia['git']['branch'] == '':
+
+ # Just use the hg params...
+ return {
+ 'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
+ 'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
+ 'gaia_ref': gaia['revision'],
+ 'gaia_rev': gaia['revision']
+ }
+
+ else:
+ # Use git
+ return {
+ 'gaia_base_repository': gaia['git']['remote'],
+ 'gaia_head_repository': gaia['git']['remote'],
+ 'gaia_rev': gaia['git']['git_revision'],
+ 'gaia_ref': gaia['git']['branch'],
+ }
+
+def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
+ """
+ Configure a build dependent task. This is shared between post-build and test tasks.
+
+ :param task_path: location to the task yaml
+ :param parameters: parameters to load the template
+ :param taskid: taskid of the dependent task
+ :param templates: reference to the template builder
+ :param build_treeherder_config: parent treeherder config
+ :return: the configured task
+ """
+ task = templates.load(task_path, parameters)
+ task['taskId'] = taskid
+
+ if 'requires' not in task:
+ task['requires'] = []
+
+ task['requires'].append(parameters['build_slugid'])
+
+ if 'treeherder' not in task['task']['extra']:
+ task['task']['extra']['treeherder'] = {}
+
+ # Copy over any treeherder configuration from the build so
+ # tests show up under the same platform...
+ treeherder_config = task['task']['extra']['treeherder']
+
+ treeherder_config['collection'] = \
+ build_treeherder_config.get('collection', {})
+
+ treeherder_config['build'] = \
+ build_treeherder_config.get('build', {})
+
+ treeherder_config['machine'] = \
+ build_treeherder_config.get('machine', {})
+
+ if 'routes' not in task['task']:
+ task['task']['routes'] = []
+
+ if 'scopes' not in task['task']:
+ task['task']['scopes'] = []
+
+ return task
+
+def set_interactive_task(task, interactive):
+ r"""Make the task interactive.
+
+ :param task: task definition.
+ :param interactive: True if the task should be interactive.
+ """
+ if not interactive:
+ return
+
+ payload = task["task"]["payload"]
+ if "features" not in payload:
+ payload["features"] = {}
+ payload["features"]["interactive"] = True
+
+def remove_caches_from_task(task):
+ r"""Remove all caches but tc-vcs from the task.
+
+ :param task: task definition.
+ """
+ whitelist = [
+ re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
+ re.compile("^tooltool-cache$"),
+ ]
+ try:
+ caches = task["task"]["payload"]["cache"]
+ for cache in caches.keys():
+ if not any(pat.match(cache) for pat in whitelist):
+ caches.pop(cache)
+ except KeyError:
+ pass
+
+def query_vcs_info(repository, revision):
+ """Query the pushdate and pushid of a repository/revision.
+ This is intended to be used on hg.mozilla.org/mozilla-central and
+ similar. It may or may not work for other hg repositories.
+ """
+ if not repository or not revision:
+ sys.stderr.write('cannot query vcs info because vcs info not provided\n')
+ return None
+
+ VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
+
+ try:
+ import requests
+ url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
+ revision)
+ sys.stderr.write("Querying version control for metadata: %s\n" % url)
+ contents = requests.get(url).json()
+
+ changesets = []
+ for c in contents['changesets']:
+ changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
+
+ pushid = contents['changesets'][-1]['pushid']
+ pushdate = contents['changesets'][-1]['pushdate'][0]
+
+ return VCSInfo(pushid, pushdate, changesets)
+
+ except Exception:
+ sys.stderr.write(
+ "Error querying VCS info for '%s' revision '%s'\n" % (
+ repository, revision,
+ )
+ )
+ return None
+
+