Bug 1280129 - Update tasks to use v2 treeherder routes r=dustin
Jobs reporting to treeherder should rely on the task route for project,
revision, and pushlog ID rather than things stuffed into task.extra.treeherder.
This also removes the need for a revision_hash that was calculated by mozilla-taskcluster.
MozReview-Commit-ID: EcQM9QRZzgG
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -55,18 +55,18 @@ tasks:
- "docker-worker:cache:level-{{level}}-*"
- "docker-worker:cache:tooltool-cache"
- "secrets:get:project/taskcluster/gecko/hgfingerprint"
# mozilla-taskcluster will append the appropriate assume:repo:<repo>
# scope here.
routes:
- "index.gecko.v2.{{project}}.latest.firefox.decision"
- - "tc-treeherder.{{project}}.{{revision_hash}}"
- - "tc-treeherder-stage.{{project}}.{{revision_hash}}"
+ - "tc-treeherder.v2.{{project}}.{{revision}}.{{pushlog_id}}"
+ - "tc-treeherder-stage.v2.{{project}}.{{revision}}.{{pushlog_id}}"
payload:
env:
# checkout-gecko uses these to check out the source; the inputs
# to `mach taskgraph decision` are all on the command line.
GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
GECKO_HEAD_REPOSITORY: '{{{url}}}'
GECKO_HEAD_REF: '{{revision}}'
--- a/taskcluster/ci/docker-image/image.yml
+++ b/taskcluster/ci/docker-image/image.yml
@@ -13,18 +13,18 @@ task:
workerType: taskcluster-images
provisionerId: aws-provisioner-v1
schedulerId: task-graph-scheduler
routes:
- index.docker.images.v1.{{project}}.{{image_name}}.latest
- index.docker.images.v1.{{project}}.{{image_name}}.pushdate.{{year}}.{{month}}-{{day}}-{{pushtime}}
- index.docker.images.v1.{{project}}.{{image_name}}.hash.{{context_hash}}
- - tc-treeherder.{{project}}.{{revision_hash}}
- - tc-treeherder-stage.{{project}}.{{revision_hash}}
+ - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
+ - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
payload:
env:
HASH: '{{context_hash}}'
PROJECT: '{{project}}'
CONTEXT_URL: '{{context_url}}'
CONTEXT_PATH: '{{context_path}}'
BASE_REPOSITORY: '{{base_repository}}'
@@ -45,14 +45,12 @@ task:
path: '/artifacts/image.tar'
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:
- staging
- production
treeherder:
jobKind: other
- revision: {{head_rev}}
- revision_hash: {{revision_hash}}
build:
platform: 'taskcluster-images'
symbol: 'I'
--- a/taskcluster/ci/legacy/tasks/decision/branch.yml
+++ b/taskcluster/ci/legacy/tasks/decision/branch.yml
@@ -12,18 +12,18 @@ scopes:
- queue:*
- docker-worker:*
- scheduler:*
tasks:
- taskId: '{{#as_slugid}}decision task{{/as_slugid}}'
task:
routes:
- index.gecko.v2.{{project}}.latest.firefox.decision
- - tc-treeherder-stage.{{project}}.{{revision_hash}}
- - tc-treeherder.{{project}}.{{revision_hash}}
+ - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
+ - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
created: '{{now}}'
deadline: '{{#from_now}}1 day{{/from_now}}'
metadata:
owner: mozilla-taskcluster-maintenance@mozilla.com
source: {{{source}}}
name: Initial decision task for {{project}} specific graphs
description: |
This is the single most important task as it decides how all other tasks
@@ -74,26 +74,23 @@ tasks:
checkout-gecko workspace &&
cd workspace/gecko &&
./mach taskcluster-graph
--pushlog-id='{{pushlog_id}}'
--project='{{project}}'
--message='{{comment}}'
--owner='{{owner}}'
--level='{{level}}'
- --revision-hash='{{revision_hash}}'
--extend-graph > /home/worker/artifacts/graph.json &&
cp /home/worker/artifacts/graph.json /home/worker/artifacts/all_tasks.json
graphs:
- /home/worker/artifacts/graph.json
artifacts:
'public':
type: 'directory'
path: '/home/worker/artifacts'
# Arbitrary value for keeping these artifacts around. They are just the
# graph.json and context directories for now, so nothing that needs
# to stay around for long.
expires: '{{#from_now}}7 days{{/from_now}}'
extra:
treeherder:
symbol: D
- revision: '{{revision}}'
- revision_hash: '{{revision_hash}}'
--- a/taskcluster/ci/legacy/tasks/decision/try.yml
+++ b/taskcluster/ci/legacy/tasks/decision/try.yml
@@ -36,18 +36,18 @@ tasks:
# Bug 1269443: cache scopes, etc. must be listed explicitly
- "docker-worker:cache:level-1-*"
- "docker-worker:cache:tooltool-cache"
- "secrets:get:project/taskcluster/gecko/hgfingerprint"
- "assume:repo:hg.mozilla.org/try:*"
routes:
- "index.gecko.v2.{{project}}.latest.firefox.decision"
- - "tc-treeherder.{{project}}.{{revision_hash}}"
- - "tc-treeherder-stage.{{project}}.{{revision_hash}}"
+ - "tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
+ - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
payload:
env:
GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
GECKO_HEAD_REPOSITORY: '{{{url}}}'
GECKO_HEAD_REF: '{{revision}}'
GECKO_HEAD_REV: '{{revision}}'
@@ -85,21 +85,18 @@ tasks:
--project='{{project}}'
--message='{{comment}}'
--owner='{{owner}}'
--level='{{level}}'
--base-repository='https://hg.mozilla.org/mozilla-central'
--head-repository='{{{url}}}'
--head-ref='{{revision}}'
--head-rev='{{revision}}'
- --revision-hash='{{revision_hash}}'
artifacts:
'public':
type: 'directory'
path: '/home/worker/artifacts'
expires: '{{#from_now}}7 days{{/from_now}}'
extra:
treeherder:
symbol: D
- revision: '{{revision}}'
- revision_hash: '{{revision_hash}}'
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -30,19 +30,16 @@ Push Information
The revision to check out; this can be a short revision string
``head_ref``
For Mercurial repositories, this is the same as ``head_rev``. For
git repositories, which do not allow pulling explicit revisions, this gives
the symbolic ref containing ``head_rev`` that should be pulled from
``head_repository``.
-``revision_hash``
- The full-length revision string
-
``owner``
Email address indicating the person who made the push. Note that this
value may be forged and *must not* be relied on for authentication.
``message``
The commit message
``pushlog_id``
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -85,17 +85,16 @@ def get_decision_parameters(options):
This also applies per-project parameters, based on the given project.
"""
parameters = {n: options[n] for n in [
'base_repository',
'head_repository',
'head_rev',
'head_ref',
- 'revision_hash',
'message',
'project',
'pushlog_id',
'owner',
'level',
'target_tasks_method',
] if n in options}
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -32,32 +32,32 @@ INDEX_URL = 'https://index.taskcluster.n
class DockerImageKind(base.Kind):
def load_tasks(self, params):
# TODO: make this match the pushdate (get it from a parameter rather than vcs)
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
parameters = {
+ 'pushlog_id': params.get('pushlog_id', 0),
'pushdate': pushdate,
'pushtime': pushdate[8:],
'year': pushdate[0:4],
'month': pushdate[4:6],
'day': pushdate[6:8],
'project': params['project'],
'docker_image': docker_image,
'base_repository': params['base_repository'] or params['head_repository'],
'head_repository': params['head_repository'],
'head_ref': params['head_ref'] or params['head_rev'],
'head_rev': params['head_rev'],
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
- 'revision_hash': params['revision_hash'],
'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
.format(repo=params['head_repository'], rev=params['head_rev']),
}
tasks = []
templates = Templates(self.path)
for image_name in self.config['images']:
context_path = os.path.join('testing', 'docker', image_name)
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -34,17 +34,16 @@ GECKO = os.path.realpath(os.path.join(RO
TASKID_PLACEHOLDER = 'TaskLabel=={}'
DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
DEFAULT_TRY = 'try: -b do -p all -u all -t all'
DEFAULT_JOB_PATH = os.path.join(
'tasks', 'branches', 'base_jobs.yml'
)
-TREEHERDER_ROUTE_PREFIX = 'tc-treeherder-stage'
TREEHERDER_ROUTES = {
'staging': 'tc-treeherder-stage',
'production': 'tc-treeherder'
}
# time after which a try build's results will expire
TRY_EXPIRATION = "14 days"
@@ -205,45 +204,48 @@ def set_expiration(task, timestamp):
return
# for docker-worker, artifacts is a dictionary
# for generic-worker, artifacts is a list
# for taskcluster-worker, it will depend on what we do in artifacts plugin
for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
artifact['expires'] = timestamp
-def add_treeherder_revision_info(task, revision, revision_hash):
- # Only add treeherder information if task.extra.treeherder is present
- if 'extra' not in task and 'treeherder' not in task.extra:
- return
+def format_treeherder_route(destination, project, revision, pushlog_id):
+ return "{}.v2.{}.{}.{}".format(destination,
+ project,
+ revision,
+ pushlog_id)
- task['extra']['treeherder']['revision'] = revision
- task['extra']['treeherder']['revision_hash'] = revision_hash
-
-
-def decorate_task_treeherder_routes(task, suffix):
+def decorate_task_treeherder_routes(task, project, revision, pushlog_id):
"""Decorate the given task with treeherder routes.
Uses task.extra.treeherderEnv if available otherwise defaults to only
staging.
:param dict task: task definition.
- :param str suffix: The project/revision_hash portion of the route.
+ :param str project: The project the tasks are running for.
+ :param str revision: The revision for the push
+ :param str pushlog_id: The ID of the push
"""
if 'extra' not in task:
return
if 'routes' not in task:
task['routes'] = []
treeheder_env = task['extra'].get('treeherderEnv', ['staging'])
for env in treeheder_env:
- task['routes'].append('{}.{}'.format(TREEHERDER_ROUTES[env], suffix))
+ route = format_treeherder_route(TREEHERDER_ROUTES[env],
+ project,
+ revision,
+ pushlog_id)
+ task['routes'].append(route)
def decorate_task_json_routes(task, json_routes, parameters):
"""Decorate the given task with routes.json routes.
:param dict task: task definition.
:param json_routes: the list of routes to use from routes.json
:param parameters: dictionary of parameters to use in route templates
"""
@@ -337,42 +339,36 @@ class LegacyKind(base.Kind):
'pushtime': pushdate[8:],
'year': pushdate[0:4],
'month': pushdate[4:6],
'day': pushdate[6:8],
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
- 'revision_hash': params['revision_hash']
}.items())
- treeherder_route = '{}.{}'.format(
- params['project'],
- params.get('revision_hash', '')
- )
-
routes_file = os.path.join(root, 'routes.json')
with open(routes_file) as f:
contents = json.load(f)
json_routes = contents['routes']
# TODO: Nightly and/or l10n routes
# Task graph we are generating for taskcluster...
graph = {
'tasks': [],
'scopes': set(),
}
- if params['revision_hash']:
- for env in TREEHERDER_ROUTES:
- route = 'queue:route:{}.{}'.format(
- TREEHERDER_ROUTES[env],
- treeherder_route)
- graph['scopes'].add(route)
+ for env in TREEHERDER_ROUTES:
+ route = format_treeherder_route(TREEHERDER_ROUTES[env],
+ parameters['project'],
+ parameters['head_rev'],
+ parameters['pushlog_id'])
+ graph['scopes'].add("queue:route:{}".format(route))
graph['metadata'] = {
'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
'owner': params['owner'],
# TODO: Add full mach commands to this example?
'description': 'Task graph generated via ./mach taskcluster-graph',
'name': 'task graph local'
}
@@ -433,25 +429,23 @@ class LegacyKind(base.Kind):
set_interactive_task(build_task, interactive)
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
- if params['revision_hash']:
- add_treeherder_revision_info(build_task['task'],
- params['head_rev'],
- params['revision_hash'])
- decorate_task_treeherder_routes(build_task['task'],
- treeherder_route)
- decorate_task_json_routes(build_task['task'],
- json_routes,
- build_parameters)
+ decorate_task_treeherder_routes(build_task['task'],
+ build_parameters['project'],
+ build_parameters['head_rev'],
+ build_parameters['pushlog_id'])
+ decorate_task_json_routes(build_task['task'],
+ json_routes,
+ build_parameters)
# Ensure each build graph is valid after construction.
validate_build_task(build_task)
attributes = build_task['attributes'] = {'kind':'legacy', 'legacy_kind': 'build'}
if 'build_name' in build:
attributes['build_platform'] = build['build_name']
if 'build_type' in task_extra:
attributes['build_type'] = {'dbg': 'debug'}.get(task_extra['build_type'],
@@ -510,19 +504,16 @@ class LegacyKind(base.Kind):
post_parameters = merge_dicts(build_parameters,
post_build.get('additional-parameters', {}))
post_task = configure_dependent_task(post_build['task'],
post_parameters,
mklabel(),
templates,
build_treeherder_config)
set_interactive_task(post_task, interactive)
- add_treeherder_revision_info(post_task['task'],
- params['head_rev'],
- params['revision_hash'])
if project == "try":
set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
post_task['attributes'] = attributes.copy()
post_task['attributes']['legacy_kind'] = 'post_build'
post_task['attributes']['post_build'] = post_build['job_flag']
graph['tasks'].append(post_task)
@@ -560,24 +551,20 @@ class LegacyKind(base.Kind):
test_parameters['chunk'] = chunk
test_task = configure_dependent_task(test['task'],
test_parameters,
mklabel(),
templates,
build_treeherder_config)
set_interactive_task(test_task, interactive)
- if params['revision_hash']:
- add_treeherder_revision_info(test_task['task'],
- params['head_rev'],
- params['revision_hash'])
- decorate_task_treeherder_routes(
- test_task['task'],
- treeherder_route
- )
+ decorate_task_treeherder_routes(test_task['task'],
+ test_parameters['project'],
+ test_parameters['head_rev'],
+ test_parameters['pushlog_id'])
if project == "try":
set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
test_task['attributes'] = attributes.copy()
test_task['attributes']['legacy_kind'] = 'unittest'
test_task['attributes']['test_platform'] = attributes['build_platform']
test_task['attributes']['unittest_try_name'] = test['unittest_try_name']