--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -1,139 +1,193 @@
# This file is rendered via JSON-e by
# - mozilla-taskcluster - https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
# - cron tasks - taskcluster/taskgraph/cron/decision.py
+# - action tasks - taskcluster/taskgraph/actions/registry.py
version: 1
tasks:
- $let:
- # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
- ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
- # ensure there's no trailing `/` on the repo URL
- repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
- in:
- - taskId: '${as_slugid("decision")}'
- taskGroupId: '${as_slugid("decision")}' # same as tsakId; this is how automation identifies a decision tsak
- schedulerId: 'gecko-level-${repository.level}'
-
- created: {$fromNow: ''}
- deadline: {$fromNow: '1 day'}
- expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
- metadata:
- $merge:
- - owner: "${ownerEmail}"
- source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
- - $if: 'tasks_for == "hg-push"'
- then:
- name: "Gecko Decision Task"
- description: 'The task that creates all of the other tasks in the task graph'
- else:
- name: "Decision Task for cron job ${cron.job_name}"
- description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
-
- provisionerId: "aws-provisioner-v1"
- workerType: "gecko-${repository.level}-decision"
-
- tags:
- $if: 'tasks_for == "hg-push"'
- then: {createdForUser: "${ownerEmail}"}
-
- routes:
- $if: 'tasks_for == "hg-push"'
- then:
- - "index.gecko.v2.${repository.project}.latest.firefox.decision"
- - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
- - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
- - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
- - "notify.email.${ownerEmail}.on-failed"
- - "notify.email.${ownerEmail}.on-exception"
- else:
- - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
- - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
- - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-
- scopes:
- $if: 'tasks_for == "hg-push"'
- then:
- - 'assume:repo:${repoUrl[8:]}:*'
- - 'queue:route:notify.email.${ownerEmail}.*'
- else:
- - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
-
- dependencies: []
- requires: all-completed
-
- priority: lowest
- retries: 5
+ - $let:
+ # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
+ ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
+ # ensure there's no trailing `/` on the repo URL
+ repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
+ in:
+ taskId: {$if: 'tasks_for != "action"', then: '${as_slugid("decision")}'}
+ taskGroupId:
+ $if: 'tasks_for == "action"'
+ then:
+ '${action.taskGroupId}'
+ else:
+ '${as_slugid("decision")}' # same as taskId; this is how automation identifies a decision tsak
+ schedulerId: 'gecko-level-${repository.level}'
- payload:
- env:
- # checkout-gecko uses these to check out the source; the inputs
- # to `mach taskgraph decision` are all on the command line.
- GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
- GECKO_HEAD_REPOSITORY: '${repoUrl}'
- GECKO_HEAD_REF: '${push.revision}'
- GECKO_HEAD_REV: '${push.revision}'
- GECKO_COMMIT_MSG: '${push.comment}'
- HG_STORE_PATH: /builds/worker/checkouts/hg-store
- TASKCLUSTER_CACHES: /builds/worker/checkouts
-
- cache:
- level-${repository.level}-checkouts-sparse-v1: /builds/worker/checkouts
-
- features:
- taskclusterProxy: true
- chainOfTrust: true
-
- # Note: This task is built server side without the context or tooling that
- # exist in tree so we must hard code the hash
- # XXX Changing this will break Chain of Trust without an associated puppet and
- # scriptworker patch!
- image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef'
-
- maxRunTime: 1800
-
- # TODO use mozilla-unified for the base repository once the tc-vcs
- # tar.gz archives are created or tc-vcs isn't being used.
- command:
- - /builds/worker/bin/run-task
- - '--vcs-checkout=/builds/worker/checkouts/gecko'
- - '--sparse-profile=build/sparse-profiles/taskgraph'
- - '--'
- - bash
- - -cx
- - $let:
- extraArgs: {$if: 'tasks_for == "hg-push"', then: '', else: '${cron.quoted_args}'}
- # NOTE: the explicit reference to mozilla-central below is required because android-stuff
- # still uses tc-vcs, which does not support mozilla-unified
- # https://bugzilla.mozilla.org/show_bug.cgi?id=1383973
- in: >
- cd /builds/worker/checkouts/gecko &&
- ln -s /builds/worker/artifacts artifacts &&
- ./mach --log-no-times taskgraph decision
- --pushlog-id='${push.pushlog_id}'
- --pushdate='${push.pushdate}'
- --project='${repository.project}'
- --message="$GECKO_COMMIT_MSG"
- --owner='${ownerEmail}'
- --level='${repository.level}'
- --base-repository='https://hg.mozilla.org/mozilla-central'
- --head-repository="$GECKO_HEAD_REPOSITORY"
- --head-ref="$GECKO_HEAD_REF"
- --head-rev="$GECKO_HEAD_REV"
- ${extraArgs}
-
- artifacts:
- 'public':
- type: 'directory'
- path: '/builds/worker/artifacts'
- expires: {$fromNow: '1 year'}
-
- extra:
- treeherder:
+ created: {$fromNow: ''}
+ deadline: {$fromNow: '1 day'}
+ expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
+ metadata:
$merge:
- - machine:
- platform: gecko-decision
+ - owner: "${ownerEmail}"
+ source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
- $if: 'tasks_for == "hg-push"'
then:
- symbol: D
+ name: "Gecko Decision Task"
+ description: 'The task that creates all of the other tasks in the task graph'
else:
- groupSymbol: cron
- symbol: "${cron.job_symbol}"
+ $if: 'tasks_for == "action"'
+ then:
+ name: "Action: ${action.title}"
+ description: '${action.description}'
+ else:
+ name: "Decision Task for cron job ${cron.job_name}"
+ description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
+
+ provisionerId: "aws-provisioner-v1"
+ workerType: "gecko-${repository.level}-decision"
+
+ tags:
+ $if: 'tasks_for == "hg-push"'
+ then: {createdForUser: "${ownerEmail}"}
+ else:
+ $if: 'tasks_for == "action"'
+ then:
+ createdForUser: '${ownerEmail}'
+ kind: 'action-callback'
+
+ routes:
+ $if: 'tasks_for == "hg-push"'
+ then:
+ - "index.gecko.v2.${repository.project}.latest.firefox.decision"
+ - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
+ - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+ - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+ - "notify.email.${ownerEmail}.on-failed"
+ - "notify.email.${ownerEmail}.on-exception"
+ else:
+ - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+ - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+ - $if: 'tasks_for == "action"'
+ then: "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.$ownTaskId"
+ else: "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
+
+ scopes:
+ $if: 'tasks_for == "hg-push"'
+ then:
+ - 'assume:repo:${repoUrl[8:]}:*'
+ - 'queue:route:notify.email.${ownerEmail}.*'
+ else:
+ $if: 'tasks_for == "action"'
+ then:
+ - '${action.repo_scope}'
+ else:
+ - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
+
+ dependencies: []
+ requires: all-completed
+
+ priority: lowest
+ retries: 5
+
+ payload:
+ env:
+ # checkout-gecko uses these to check out the source; the inputs
+ # to `mach taskgraph decision` are all on the command line.
+ $merge:
+ - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
+ GECKO_HEAD_REPOSITORY: '${repoUrl}'
+ GECKO_HEAD_REF: '${push.revision}'
+ GECKO_HEAD_REV: '${push.revision}'
+ GECKO_COMMIT_MSG: {$if: 'tasks_for != "action"', then: '${push.comment}'}
+ HG_STORE_PATH: /builds/worker/checkouts/hg-store
+ TASKCLUSTER_CACHES: /builds/worker/checkouts
+ - $if: 'tasks_for == "action"'
+ then:
+ ACTION_TASK_GROUP_ID: '${action.taskGroupId}'
+ ACTION_TASK_ID: {$json: {$eval: 'taskId'}}
+ ACTION_TASK: {$json: {$eval: 'task'}}
+ ACTION_INPUT: {$json: {$eval: 'input'}}
+ ACTION_CALLBACK: '${action.cb_name}'
+ ACTION_PARAMETERS: {$json: {$eval: 'parameters'}}
+
+ cache:
+ level-${repository.level}-checkouts-sparse-v1: /builds/worker/checkouts
+
+ features:
+ taskclusterProxy: true
+ chainOfTrust: true
+
+ # Note: This task is built server side without the context or tooling that
+ # exist in tree so we must hard code the hash
+ # XXX Changing this will break Chain of Trust without an associated puppet and
+ # scriptworker patch!
+ image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef'
+
+ maxRunTime: 1800
+
+ # TODO use mozilla-unified for the base repository once the tc-vcs
+ # tar.gz archives are created or tc-vcs isn't being used.
+ command:
+ - /builds/worker/bin/run-task
+ - '--vcs-checkout=/builds/worker/checkouts/gecko'
+ - '--sparse-profile=build/sparse-profiles/taskgraph'
+ - '--'
+ - bash
+ - -cx
+ - $let:
+ extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
+ # NOTE: the explicit reference to mozilla-central below is required because android-stuff
+ # still uses tc-vcs, which does not support mozilla-unified
+ # https://bugzilla.mozilla.org/show_bug.cgi?id=1383973
+ in:
+ $if: 'tasks_for == "action"'
+ then: >
+ cd /builds/worker/checkouts/gecko &&
+ ln -s /builds/worker/artifacts artifacts &&
+ ./mach --log-no-times taskgraph action-callback
+ else: >
+ cd /builds/worker/checkouts/gecko &&
+ ln -s /builds/worker/artifacts artifacts &&
+ ./mach --log-no-times taskgraph decision
+ --pushlog-id='${push.pushlog_id}'
+ --pushdate='${push.pushdate}'
+ --project='${repository.project}'
+ --message="$GECKO_COMMIT_MSG"
+ --owner='${ownerEmail}'
+ --level='${repository.level}'
+ --base-repository='https://hg.mozilla.org/mozilla-central'
+ --head-repository="$GECKO_HEAD_REPOSITORY"
+ --head-ref="$GECKO_HEAD_REF"
+ --head-rev="$GECKO_HEAD_REV"
+ ${extraArgs}
+
+ artifacts:
+ 'public':
+ type: 'directory'
+ path: '/builds/worker/artifacts'
+ expires: {$fromNow: '1 year'}
+
+ extra:
+ $merge:
+ - treeherder:
+ $merge:
+ - machine:
+ platform: gecko-decision
+ - $if: 'tasks_for == "hg-push"'
+ then:
+ symbol: D
+ else:
+ $if: 'tasks_for == "action"'
+ then:
+ groupName: 'action-callback'
+ groupSymbol: AC
+ symbol: "${action.symbol}"
+ else:
+ groupSymbol: cron
+ symbol: "${cron.job_symbol}"
+ - $if: 'tasks_for == "action"'
+ then:
+ parent: '${action.taskGroupId}'
+ action:
+ name: '${action.name}'
+ context:
+ taskGroupId: '${action.taskGroupId}'
+ taskId: {$eval: 'taskId'}
+ input: {$eval: 'input'}
+ parameters: {$eval: 'parameters'}
--- a/taskcluster/taskgraph/actions/registry.py
+++ b/taskcluster/taskgraph/actions/registry.py
@@ -3,31 +3,27 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
-import inspect
import re
import yaml
from slugid import nice as slugid
from mozbuild.util import memoize
from types import FunctionType
from collections import namedtuple
-from taskgraph import create
+from taskgraph import create, GECKO
from taskgraph.util import taskcluster
-from taskgraph.util.docker import docker_image
from taskgraph.parameters import Parameters
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
-
actions = []
callbacks = {}
Action = namedtuple('Action', [
'name', 'title', 'description', 'order', 'context', 'schema', 'task_template_builder',
])
@@ -168,123 +164,65 @@ def register_callback_action(name, title
mem = {"registered": False} # workaround nonlocal missing in 2.x
def register_callback(cb):
assert isinstance(cb, FunctionType), 'callback must be a function'
assert isinstance(symbol, basestring), 'symbol must be a string'
assert 1 <= len(symbol) <= 25, 'symbol must be between 1 and 25 characters'
assert not mem['registered'], 'register_callback_action must be used as decorator'
assert cb.__name__ not in callbacks, 'callback name {} is not unique'.format(cb.__name__)
- source_path = os.path.relpath(inspect.stack()[1][1], GECKO)
@register_task_action(name, title, description, order, context, schema)
def build_callback_action_task(parameters):
if not available(parameters):
return None
match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters['head_repository'])
if not match:
raise Exception('Unrecognized head_repository')
repo_scope = 'assume:repo:{}/{}:*'.format(
match.group(1), match.group(2))
task_group_id = os.environ.get('TASK_ID', slugid())
- return {
- 'created': {'$fromNow': ''},
- 'deadline': {'$fromNow': '12 hours'},
- 'expires': {'$fromNow': '1 year'},
- 'metadata': {
- 'owner': 'mozilla-taskcluster-maintenance@mozilla.com',
- 'source': '{}/raw-file/{}/{}'.format(
- parameters['head_repository'], parameters['head_rev'], source_path,
- ),
- 'name': 'Action: {}'.format(title),
- 'description': 'Task executing callback for action.\n\n---\n' + description,
- },
- 'workerType': 'gecko-{}-decision'.format(parameters['level']),
- 'provisionerId': 'aws-provisioner-v1',
- 'taskGroupId': task_group_id,
- 'schedulerId': 'gecko-level-{}'.format(parameters['level']),
- 'scopes': [
- repo_scope,
- ],
- 'tags': {
- 'createdForUser': parameters['owner'],
- 'kind': 'action-callback',
- },
- 'routes': [
- 'tc-treeherder.v2.{}.{}.{}'.format(
- parameters['project'], parameters['head_rev'], parameters['pushlog_id']),
- 'tc-treeherder-stage.v2.{}.{}.{}'.format(
- parameters['project'], parameters['head_rev'], parameters['pushlog_id']),
- 'index.gecko.v2.{}.pushlog-id.{}.actions.${{ownTaskId}}'.format(
- parameters['project'], parameters['pushlog_id'])
- ],
- 'payload': {
- 'env': {
- 'GECKO_BASE_REPOSITORY': 'https://hg.mozilla.org/mozilla-unified',
- 'GECKO_HEAD_REPOSITORY': parameters['head_repository'],
- 'GECKO_HEAD_REF': parameters['head_ref'],
- 'GECKO_HEAD_REV': parameters['head_rev'],
- 'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
- 'ACTION_TASK_GROUP_ID': task_group_id,
- 'ACTION_TASK_ID': {'$json': {'$eval': 'taskId'}},
- 'ACTION_TASK': {'$json': {'$eval': 'task'}},
- 'ACTION_INPUT': {'$json': {'$eval': 'input'}},
- 'ACTION_CALLBACK': cb.__name__,
- 'ACTION_PARAMETERS': {'$json': {'$eval': 'parameters'}},
- 'TASKCLUSTER_CACHES': '/builds/worker/checkouts',
- },
- 'artifacts': {
- 'public': {
- 'type': 'directory',
- 'path': '/builds/worker/artifacts',
- 'expires': {'$fromNow': '1 year'},
+ template = os.path.join(GECKO, '.taskcluster.yml')
+
+ with open(template, 'r') as f:
+ taskcluster_yml = yaml.safe_load(f)
+ if taskcluster_yml['version'] != 1:
+ raise Exception('actions.json must be updated to work with .taskcluster.yml')
+ if not isinstance(taskcluster_yml['tasks'], list):
+ raise Exception('.taskcluster.yml "tasks" must be a list for action tasks')
+
+ return {
+ '$let': {
+ 'tasks_for': 'action',
+ 'repository': {
+ 'url': parameters['head_repository'],
+ 'project': parameters['project'],
+ 'level': parameters['level'],
+ },
+ 'push': {
+ 'owner': 'mozilla-taskcluster-maintenance@mozilla.com',
+ 'pushlog_id': parameters['pushlog_id'],
+ 'revision': parameters['head_rev'],
+ },
+ 'action': {
+ 'name': name,
+ 'title': title,
+ 'description': description,
+ 'taskGroupId': task_group_id,
+ 'repo_scope': repo_scope,
+ 'cb_name': cb.__name__,
+ 'symbol': symbol,
},
},
- 'cache': {
- 'level-{}-checkouts-sparse-v1'.format(parameters['level']):
- '/builds/worker/checkouts',
- },
- 'features': {
- 'taskclusterProxy': True,
- 'chainOfTrust': True,
- },
- 'image': docker_image('decision'),
- 'maxRunTime': 1800,
- 'command': [
- '/builds/worker/bin/run-task',
- '--vcs-checkout=/builds/worker/checkouts/gecko',
- '--sparse-profile=build/sparse-profiles/taskgraph',
- '--', 'bash', '-cx',
- """\
-cd /builds/worker/checkouts/gecko &&
-ln -s /builds/worker/artifacts artifacts &&
-./mach --log-no-times taskgraph action-callback""",
- ],
- },
- 'extra': {
- 'treeherder': {
- 'groupName': 'action-callback',
- 'groupSymbol': 'AC',
- 'symbol': symbol,
- },
- 'parent': task_group_id,
- 'action': {
- 'name': name,
- 'context': {
- 'taskGroupId': task_group_id,
- 'taskId': {'$eval': 'taskId'},
- 'input': {'$eval': 'input'},
- 'parameters': {'$eval': 'parameters'},
- },
- },
- },
- }
+ 'in': taskcluster_yml['tasks'][0]
+ }
+
mem['registered'] = True
callbacks[cb.__name__] = cb
return register_callback
def render_actions_json(parameters):
"""
Render JSON object for the ``public/actions.json`` artifact.