Bug 1382707: add ./mach taskgraph test-action-callback; r?bstack
MozReview-Commit-ID: 64kBJGLarY3
--- a/taskcluster/actions/registry.py
+++ b/taskcluster/actions/registry.py
@@ -1,16 +1,17 @@
import json
import os
import inspect
import re
from types import FunctionType
from collections import namedtuple
from taskgraph.util.docker import docker_image
from taskgraph.parameters import Parameters
+from actions import util
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
actions = []
callbacks = {}
Action = namedtuple('Action', [
@@ -280,30 +281,29 @@ def render_actions_json(parameters):
'version': 1,
'variables': {
'parameters': dict(**parameters),
},
'actions': result,
}
-def trigger_action_callback():
- """
- Trigger action callback using arguments from environment variables.
+def trigger_action_callback(task_group_id, task_id, task, input, callback, parameters,
+ test=False):
"""
- global callbacks
- task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
- task_id = json.loads(os.environ.get('ACTION_TASK_ID', 'null'))
- task = json.loads(os.environ.get('ACTION_TASK', 'null'))
- input = json.loads(os.environ.get('ACTION_INPUT', 'null'))
- callback = os.environ.get('ACTION_CALLBACK', None)
- parameters = json.loads(os.environ.get('ACTION_PARAMETERS', 'null'))
+ Trigger action callback with the given inputs. If `test` is true, then run
+ the action callback in testing mode, without actually creating tasks.
+ """
cb = callbacks.get(callback, None)
if not cb:
raise Exception('Unknown callback: {}'.format(callback))
+
+ if test:
+ util.testing = True
+
cb(Parameters(**parameters), input, task_group_id, task_id, task)
# Load all modules from this folder, relying on the side-effects of register_
# functions to populate the action registry.
for f in os.listdir(os.path.dirname(__file__)):
if f.endswith('.py') and f not in ('__init__.py', 'registry.py'):
__import__('actions.' + f[:-3])
new file mode 100644
--- /dev/null
+++ b/taskcluster/actions/util.py
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import sys
+
+from taskgraph import create
+from taskgraph.util.taskcluster import get_session
+
+# this is set to true for `mach taskgraph action-callback --test`
+testing = False
+
+
+def create_task(task_id, task_def):
+ """Create a new task. The task definition will have {relative-datestamp':
+ '..'} rendered just like in a decision task. Action callbacks should use
+ this function to create new tasks, as it has the additional advantage of
+ allowing easy debugging with `mach taskgraph action-callback --test`."""
+ if testing:
+ json.dump([task_id, task_def], sys.stdout,
+ sort_keys=True, indent=4, separators=(',', ': '))
+ return
+ label = task_def['metadata']['name']
+ session = get_session()
+ create.create_task(session, task_id, label, task_def)
--- a/taskcluster/docs/how-tos.rst
+++ b/taskcluster/docs/how-tos.rst
@@ -52,16 +52,32 @@ 3. Make your modifications under ``taskc
4. Run the same ``mach taskgraph`` command, sending the output to a new file,
and use ``diff`` to compare the old and new files. Make sure your changes
have the desired effect and no undesirable side-effects.
5. When you are satisfied with the changes, push them to try to ensure that the
modified tasks work as expected.
+Hacking Actions
+...............
+
+If you are working on an action task and wish to test it out locally, use the
+``./mach taskgraph test-action-callback`` command:
+
+ .. code-block:: none
+
+ ./mach taskgraph test-action-task \
+ --task-id I4gu9KDmSZWu3KHx6ba6tw --task-group-id sMO4ybV9Qb2tmcI1sDHClQ \
+ -p parameters.yml --input input.yml \
+ hello_world_action
+
+This invocation will run the hello world callback with the given inputs and
+print any created tasks to stdout, rather than actually creating them.
+
Common Changes
--------------
Changing Test Characteristics
.............................
First, find the test description. This will be in
``taskcluster/ci/*/tests.yml``, for the appropriate kind (consult
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -4,16 +4,17 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import json
import logging
+import os
import sys
import traceback
import re
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
@@ -291,17 +292,93 @@ class MachCommands(MachCommandBase):
sys.exit(1)
@SubCommand('taskgraph', 'action-callback',
description='Run action callback used by action tasks')
def action_callback(self, **options):
import actions
try:
self.setup_logging()
- return actions.trigger_action_callback()
+
+ task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
+ task_id = json.loads(os.environ.get('ACTION_TASK_ID', 'null'))
+ task = json.loads(os.environ.get('ACTION_TASK', 'null'))
+ input = json.loads(os.environ.get('ACTION_INPUT', 'null'))
+ callback = os.environ.get('ACTION_CALLBACK', None)
+ parameters = json.loads(os.environ.get('ACTION_PARAMETERS', '{}'))
+
+ return actions.trigger_action_callback(
+ task_group_id=task_group_id,
+ tsak_id=task_id,
+ task=task,
+ input=input,
+ callback=callback,
+ parameters=parameters,
+ test=False)
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+ @SubCommand('taskgraph', 'test-action-callback',
+ description='Run an action callback in a testing mode')
+ @CommandArgument('--parameters', '-p', default='project=mozilla-central',
+ help='parameters file (.yml or .json; see '
+ '`taskcluster/docs/parameters.rst`)`')
+ @CommandArgument('--task-id', default=None,
+ help='TaskId to which the action applies')
+ @CommandArgument('--task-group-id', default=None,
+ help='TaskGroupId to which the action applies')
+ @CommandArgument('--input', default=None,
+ help='Action input (.yml or .json)')
+ @CommandArgument('--task', default=None,
+ help='Task definition (.yml or .json; if omitted, the task will be'
+ 'fetched from the queue)')
+ @CommandArgument('callback', default=None,
+ help='Action callback name (Python function name)')
+ def test_action_callback(self, **options):
+ import taskgraph.parameters
+ from taskgraph.util.taskcluster import get_task_definition
+ import actions
+ import yaml
+
+ def load_data(filename):
+ with open(filename) as f:
+ if filename.endswith('.yml'):
+ return yaml.safe_load(f)
+ elif filename.endswith('.json'):
+ return json.load(f)
+ else:
+ raise Exception("unknown filename {}".format(filename))
+
+ try:
+ self.setup_logging()
+ task_id = options['task_id']
+ if options['task']:
+ task = load_data(options['task'])
+ elif task_id:
+ task = get_task_definition(task_id)
+ else:
+ task = None
+
+ if options['input']:
+ input = load_data(options['input'])
+ else:
+ input = None
+
+ parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
+ parameters.check()
+
+ return actions.trigger_action_callback(
+ task_group_id=options['task_group_id'],
+ task_id=task_id,
+ task=task,
+ input=input,
+ callback=options['callback'],
+ parameters=parameters,
+ test=True)
except Exception:
traceback.print_exc()
sys.exit(1)
def setup_logging(self, quiet=False, verbose=True):
"""
Set up Python logging for all loggers, sending results to stderr (so
that command output can be redirected easily) and adding the typical
--- a/taskcluster/taskgraph/util/taskcluster.py
+++ b/taskcluster/taskgraph/util/taskcluster.py
@@ -73,8 +73,21 @@ def get_index_url(index_path, use_proxy=
else:
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
return INDEX_URL.format(index_path)
def find_task_id(index_path, use_proxy=False):
response = _do_request(get_index_url(index_path, use_proxy))
return response.json()['taskId']
+
+
+def get_task_url(task_id, use_proxy=False):
+ if use_proxy:
+ TASK_URL = 'http://taskcluster/queue/v1/task/{}'
+ else:
+ TASK_URL = 'https://queue.taskcluster.net/v1/task/{}'
+ return TASK_URL.format(task_id)
+
+
+def get_task_definition(task_id, use_proxy=False):
+ response = _do_request(get_task_url(task_id, use_proxy))
+ return response.json()