Bug 1382707: add ./mach taskgraph test-action-callback; r?bstack draft
authorDustin J. Mitchell <dustin@mozilla.com>
Thu, 20 Jul 2017 17:11:28 +0000
changeset 612448 cfab58f60c357fa310c63c7d737584fa5b74b9b6
parent 611409 4b32e7ce740eaf6434180bc9e44731dab0aa67cc
child 612540 dad72cf0cd35259b1235eccddbb481d3cabf4b90
push id69484
push userdmitchell@mozilla.com
push dateThu, 20 Jul 2017 17:33:05 +0000
reviewersbstack
bugs1382707
milestone56.0a1
Bug 1382707: add ./mach taskgraph test-action-callback; r?bstack MozReview-Commit-ID: 64kBJGLarY3
taskcluster/actions/registry.py
taskcluster/actions/util.py
taskcluster/docs/how-tos.rst
taskcluster/mach_commands.py
taskcluster/taskgraph/util/taskcluster.py
--- a/taskcluster/actions/registry.py
+++ b/taskcluster/actions/registry.py
@@ -1,16 +1,17 @@
 import json
 import os
 import inspect
 import re
 from types import FunctionType
 from collections import namedtuple
 from taskgraph.util.docker import docker_image
 from taskgraph.parameters import Parameters
+from actions import util
 
 
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
 
 actions = []
 callbacks = {}
 
 Action = namedtuple('Action', [
@@ -280,30 +281,29 @@ def render_actions_json(parameters):
         'version': 1,
         'variables': {
             'parameters': dict(**parameters),
         },
         'actions': result,
     }
 
 
-def trigger_action_callback():
-    """
-    Trigger action callback using arguments from environment variables.
+def trigger_action_callback(task_group_id, task_id, task, input, callback, parameters,
+                            test=False):
     """
-    global callbacks
-    task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
-    task_id = json.loads(os.environ.get('ACTION_TASK_ID', 'null'))
-    task = json.loads(os.environ.get('ACTION_TASK', 'null'))
-    input = json.loads(os.environ.get('ACTION_INPUT', 'null'))
-    callback = os.environ.get('ACTION_CALLBACK', None)
-    parameters = json.loads(os.environ.get('ACTION_PARAMETERS', 'null'))
+    Trigger action callback with the given inputs. If `test` is true, then run
+    the action callback in testing mode, without actually creating tasks.
+    """
     cb = callbacks.get(callback, None)
     if not cb:
         raise Exception('Unknown callback: {}'.format(callback))
+
+    if test:
+        util.testing = True
+
     cb(Parameters(**parameters), input, task_group_id, task_id, task)
 
 
 # Load all modules from this folder, relying on the side-effects of register_
 # functions to populate the action registry.
 for f in os.listdir(os.path.dirname(__file__)):
         if f.endswith('.py') and f not in ('__init__.py', 'registry.py'):
             __import__('actions.' + f[:-3])
new file mode 100644
--- /dev/null
+++ b/taskcluster/actions/util.py
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import sys
+
+from taskgraph import create
+from taskgraph.util.taskcluster import get_session
+
+# this is set to true for `mach taskgraph action-callback --test`
+testing = False
+
+
+def create_task(task_id, task_def):
+    """Create a new task.  The task definition will have {relative-datestamp':
+    '..'} rendered just like in a decision task.  Action callbacks should use
+    this function to create new tasks, as it has the additional advantage of
+    allowing easy debugging with `mach taskgraph action-callback --test`."""
+    if testing:
+        json.dump([task_id, task_def], sys.stdout,
+                  sort_keys=True, indent=4, separators=(',', ': '))
+        return
+    label = task_def['metadata']['name']
+    session = get_session()
+    create.create_task(session, task_id, label, task_def)
--- a/taskcluster/docs/how-tos.rst
+++ b/taskcluster/docs/how-tos.rst
@@ -52,16 +52,32 @@ 3. Make your modifications under ``taskc
 
 4. Run the same ``mach taskgraph`` command, sending the output to a new file,
    and use ``diff`` to compare the old and new files.  Make sure your changes
    have the desired effect and no undesirable side-effects.
 
 5. When you are satisfied with the changes, push them to try to ensure that the
    modified tasks work as expected.
 
+Hacking Actions
+...............
+
+If you are working on an action task and wish to test it out locally, use the
+``./mach taskgraph test-action-callback`` command:
+
+   .. code-block:: none
+
+        ./mach taskgraph test-action-task \
+            --task-id I4gu9KDmSZWu3KHx6ba6tw --task-group-id sMO4ybV9Qb2tmcI1sDHClQ \
+            -p parameters.yml --input input.yml \
+            hello_world_action
+
+This invocation will run the hello world callback with the given inputs and
+print any created tasks to stdout, rather than actually creating them.
+
 Common Changes
 --------------
 
 Changing Test Characteristics
 .............................
 
 First, find the test description.  This will be in
 ``taskcluster/ci/*/tests.yml``, for the appropriate kind (consult
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -4,16 +4,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import logging
+import os
 import sys
 import traceback
 import re
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
@@ -291,17 +292,93 @@ class MachCommands(MachCommandBase):
             sys.exit(1)
 
     @SubCommand('taskgraph', 'action-callback',
                 description='Run action callback used by action tasks')
     def action_callback(self, **options):
         import actions
         try:
             self.setup_logging()
-            return actions.trigger_action_callback()
+
+            task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
+            task_id = json.loads(os.environ.get('ACTION_TASK_ID', 'null'))
+            task = json.loads(os.environ.get('ACTION_TASK', 'null'))
+            input = json.loads(os.environ.get('ACTION_INPUT', 'null'))
+            callback = os.environ.get('ACTION_CALLBACK', None)
+            parameters = json.loads(os.environ.get('ACTION_PARAMETERS', '{}'))
+
+            return actions.trigger_action_callback(
+                    task_group_id=task_group_id,
+                    tsak_id=task_id,
+                    task=task,
+                    input=input,
+                    callback=callback,
+                    parameters=parameters,
+                    test=False)
+        except Exception:
+            traceback.print_exc()
+            sys.exit(1)
+
+    @SubCommand('taskgraph', 'test-action-callback',
+                description='Run an action callback in a testing mode')
+    @CommandArgument('--parameters', '-p', default='project=mozilla-central',
+                     help='parameters file (.yml or .json; see '
+                          '`taskcluster/docs/parameters.rst`)`')
+    @CommandArgument('--task-id', default=None,
+                     help='TaskId to which the action applies')
+    @CommandArgument('--task-group-id', default=None,
+                     help='TaskGroupId to which the action applies')
+    @CommandArgument('--input', default=None,
+                     help='Action input (.yml or .json)')
+    @CommandArgument('--task', default=None,
+                     help='Task definition (.yml or .json; if omitted, the task will be'
+                          'fetched from the queue)')
+    @CommandArgument('callback', default=None,
+                     help='Action callback name (Python function name)')
+    def test_action_callback(self, **options):
+        import taskgraph.parameters
+        from taskgraph.util.taskcluster import get_task_definition
+        import actions
+        import yaml
+
+        def load_data(filename):
+            with open(filename) as f:
+                if filename.endswith('.yml'):
+                    return yaml.safe_load(f)
+                elif filename.endswith('.json'):
+                    return json.load(f)
+                else:
+                    raise Exception("unknown filename {}".format(filename))
+
+        try:
+            self.setup_logging()
+            task_id = options['task_id']
+            if options['task']:
+                task = load_data(options['task'])
+            elif task_id:
+                task = get_task_definition(task_id)
+            else:
+                task = None
+
+            if options['input']:
+                input = load_data(options['input'])
+            else:
+                input = None
+
+            parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
+            parameters.check()
+
+            return actions.trigger_action_callback(
+                    task_group_id=options['task_group_id'],
+                    task_id=task_id,
+                    task=task,
+                    input=input,
+                    callback=options['callback'],
+                    parameters=parameters,
+                    test=True)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
 
     def setup_logging(self, quiet=False, verbose=True):
         """
         Set up Python logging for all loggers, sending results to stderr (so
         that command output can be redirected easily) and adding the typical
--- a/taskcluster/taskgraph/util/taskcluster.py
+++ b/taskcluster/taskgraph/util/taskcluster.py
@@ -73,8 +73,21 @@ def get_index_url(index_path, use_proxy=
     else:
         INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
     return INDEX_URL.format(index_path)
 
 
 def find_task_id(index_path, use_proxy=False):
     response = _do_request(get_index_url(index_path, use_proxy))
     return response.json()['taskId']
+
+
+def get_task_url(task_id, use_proxy=False):
+    if use_proxy:
+        TASK_URL = 'http://taskcluster/queue/v1/task/{}'
+    else:
+        TASK_URL = 'https://queue.taskcluster.net/v1/task/{}'
+    return TASK_URL.format(task_id)
+
+
+def get_task_definition(task_id, use_proxy=False):
+    response = _do_request(get_task_url(task_id, use_proxy))
+    return response.json()