Bug 1277665 - add --dry-run functionality to mach taskgraph; r?dustin
MozReview-Commit-ID: JByEMoV8OPP
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -133,16 +133,19 @@ class MachCommands(MachCommandBase):
required=True,
default=0)
@CommandArgument('--owner',
required=True,
help='email address of who owns this graph')
@CommandArgument('--level',
required=True,
help='SCM level of this repository')
+ @CommandArgument('--dry-run',
+ action='store_true', default=False,
+ help='Stub out variable data (taskIds, dates, etc) from task definitions.')
def taskgraph_decision(self, **options):
"""Run the decision task: generate a task graph and submit to
TaskCluster. This is only meant to be called within decision tasks,
and requires a great many arguments. Commands like `mach taskgraph
optimized` are better suited to use on the command line, and can take
the parameters file generated by a decision task. """
import taskgraph.decision
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -56,16 +56,20 @@ def taskgraph_decision(options):
# create a TaskGraphGenerator instance
target_tasks_method = parameters.get('target_tasks_method', 'all_tasks')
target_tasks_method = get_method(target_tasks_method)
tgg = TaskGraphGenerator(
root_dir=options['root'],
parameters=parameters,
target_tasks_method=target_tasks_method)
+ if options['dry_run']:
+ print(json.dumps(tgg.full_task_graph.to_json(), indent=4, sort_keys=True))
+ return
+
# write out the parameters used to generate this graph
write_artifact('parameters.yml', dict(**parameters))
# write out the full graph for reference
write_artifact('full-task-graph.json', tgg.full_task_graph.to_json())
# write out the target task set to allow reproducing this as input
write_artifact('target-tasks.json', tgg.target_task_set.tasks.keys())
@@ -91,16 +95,17 @@ def get_decision_parameters(options):
'head_rev',
'head_ref',
'message',
'project',
'pushlog_id',
'owner',
'level',
'target_tasks_method',
+ 'dry_run',
] if n in options}
project = parameters['project']
try:
parameters.update(PER_PROJECT_PARAMETERS[project])
except KeyError:
logger.warning("using default project parameters; add {} to "
"PER_PROJECT_PARAMETERS in {} to customize behavior "
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -13,30 +13,37 @@ import time
from . import base
from ..types import Task
from taskgraph.util.docker import (
docker_image,
generate_context_hash
)
from taskgraph.util.templates import Templates
-from taskgraph.util.time import (
- json_time_from_now,
- current_json_time,
-)
logger = logging.getLogger(__name__)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
class DockerImageKind(base.Kind):
def load_tasks(self, params):
+ if params['dry_run']:
+ from .dry_run import (
+ current_json_time,
+ json_time_from_now,
+ )
+ else:
+ from taskgraph.util.time import (
+ current_json_time,
+ json_time_from_now,
+ )
+
# TODO: make this match the pushdate (get it from a parameter rather than vcs)
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
parameters = {
'pushlog_id': params.get('pushlog_id', 0),
'pushdate': pushdate,
'pushtime': pushdate[8:],
'year': pushdate[0:4],
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/kind/dry_run.py
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+from taskgraph.util.time import value_of
+
+# The current_task counter ensures a unique taskId for each task, but remains
+# consistent between dry runs.
+current_task = 0
+
+
+def json_time_from_now(input_str, now=None):
+ '''
+ :param str input_str: Input string (see value of)
+ :param datetime now: Optionally set the definition of `now`
+ :returns: JSON string representation of time in future.
+ '''
+
+ if now is None:
+ now = datetime.date.fromordinal(1)
+
+ time = now + value_of(input_str)
+
+ # Sorta a big hack but the json schema validator for date does not like the
+ # ISO dates until 'Z' (for timezone) is added...
+ return time.isoformat() + 'Z'
+
+
+def current_json_time():
+ '''
+ :returns: JSON string representation of the current time.
+ '''
+
+ return datetime.date.fromordinal(1).isoformat() + 'Z'
+
+
+def mklabel():
+ global current_task
+ current_task += 1
+ return 'TaskLabel==%04d' % current_task
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -12,20 +12,16 @@ import re
import time
from collections import namedtuple
from . import base
from ..types import Task
from mozpack.path import match as mozpackmatch
from slugid import nice as slugid
from taskgraph.util.legacy_commit_parser import parse_commit
-from taskgraph.util.time import (
- json_time_from_now,
- current_json_time,
-)
from taskgraph.util.templates import Templates
from taskgraph.util.docker import docker_image
ROOT = os.path.dirname(os.path.realpath(__file__))
GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..'))
# TASKID_PLACEHOLDER is the "internal" form of a taskid; it is substituted with
# actual taskIds at the very last minute, in get_task_definition
@@ -43,20 +39,16 @@ TREEHERDER_ROUTES = {
}
# time after which a try build's results will expire
TRY_EXPIRATION = "14 days"
logger = logging.getLogger(__name__)
-def mklabel():
- return TASKID_PLACEHOLDER.format(slugid())
-
-
def merge_dicts(*dicts):
merged_dict = {}
for dictionary in dicts:
merged_dict.update(dictionary)
return merged_dict
def gaia_info():
@@ -298,16 +290,30 @@ class LegacyKind(base.Kind):
`testing/taskcluster/tasks`. The tasks already have dependency links.
The existing task-graph generation generates slugids for tasks during task
generation, so this kind labels tasks using those slugids, with a prefix of
"TaskLabel==". These labels are unfortunately not stable from run to run.
"""
def load_tasks(self, params):
+ if params['dry_run']:
+ from .dry_run import (
+ mklabel,
+ current_json_time,
+ json_time_from_now,
+ )
+ else:
+ def mklabel():
+ return TASKID_PLACEHOLDER.format(slugid())
+ from taskgraph.util.time import (
+ current_json_time,
+ json_time_from_now,
+ )
+
root = os.path.abspath(os.path.join(self.path, self.config['legacy_path']))
project = params['project']
# NOTE: message is ignored here; we always use DEFAULT_TRY, then filter the
# resulting task graph later
message = DEFAULT_TRY
templates = Templates(root)