Bug 1302800 - Verify taskgraph implementations against documentation; r?dustin
MozReview-Commit-ID: J8djr4ifvzm
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -6,16 +6,20 @@ This section lists and documents the ava
build
------
Builds are tasks that produce an installer or other output that can be run by
users or automated tests. This is more restrictive than most definitions of
"build" in a Mozilla context: it does not include tasks that run build-like
actions for static analysis or to produce instrumented artifacts.
+build-signing
+--------------
+
+
artifact-build
--------------
This kind performs an artifact build: one based on precompiled binaries
discovered via the TaskCluster index. This task verifies that such builds
continue to work correctly.
hazard
@@ -137,8 +141,11 @@ The tasks to generate each docker image
Docker images are built from subdirectories of ``testing/docker``, using
``docker build``. There is currently no capability for one Docker image to
depend on another in-tree docker image, without uploading the latter to a
Docker repository
The task definition used to create the image-building tasks is given in
``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
Template <yaml-templates>`.
+
+android-stuff
+--------------
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -144,16 +144,26 @@ Any other task-description information i
augmented by the run-using implementation.
The run-using implementations are all located in
``taskcluster/taskgraph/transforms/job``, along with the schemas for their
implementations. Those well-commented source files are the canonical
documentation for what constitutes a job description, and should be considered
part of the documentation.
+following ``run-using`` are available
+
+ ``hazard``
+ ``mach``
+ ``mozharness``
+ ``run-task``
+ ``spidermonkey`` or ``spidermonkey-package`` or ``spidermonkey-mozjs-crate``
+ ``toolchain-script``
+
+
Task Descriptions
-----------------
Every kind needs to create tasks, and all of those tasks have some things in
common. They all run on one of a small set of worker implementations, each
with their own idiosyncracies. And they all report to TreeHerder in a similar
way.
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -1,10 +1,9 @@
# -*- coding: utf-8 -*-
-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import os
import json
@@ -12,16 +11,17 @@ import logging
import time
import yaml
from .generator import TaskGraphGenerator
from .create import create_tasks
from .parameters import Parameters
from .taskgraph import TaskGraph
+from .util.verifydoc import verify_docs
from taskgraph.util.templates import Templates
from taskgraph.util.time import (
json_time_from_now,
current_json_time,
)
logger = logging.getLogger(__name__)
@@ -71,17 +71,17 @@ def taskgraph_decision(options):
* processing decision task command-line options into parameters
* running task-graph generation exactly the same way the other `mach
taskgraph` commands do
* generating a set of artifacts to memorialize the graph
* calling TaskCluster APIs to create the graph
"""
parameters = get_decision_parameters(options)
-
+ verify_parameters(parameters)
# create a TaskGraphGenerator instance
tgg = TaskGraphGenerator(
root_dir=options['root'],
parameters=parameters)
# write out the parameters used to generate this graph
write_artifact('parameters.yml', dict(**parameters))
@@ -182,8 +182,17 @@ def get_action_yml(parameters):
action_parameters = parameters.copy()
action_parameters.update({
"decision_task_id": "{{decision_task_id}}",
"task_labels": "{{task_labels}}",
"from_now": json_time_from_now,
"now": current_json_time()
})
return templates.load('action.yml', action_parameters)
+
+
+def verify_parameters(parameters):
+ parameters_dict = dict(**parameters)
+ verify_docs(
+ filename="parameters.rst",
+ identifiers=parameters_dict.keys(),
+ appearing_as="inline-literal"
+ )
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -7,16 +7,17 @@ import logging
import os
import yaml
from . import filter_tasks
from .graph import Graph
from .taskgraph import TaskGraph
from .optimize import optimize_task_graph
from .util.python_path import find_object
+from .util.verifydoc import verify_docs
logger = logging.getLogger(__name__)
class Kind(object):
def __init__(self, name, path, config):
self.name = name
@@ -152,16 +153,18 @@ class TaskGraphGenerator(object):
yield Kind(kind_name, path, config)
def _run(self):
logger.info("Loading kinds")
# put the kinds into a graph and sort topologically so that kinds are loaded
# in post-order
kinds = {kind.name: kind for kind in self._load_kinds()}
+ self.verify_kinds(kinds)
+
edges = set()
for kind in kinds.itervalues():
for dep in kind.config.get('kind-dependencies', []):
edges.add((kind.name, dep, 'kind-dependency'))
kind_graph = Graph(set(kinds), edges)
logger.info("Generating full task set")
all_tasks = {}
@@ -170,16 +173,18 @@ class TaskGraphGenerator(object):
kind = kinds[kind_name]
new_tasks = kind.load_tasks(self.parameters, list(all_tasks.values()))
for task in new_tasks:
if task.label in all_tasks:
raise Exception("duplicate tasks with label " + task.label)
all_tasks[task.label] = task
logger.info("Generated {} tasks for kind {}".format(len(new_tasks), kind_name))
full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
+ self.verify_attributes(all_tasks)
+ self.verify_run_using()
yield 'full_task_set', full_task_set
logger.info("Generating full task graph")
edges = set()
for t in full_task_set:
for dep, depname in t.get_dependencies(full_task_set):
edges.add((t.label, dep, depname))
@@ -209,24 +214,50 @@ class TaskGraphGenerator(object):
target_graph = full_task_graph.graph.transitive_closure(target_tasks)
target_task_graph = TaskGraph(
{l: all_tasks[l] for l in target_graph.nodes},
target_graph)
yield 'target_task_graph', target_task_graph
logger.info("Generating optimized task graph")
do_not_optimize = set()
+
if not self.parameters.get('optimize_target_tasks', True):
do_not_optimize = target_task_set.graph.nodes
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
self.parameters,
do_not_optimize)
yield 'label_to_taskid', label_to_taskid
yield 'optimized_task_graph', optimized_task_graph
def _run_until(self, name):
while name not in self._run_results:
try:
k, v = self._run.next()
except StopIteration:
raise AttributeError("No such run result {}".format(name))
self._run_results[k] = v
return self._run_results[name]
+
+ def verify_kinds(self, kinds):
+ verify_docs(
+ filename="kinds.rst",
+ identifiers=kinds.keys(),
+ appearing_as="heading"
+ )
+
+ def verify_attributes(self, all_tasks):
+ attribute_set = set()
+ for label, task in all_tasks.iteritems():
+ attribute_set.update(task.attributes.keys())
+ verify_docs(
+ filename="attributes.rst",
+ identifiers=list(attribute_set),
+ appearing_as="heading"
+ )
+
+ def verify_run_using(self):
+ from .transforms.job import registry
+ verify_docs(
+ filename="transforms.rst",
+ identifiers=registry.keys(),
+ appearing_as="inline-literal"
+ )
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/verifydoc.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+import os
+
+base_path = os.path.join(os.getcwd(), "taskcluster/docs/")
+
+
+def verify_docs(filename, identifiers, appearing_as):
+ with open(os.path.join(base_path, filename)) as fileObject:
+ doctext = "".join(fileObject.readlines())
+ if appearing_as == "inline-literal":
+ expression_list = ["``" + identifier + "``" for identifier in identifiers]
+ elif appearing_as == "heading":
+ expression_list = [identifier + "\n[-+\n*]+|[.+\n*]+" for identifier in identifiers]
+ else:
+ raise Exception("appearing_as = {} not defined".format(appearing_as))
+
+ for expression, identifier in zip(expression_list, identifiers):
+ match_group = re.search(expression, doctext)
+ if not match_group:
+ raise Exception(
+ "{}: {} missing from doc file: {}".format(appearing_as, identifier, filename)
+ )