bug 1425985 - Add check for max dependencies. r=aki
MozReview-Commit-ID: G3uriZShTCw
--- a/taskcluster/taskgraph/__init__.py
+++ b/taskcluster/taskgraph/__init__.py
@@ -3,12 +3,16 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import os
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+# Maximum number of dependencies a single task can have
+# https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask
+MAX_DEPENDENCIES = 100
+
# Enable fast task generation for local debugging
# This is normally switched on via the --fast/-F flag to `mach taskgraph`
# Currently this skips toolchain task optimizations
fast = False
--- a/taskcluster/taskgraph/transforms/reverse_chunk_deps.py
+++ b/taskcluster/taskgraph/transforms/reverse_chunk_deps.py
@@ -1,28 +1,25 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
-Adjust dependencies to not exceed MAX_DEPS
+Adjust dependencies to not exceed MAX_DEPENDENCIES
"""
from __future__ import absolute_import, print_function, unicode_literals
from copy import deepcopy
from taskgraph.transforms.base import TransformSequence
import taskgraph.transforms.release_deps as release_deps
from taskgraph.util.treeherder import split_symbol, join_symbol
+from taskgraph import MAX_DEPENDENCIES
transforms = TransformSequence()
-# Max dependency limit per task.
-# https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask
-MAX_DEPS = 100
-
def yield_job(orig_job, deps, count):
job = deepcopy(orig_job)
job['dependencies'] = deps
job['name'] = "{}-{}".format(orig_job['name'], count)
if 'treeherder' in job:
groupSymbol, symbol = split_symbol(job['treeherder']['symbol'])
symbol += '-'
@@ -35,15 +32,15 @@ def yield_job(orig_job, deps, count):
@transforms.add
def add_dependencies(config, jobs):
for job in release_deps.add_dependencies(config, jobs):
count = 1
deps = {}
for dep_label in job['dependencies'].keys():
deps[dep_label] = dep_label
- if len(deps) == MAX_DEPS:
+ if len(deps) == MAX_DEPENDENCIES:
yield yield_job(job, deps, count)
deps = {}
count += 1
if deps:
yield yield_job(job, deps, count)
count += 1
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -21,17 +21,17 @@ from mozbuild.util import memoize
from mozbuild import schedules
from taskgraph.util.attributes import TRUNK_PROJECTS
from taskgraph.util.hash import hash_path
from taskgraph.util.treeherder import split_symbol
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
from taskgraph.util.scriptworker import get_release_config
from voluptuous import Any, Required, Optional, Extra
-from taskgraph import GECKO
+from taskgraph import GECKO, MAX_DEPENDENCIES
from ..util import docker as dockerutil
from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
RUN_TASK = os.path.join(GECKO, 'taskcluster', 'docker', 'recipes', 'run-task')
@@ -1527,16 +1527,28 @@ def check_task_identifiers(config, tasks
for attr in ('workerType', 'provisionerId'):
if not e.match(task['task'][attr]):
raise Exception(
'task {}.{} is not a valid identifier: {}'.format(
task['label'], attr, task['task'][attr]))
yield task
+@transforms.add
+def check_task_dependencies(config, tasks):
+ """Ensures that tasks don't have more than 100 dependencies."""
+ for task in tasks:
+ if len(task['dependencies']) > MAX_DEPENDENCIES:
+ raise Exception(
+ 'task {}/{} has too many dependencies ({} > {})'.format(
+ config.kind, task['label'], len(task['dependencies']),
+ MAX_DEPENDENCIES))
+ yield task
+
+
def check_caches_are_volumes(task):
"""Ensures that all cache paths are defined as volumes.
Caches and volumes are the only filesystem locations whose content
isn't defined by the Docker image itself. Some caches are optional
depending on the job environment. We want paths that are potentially
caches to have as similar behavior regardless of whether a cache is
used. To help enforce this, we require that all paths used as caches