bug 1425985 - Add check for max dependencies. r=aki draft
authorChris AtLee <catlee@mozilla.com>
Sat, 16 Dec 2017 15:09:25 -0600
changeset 712873 89090e7f3c33f8df50967225fae7af1e7d910ba9
parent 712872 70f5b742f2164a3f4ca153358f8b1b5964ea0bda
child 712874 39700863e3223443b6fd3654ef12dc1f78673267
push id93456
push userasasaki@mozilla.com
push dateMon, 18 Dec 2017 22:50:23 +0000
reviewersaki
bugs1425985
milestone59.0a1
bug 1425985 - Add check for max dependencies. r=aki MozReview-Commit-ID: G3uriZShTCw
taskcluster/taskgraph/__init__.py
taskcluster/taskgraph/transforms/reverse_chunk_deps.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/taskgraph/__init__.py
+++ b/taskcluster/taskgraph/__init__.py
@@ -3,12 +3,16 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
 
+# Maximum number of dependencies a single task can have
+# https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask
+MAX_DEPENDENCIES = 100
+
 # Enable fast task generation for local debugging
 # This is normally switched on via the --fast/-F flag to `mach taskgraph`
 # Currently this skips toolchain task optimizations
 fast = False
--- a/taskcluster/taskgraph/transforms/reverse_chunk_deps.py
+++ b/taskcluster/taskgraph/transforms/reverse_chunk_deps.py
@@ -1,28 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
-Adjust dependencies to not exceed MAX_DEPS
+Adjust dependencies to not exceed MAX_DEPENDENCIES
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 from copy import deepcopy
 
 from taskgraph.transforms.base import TransformSequence
 import taskgraph.transforms.release_deps as release_deps
 from taskgraph.util.treeherder import split_symbol, join_symbol
+from taskgraph import MAX_DEPENDENCIES
 
 transforms = TransformSequence()
 
-# Max dependency limit per task.
-# https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask
-MAX_DEPS = 100
-
 
 def yield_job(orig_job, deps, count):
     job = deepcopy(orig_job)
     job['dependencies'] = deps
     job['name'] = "{}-{}".format(orig_job['name'], count)
     if 'treeherder' in job:
         groupSymbol, symbol = split_symbol(job['treeherder']['symbol'])
         symbol += '-'
@@ -35,15 +32,15 @@ def yield_job(orig_job, deps, count):
 @transforms.add
 def add_dependencies(config, jobs):
     for job in release_deps.add_dependencies(config, jobs):
         count = 1
         deps = {}
 
         for dep_label in job['dependencies'].keys():
             deps[dep_label] = dep_label
-            if len(deps) == MAX_DEPS:
+            if len(deps) == MAX_DEPENDENCIES:
                 yield yield_job(job, deps, count)
                 deps = {}
                 count += 1
         if deps:
             yield yield_job(job, deps, count)
             count += 1
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -21,17 +21,17 @@ from mozbuild.util import memoize
 from mozbuild import schedules
 from taskgraph.util.attributes import TRUNK_PROJECTS
 from taskgraph.util.hash import hash_path
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
 from taskgraph.util.scriptworker import get_release_config
 from voluptuous import Any, Required, Optional, Extra
-from taskgraph import GECKO
+from taskgraph import GECKO, MAX_DEPENDENCIES
 from ..util import docker as dockerutil
 
 from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
 
 
 RUN_TASK = os.path.join(GECKO, 'taskcluster', 'docker', 'recipes', 'run-task')
 
 
@@ -1527,16 +1527,28 @@ def check_task_identifiers(config, tasks
         for attr in ('workerType', 'provisionerId'):
             if not e.match(task['task'][attr]):
                 raise Exception(
                     'task {}.{} is not a valid identifier: {}'.format(
                         task['label'], attr, task['task'][attr]))
         yield task
 
 
+@transforms.add
+def check_task_dependencies(config, tasks):
+    """Ensures that tasks don't have more than 100 dependencies."""
+    for task in tasks:
+        if len(task['dependencies']) > MAX_DEPENDENCIES:
+            raise Exception(
+                    'task {}/{} has too many dependencies ({} > {})'.format(
+                        config.kind, task['label'], len(task['dependencies']),
+                        MAX_DEPENDENCIES))
+        yield task
+
+
 def check_caches_are_volumes(task):
     """Ensures that all cache paths are defined as volumes.
 
     Caches and volumes are the only filesystem locations whose content
     isn't defined by the Docker image itself. Some caches are optional
     depending on the job environment. We want paths that are potentially
     caches to have as similar behavior regardless of whether a cache is
     used. To help enforce this, we require that all paths used as caches