bug 1423081 - reverse chunk deps. r=callek
This allows us to funnel large numbers of tasks down to avoid hitting
MAX_DEPENDENCIES. I avoided using a morph here because we might break
certain cot assumptions.
MozReview-Commit-ID: BIILM9O6CI4
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/post-balrog-dummy/kind.yml
@@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+ - taskgraph.transforms.reverse_chunk_deps:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - balrog
+
+jobs:
+ firefox:
+ name: post-balrog-dummy
+ description: Dummy task to deal with max_dependencies
+ run-on-projects: []
+ shipping-phase: promote
+ shipping-product: firefox
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ os: linux
+ docker-image: "ubuntu:16.10"
+ max-run-time: 600
+ command:
+ - /bin/bash
+ - -c
+ - echo "Dummy task"
+ treeherder:
+ symbol: Rel(BD)
+ platform: linux/opt
+ kind: test
+ tier: 1
+
+ devedition:
+ name: post-balrog-dummy
+ description: Dummy task to deal with max_dependencies
+ run-on-projects: []
+ shipping-phase: promote
+ shipping-product: devedition
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ os: linux
+ docker-image: "ubuntu:16.10"
+ max-run-time: 600
+ command:
+ - /bin/bash
+ - -c
+ - echo "Dummy task"
+ treeherder:
+ symbol: Rel(BD)
+ platform: linux-devedition/opt
+ kind: test
+ tier: 1
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/post-beetmover-dummy/kind.yml
@@ -0,0 +1,92 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+ - taskgraph.transforms.reverse_chunk_deps:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - beetmover-checksums
+ - beetmover-repackage
+
+jobs:
+ firefox-ship:
+ name: post-beetmover-dummy
+ description: Dummy task to deal with max_dependencies
+ run-on-projects: []
+ shipping-phase: ship
+ shipping-product: firefox
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ os: linux
+ docker-image: "ubuntu:16.10"
+ max-run-time: 600
+ command:
+ - /bin/bash
+ - -c
+ - echo "Dummy task"
+
+ firefox-promote:
+ name: post-beetmover-dummy
+ description: Dummy task to deal with max_dependencies
+ run-on-projects: []
+ shipping-phase: promote
+ shipping-product: firefox
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ os: linux
+ docker-image: "ubuntu:16.10"
+ max-run-time: 600
+ command:
+ - /bin/bash
+ - -c
+ - echo "Dummy task"
+ treeherder:
+ symbol: Rel(BMD)
+ platform: linux/opt
+ kind: test
+ tier: 1
+
+ devedition-ship:
+ name: post-beetmover-dummy
+ description: Dummy task to deal with max_dependencies
+ run-on-projects: []
+ shipping-phase: ship
+ shipping-product: devedition
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ os: linux
+ docker-image: "ubuntu:16.10"
+ max-run-time: 600
+ command:
+ - /bin/bash
+ - -c
+ - echo "Dummy task"
+
+ devedition-promote:
+ name: post-beetmover-dummy
+ description: Dummy task to deal with max_dependencies
+ run-on-projects: []
+ shipping-phase: promote
+ shipping-product: devedition
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ os: linux
+ docker-image: "ubuntu:16.10"
+ max-run-time: 600
+ command:
+ - /bin/bash
+ - -c
+ - echo "Dummy task"
+ treeherder:
+ symbol: Rel(BMD)
+ platform: linux-devedition/opt
+ kind: test
+ tier: 1
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/reverse_chunk_deps.py
@@ -0,0 +1,49 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Adjust dependencies to not exceed MAX_DEPS
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+from copy import deepcopy
+
+from taskgraph.transforms.base import TransformSequence
+import taskgraph.transforms.release_deps as release_deps
+from taskgraph.util.treeherder import split_symbol, join_symbol
+
+transforms = TransformSequence()
+
+# Max dependency limit per task.
+# https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask
+MAX_DEPS = 100
+
+
+def yield_job(orig_job, deps, count):
+ job = deepcopy(orig_job)
+ job['dependencies'] = deps
+ job['name'] = "{}-{}".format(orig_job['name'], count)
+ if 'treeherder' in job:
+ groupSymbol, symbol = split_symbol(job['treeherder']['symbol'])
+ symbol += '-'
+ symbol += str(count)
+ job['treeherder']['symbol'] = join_symbol(groupSymbol, symbol)
+
+ return job
+
+
+@transforms.add
+def add_dependencies(config, jobs):
+ for job in release_deps.add_dependencies(config, jobs):
+ count = 1
+ deps = {}
+
+ for dep_label in job['dependencies'].keys():
+ deps[dep_label] = dep_label
+ if len(deps) == MAX_DEPS:
+ yield yield_job(job, deps, count)
+ deps = {}
+ count += 1
+ if deps:
+ yield yield_job(job, deps, count)
+ count += 1