bug 1423081 - reverse chunk deps. r=callek draft
authorAki Sasaki <asasaki@mozilla.com>
Tue, 05 Dec 2017 19:32:44 -0800
changeset 711435 a4a155fb3a8156be6212f51adb7d25213fd23c1d
parent 711434 cc53c41f3b17791fa368ffd05c83052701afd5ab
child 711436 fbc98ebda304701de7d3094e95f15c2eaaab03b4
push id93069
push userasasaki@mozilla.com
push dateWed, 13 Dec 2017 22:57:07 +0000
reviewerscallek
bugs1423081
milestone59.0a1
bug 1423081 - reverse chunk deps. r=callek This allows us to funnel large numbers of tasks down to avoid hitting MAX_DEPENDENCIES. I avoided using a morph here because we might break certain cot assumptions. MozReview-Commit-ID: BIILM9O6CI4
taskcluster/ci/post-balrog-dummy/kind.yml
taskcluster/ci/post-beetmover-dummy/kind.yml
taskcluster/taskgraph/transforms/reverse_chunk_deps.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/post-balrog-dummy/kind.yml
@@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.reverse_chunk_deps:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - balrog
+
+jobs:
+   firefox:
+      name: post-balrog-dummy
+      description: Dummy task to deal with max_dependencies
+      run-on-projects: []
+      shipping-phase: promote
+      shipping-product: firefox
+      worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: "ubuntu:16.10"
+         max-run-time: 600
+         command:
+            - /bin/bash
+            - -c
+            - echo "Dummy task"
+      treeherder:
+         symbol: Rel(BD)
+         platform: linux/opt
+         kind: test
+         tier: 1
+
+   devedition:
+      name: post-balrog-dummy
+      description: Dummy task to deal with max_dependencies
+      run-on-projects: []
+      shipping-phase: promote
+      shipping-product: devedition
+      worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: "ubuntu:16.10"
+         max-run-time: 600
+         command:
+            - /bin/bash
+            - -c
+            - echo "Dummy task"
+      treeherder:
+         symbol: Rel(BD)
+         platform: linux-devedition/opt
+         kind: test
+         tier: 1
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/post-beetmover-dummy/kind.yml
@@ -0,0 +1,92 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.reverse_chunk_deps:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - beetmover-checksums
+   - beetmover-repackage
+
+jobs:
+   firefox-ship:
+      name: post-beetmover-dummy
+      description: Dummy task to deal with max_dependencies
+      run-on-projects: []
+      shipping-phase: ship
+      shipping-product: firefox
+      worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: "ubuntu:16.10"
+         max-run-time: 600
+         command:
+            - /bin/bash
+            - -c
+            - echo "Dummy task"
+
+   firefox-promote:
+      name: post-beetmover-dummy
+      description: Dummy task to deal with max_dependencies
+      run-on-projects: []
+      shipping-phase: promote
+      shipping-product: firefox
+      worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: "ubuntu:16.10"
+         max-run-time: 600
+         command:
+            - /bin/bash
+            - -c
+            - echo "Dummy task"
+      treeherder:
+         symbol: Rel(BMD)
+         platform: linux/opt
+         kind: test
+         tier: 1
+
+   devedition-ship:
+      name: post-beetmover-dummy
+      description: Dummy task to deal with max_dependencies
+      run-on-projects: []
+      shipping-phase: ship
+      shipping-product: devedition
+      worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: "ubuntu:16.10"
+         max-run-time: 600
+         command:
+            - /bin/bash
+            - -c
+            - echo "Dummy task"
+
+   devedition-promote:
+      name: post-beetmover-dummy
+      description: Dummy task to deal with max_dependencies
+      run-on-projects: []
+      shipping-phase: promote
+      shipping-product: devedition
+      worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: "ubuntu:16.10"
+         max-run-time: 600
+         command:
+            - /bin/bash
+            - -c
+            - echo "Dummy task"
+      treeherder:
+         symbol: Rel(BMD)
+         platform: linux-devedition/opt
+         kind: test
+         tier: 1
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/reverse_chunk_deps.py
@@ -0,0 +1,49 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Adjust dependencies to not exceed MAX_DEPS
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+from copy import deepcopy
+
+from taskgraph.transforms.base import TransformSequence
+import taskgraph.transforms.release_deps as release_deps
+from taskgraph.util.treeherder import split_symbol, join_symbol
+
+transforms = TransformSequence()
+
+# Max dependency limit per task.
+# https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask
+MAX_DEPS = 100
+
+
+def yield_job(orig_job, deps, count):
+    job = deepcopy(orig_job)
+    job['dependencies'] = deps
+    job['name'] = "{}-{}".format(orig_job['name'], count)
+    if 'treeherder' in job:
+        groupSymbol, symbol = split_symbol(job['treeherder']['symbol'])
+        symbol += '-'
+        symbol += str(count)
+        job['treeherder']['symbol'] = join_symbol(groupSymbol, symbol)
+
+    return job
+
+
+@transforms.add
+def add_dependencies(config, jobs):
+    for job in release_deps.add_dependencies(config, jobs):
+        count = 1
+        deps = {}
+
+        for dep_label in job['dependencies'].keys():
+            deps[dep_label] = dep_label
+            if len(deps) == MAX_DEPS:
+                yield yield_job(job, deps, count)
+                deps = {}
+                count += 1
+        if deps:
+            yield yield_job(job, deps, count)
+            count += 1