Bug 1383880: add only-if-dependencies-run optimization for follow-ons; r=ahal draft
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 25 Aug 2017 21:18:13 +0000
changeset 668275 d4b8ad765a20ddc522fae58df9a73e23175565e2
parent 668274 5a08cc5a1f1de88dce73602d907828c540fbd4b2
child 668276 0c6079a372cde05afa80fddb97cb817969c31a33
push id80998
push userdmitchell@mozilla.com
push dateThu, 21 Sep 2017 12:49:52 +0000
reviewersahal
bugs1383880
milestone57.0a1
Bug 1383880: add only-if-dependencies-run optimization for follow-ons; r=ahal MozReview-Commit-ID: JuhwyQIx3Mh
taskcluster/ci/upload-generated-sources/kind.yml
taskcluster/ci/upload-symbols/kind.yml
taskcluster/taskgraph/optimize.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/upload-generated-sources/kind.yml
+++ b/taskcluster/ci/upload-generated-sources/kind.yml
@@ -26,10 +26,12 @@ job-template:
   worker:
      docker-image: {in-tree: "lint"}
      max-run-time: 600
   run:
     using: run-task
     command: >
             cd /builds/worker/checkouts/gecko &&
             ./mach python build/upload_generated_sources.py ${ARTIFACT_URL}
+  optimization:
+    only-if-dependencies-run: null
   scopes:
       - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload
--- a/taskcluster/ci/upload-symbols/kind.yml
+++ b/taskcluster/ci/upload-symbols/kind.yml
@@ -38,10 +38,12 @@ job-template:
        os: linux
        max-run-time: 600
        command: ["/bin/bash", "bin/upload.sh"]
        docker-image: taskclusterprivate/upload_symbols:0.0.4
        env:
            GECKO_HEAD_REPOSITORY: # see transforms
            GECKO_HEAD_REV: # see transforms
            ARTIFACT_TASKID: {"task-reference": "<build>"}
+   optimization:
+       only-if-dependencies-run: null
    scopes:
        - docker-worker:image:taskclusterprivate/upload_symbols:0.0.4
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -72,16 +72,17 @@ def optimize_task_graph(target_task_grap
 def _make_default_strategies():
     return {
         'never': OptimizationStrategy(),  # "never" is the default behavior
         'index-search': IndexSearch(),
         'seta': SETA(),
         'skip-unless-changed': SkipUnlessChanged(),
         'skip-unless-schedules': SkipUnlessSchedules(),
         'skip-unless-schedules-or-seta': Either(SkipUnlessSchedules(), SETA()),
+        'only-if-dependencies-run': OnlyIfDependenciesRun(),
     }
 
 
 def _get_optimizations(target_task_graph, strategies):
     def optimizations(label):
         task = target_task_graph.tasks[label]
         if task.optimization:
             opt_by, arg = task.optimization.items()[0]
@@ -276,16 +277,28 @@ class Either(OptimizationStrategy):
             lambda sub, arg: sub.should_remove_task(task, params, arg))
 
     def should_replace_task(self, task, params, arg):
         return self._for_substrategies(
             arg,
             lambda sub, arg: sub.should_replace_task(task, params, arg))
 
 
+class OnlyIfDependenciesRun(OptimizationStrategy):
+    """Run this taks only if its dependencies run."""
+
+    # This takes advantage of the behavior of the second phase of optimization:
+    # a task can only be replaced if it has no un-optimized dependencies. So if
+    # should_replace_task is called, then a task has no un-optimized
+    # dependencies and can be removed (indicated by returning True)
+
+    def should_replace_task(self, task, params, arg):
+        return True
+
+
 class IndexSearch(OptimizationStrategy):
     def should_remove_task(self, task, params, index_paths):
         "If this task has no dependencies, don't run it.."
         return True
 
     def should_replace_task(self, task, params, index_paths):
         "Look for a task with one of the given index paths"
         for index_path in index_paths:
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -179,16 +179,19 @@ task_description_schema = Schema({
         # consult SETA and skip this task if it is low-value
         {'seta': None},
         # skip this task if none of the given file patterns match
         {'skip-unless-changed': [basestring]},
         # skip this task if unless the change files' SCHEDULES contains any of these components
         {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)},
         # skip if SETA or skip-unless-schedules says to
         {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)},
+        # only run this task if its dependencies will run (useful for follow-on tasks that
+        # are unnecessary if the parent tasks are not run)
+        {'only-if-dependencies-run': None}
     ),
 
     # the provisioner-id/worker-type for the task.  The following parameters will
     # be substituted in this string:
     #  {level} -- the scm level of this push
     'worker-type': basestring,
 
     # Whether the job should use sccache compiler caching.