Bug 1391789 - Validate certain caches are used with run-task; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Fri, 18 Aug 2017 15:30:32 -0700
changeset 650038 51e741d7a5429bfb0a761695c96d43c6212617a1
parent 650037 7868ce6e0cae9eef0d140a2043be825ff2a62250
child 650039 508f0dd322fd7c65e3a189cfa0666b7277eb0814
push id75226
push userbmo:gps@mozilla.com
push dateMon, 21 Aug 2017 16:19:27 +0000
reviewersdustin
bugs1391789
milestone57.0a1
Bug 1391789 - Validate certain caches are used with run-task; r?dustin run-task just grew features to aid with cache validation. Attempts by run-task to use caches not under its control will fail. So, we add a transform that audits for and ensures that certain caches are only being used with run-task. This will help catch stragglers attempting to use e.g. the legacy VCS checkouts or tooltool caches without run-task. Fortunately, there are no violations for this policy. Yay! MozReview-Commit-ID: LBCmDUdgcuM
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -72,18 +72,18 @@ def docker_worker_add_gecko_vcs_env_vars
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
     })
 
 
 def support_vcs_checkout(config, job, taskdesc):
     """Update a job/task with parameters to enable a VCS checkout.
 
-    The configuration is intended for tasks using "run-task" and its
-    VCS checkout behavior.
+    This can only be used with ``run-task`` tasks, as the cache name is
+    reserved for ``run-task`` tasks.
     """
     level = config.params['level']
 
     # native-engine does not support caches (yet), so we just do a full clone
     # every time :(
     if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
         taskdesc['worker'].setdefault('caches', []).append({
             'type': 'persistent',
@@ -133,16 +133,19 @@ def docker_worker_setup_secrets(config, 
 
 def docker_worker_add_tooltool(config, job, taskdesc, internal=False):
     """Give the task access to tooltool.
 
     Enables the tooltool cache. Adds releng proxy. Configures scopes.
 
     By default, only public tooltool access will be granted. Access to internal
     tooltool can be enabled via ``internal=True``.
+
+    This can only be used with ``run-task`` tasks, as the cache name is
+    reserved for use with ``run-task``.
     """
 
     assert job['worker']['implementation'] in ('docker-worker', 'docker-engine')
 
     level = config.params['level']
 
     taskdesc['worker'].setdefault('caches', []).append({
         'type': 'persistent',
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -7,16 +7,17 @@ task definition (along with attributes, 
 transformations is generic to any kind of task, but abstracts away some of the
 complexities of worker implementations, scopes, and treeherder annotations.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
+import re
 import time
 from copy import deepcopy
 
 from mozbuild.util import memoize
 from taskgraph.util.attributes import TRUNK_PROJECTS
 from taskgraph.util.hash import hash_path
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import TransformSequence
@@ -1111,16 +1112,60 @@ def build_task(config, tasks):
             'label': task['label'],
             'task': task_def,
             'dependencies': task.get('dependencies', {}),
             'attributes': attributes,
             'optimizations': task.get('optimizations', []),
         }
 
 
+@transforms.add
+def check_run_task_caches(config, tasks):
+    """Audit for caches requiring run-task.
+
+    run-task manages caches in certain ways. If a cache managed by run-task
+    is used by a non run-task task, it could cause problems. So we audit for
+    that and make sure certain cache names are exclusive to run-task.
+
+    IF YOU ARE TEMPTED TO MAKE EXCLUSIONS TO THIS POLICY, YOU ARE LIKELY
+    CONTRIBUTING TECHNICAL DEBT AND WILL HAVE TO SOLVE MANY OF THE PROBLEMS
+    THAT RUN-TASK ALREADY SOLVES. THINK LONG AND HARD BEFORE DOING THAT.
+    """
+    re_reserved_caches = re.compile('''^
+        (level-\d+-checkouts|level-\d+-tooltool-cache)
+    ''', re.VERBOSE)
+
+    suffix = _run_task_suffix()
+
+    for task in tasks:
+        payload = task['task'].get('payload', {})
+        command = payload.get('command') or ['']
+        command = command[0] if isinstance(command[0], basestring) else ''
+        run_task = command.endswith('run-task')
+
+        for cache in payload.get('cache', {}):
+            if not re_reserved_caches.match(cache):
+                continue
+
+            if not run_task:
+                raise Exception(
+                    '%s is using a cache (%s) reserved for run-task '
+                    'change the task to use run-task or use a different '
+                    'cache name' % (task['label'], cache))
+
+            if not cache.endswith(suffix):
+                raise Exception(
+                    '%s is using a cache (%s) reserved for run-task '
+                    'but the cache name is not dependent on the contents '
+                    'of run-task; change the cache name to conform to the '
+                    'naming requirements' % (task['label'], cache))
+
+        yield task
+
+
 # Check that the v2 route templates match those used by Mozharness.  This can
 # go away once Mozharness builds are no longer performed in Buildbot, and the
 # Mozharness code referencing routes.json is deleted.
 def check_v2_routes():
     with open(os.path.join(GECKO, "testing/mozharness/configs/routes.json"), "rb") as f:
         routes_json = json.load(f)
 
     for key in ('routes', 'nightly', 'l10n'):