Bug 1391789 - Validate certain caches are used with run-task; r?dustin
run-task just grew features to aid with cache validation.
Attempts by run-task to use caches not under its control will fail.
So, we add a transform that audits for and ensures that certain
caches are only being used with run-task. This will help catch
stragglers attempting to use e.g. the legacy VCS checkouts or
tooltool caches without run-task. Fortunately, there are no
violations for this policy. Yay!
MozReview-Commit-ID: LBCmDUdgcuM
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -72,18 +72,18 @@ def docker_worker_add_gecko_vcs_env_vars
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
})
def support_vcs_checkout(config, job, taskdesc):
"""Update a job/task with parameters to enable a VCS checkout.
- The configuration is intended for tasks using "run-task" and its
- VCS checkout behavior.
+ This can only be used with ``run-task`` tasks, as the cache name is
+ reserved for ``run-task`` tasks.
"""
level = config.params['level']
# native-engine does not support caches (yet), so we just do a full clone
# every time :(
if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
taskdesc['worker'].setdefault('caches', []).append({
'type': 'persistent',
@@ -133,16 +133,19 @@ def docker_worker_setup_secrets(config,
def docker_worker_add_tooltool(config, job, taskdesc, internal=False):
"""Give the task access to tooltool.
Enables the tooltool cache. Adds releng proxy. Configures scopes.
By default, only public tooltool access will be granted. Access to internal
tooltool can be enabled via ``internal=True``.
+
+ This can only be used with ``run-task`` tasks, as the cache name is
+ reserved for use with ``run-task``.
"""
assert job['worker']['implementation'] in ('docker-worker', 'docker-engine')
level = config.params['level']
taskdesc['worker'].setdefault('caches', []).append({
'type': 'persistent',
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -7,16 +7,17 @@ task definition (along with attributes,
transformations is generic to any kind of task, but abstracts away some of the
complexities of worker implementations, scopes, and treeherder annotations.
"""
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
+import re
import time
from copy import deepcopy
from mozbuild.util import memoize
from taskgraph.util.attributes import TRUNK_PROJECTS
from taskgraph.util.hash import hash_path
from taskgraph.util.treeherder import split_symbol
from taskgraph.transforms.base import TransformSequence
@@ -1111,16 +1112,60 @@ def build_task(config, tasks):
'label': task['label'],
'task': task_def,
'dependencies': task.get('dependencies', {}),
'attributes': attributes,
'optimizations': task.get('optimizations', []),
}
+@transforms.add
+def check_run_task_caches(config, tasks):
+ """Audit for caches requiring run-task.
+
+ run-task manages caches in certain ways. If a cache managed by run-task
+ is used by a non run-task task, it could cause problems. So we audit for
+ that and make sure certain cache names are exclusive to run-task.
+
+ IF YOU ARE TEMPTED TO MAKE EXCLUSIONS TO THIS POLICY, YOU ARE LIKELY
+ CONTRIBUTING TECHNICAL DEBT AND WILL HAVE TO SOLVE MANY OF THE PROBLEMS
+ THAT RUN-TASK ALREADY SOLVES. THINK LONG AND HARD BEFORE DOING THAT.
+ """
+ re_reserved_caches = re.compile('''^
+ (level-\d+-checkouts|level-\d+-tooltool-cache)
+ ''', re.VERBOSE)
+
+ suffix = _run_task_suffix()
+
+ for task in tasks:
+ payload = task['task'].get('payload', {})
+ command = payload.get('command') or ['']
+ command = command[0] if isinstance(command[0], basestring) else ''
+ run_task = command.endswith('run-task')
+
+ for cache in payload.get('cache', {}):
+ if not re_reserved_caches.match(cache):
+ continue
+
+ if not run_task:
+ raise Exception(
+ '%s is using a cache (%s) reserved for run-task '
+ 'change the task to use run-task or use a different '
+ 'cache name' % (task['label'], cache))
+
+ if not cache.endswith(suffix):
+ raise Exception(
+ '%s is using a cache (%s) reserved for run-task '
+ 'but the cache name is not dependent on the contents '
+ 'of run-task; change the cache name to conform to the '
+ 'naming requirements' % (task['label'], cache))
+
+ yield task
+
+
# Check that the v2 route templates match those used by Mozharness. This can
# go away once Mozharness builds are no longer performed in Buildbot, and the
# Mozharness code referencing routes.json is deleted.
def check_v2_routes():
with open(os.path.join(GECKO, "testing/mozharness/configs/routes.json"), "rb") as f:
routes_json = json.load(f)
for key in ('routes', 'nightly', 'l10n'):