Bug 1391476 - Track whether caches should be used in untrusted environments; r?dustin
Previously, we conditionally added caches to a task if the current
parameters warranted it.
In order to audit that all caches fulfill basic requirements, we need
to have unconditional knowledge of all caches.
This commit introduces an optional key on each cache entry stating
whether it should be skipped in "untrusted" environments. When we
convert a task definition to a worker payload, we filter out these
caches if necessary.
This change uncovered an inconsistency with filtering caches. In
one location we filtered on the source repo name. In others, we
filtered on the SCM level.
Setting the caches in the spidermonkey kind also changed slightly
to ensure we're not overwriting existing caches. I don't think this
has any behavior changes. But the new method is more correct.
MozReview-Commit-ID: 1crpdWHqQ68
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -8,32 +8,31 @@ consistency.
"""
from __future__ import absolute_import, print_function, unicode_literals
SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
def docker_worker_add_workspace_cache(config, job, taskdesc, extra=None):
- """Add the workspace cache based on the build platform/type and level,
- except on try where workspace caches are not used.
+ """Add the workspace cache.
- extra, is an optional kwarg passed in that supports extending the cache
+ ``extra`` is an optional kwarg passed in that supports extending the cache
key name to avoid undesired conflicts with other caches."""
- if config.params['project'] == 'try':
- return
-
taskdesc['worker'].setdefault('caches', []).append({
'type': 'persistent',
'name': 'level-{}-{}-build-{}-{}-workspace'.format(
config.params['level'], config.params['project'],
taskdesc['attributes']['build_platform'],
taskdesc['attributes']['build_type'],
),
'mount-point': "/home/worker/workspace",
+ # Don't enable the workspace cache when we can't guarantee its
+ # behavior, like on Try.
+ 'skip-untrusted': True,
})
if extra:
taskdesc['worker']['caches'][-1]['name'] += '-{}'.format(
extra
)
def docker_worker_add_tc_vcs_cache(config, job, taskdesc):
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -38,22 +38,22 @@ def common_setup(config, job, taskdesc):
@run_job_using("docker-worker", "run-task", schema=run_task_schema)
def docker_worker_run_task(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker'] = job['worker']
common_setup(config, job, taskdesc)
- if run.get('cache-dotcache') and int(config.params['level']) > 1:
- worker['caches'].append({
- 'type': 'persistent',
- 'name': 'level-{level}-{project}-dotcache'.format(**config.params),
- 'mount-point': '/home/worker/.cache',
- })
+ worker['caches'].append({
+ 'type': 'persistent',
+ 'name': 'level-{level}-{project}-dotcache'.format(**config.params),
+ 'mount-point': '/home/worker/.cache',
+ 'skip-untrusted': True,
+ })
run_command = run['command']
if isinstance(run_command, basestring):
run_command = ['bash', '-cx', run_command]
command = ['/home/worker/bin/run-task']
if run['checkout']:
command.append('--vcs-checkout=/home/worker/checkouts/gecko')
command.append('--fetch-hgfingerprint')
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -31,25 +31,23 @@ sm_run_schema = Schema({
@run_job_using("docker-worker", "spidermonkey-package", schema=sm_run_schema)
@run_job_using("docker-worker", "spidermonkey-mozjs-crate",
schema=sm_run_schema)
def docker_worker_spidermonkey(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = []
- worker['caches'] = []
-
- if int(config.params['level']) > 1:
- worker['caches'].append({
- 'type': 'persistent',
- 'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
- config.params['level'], config.params['project']),
- 'mount-point': "/home/worker/workspace",
- })
+ worker.setdefault('caches', []).append({
+ 'type': 'persistent',
+ 'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
+ config.params['level'], config.params['project']),
+ 'mount-point': "/home/worker/workspace",
+ 'skip-untrusted': True,
+ })
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_tooltool(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_DISABLE': 'true',
'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -199,16 +199,20 @@ task_description_schema = Schema({
'type': 'persistent',
# name of the cache, allowing re-use by subsequent tasks naming the
# same cache
'name': basestring,
# location in the task image where the cache will be mounted
'mount-point': basestring,
+
+ # Whether the cache is not used in untrusted environments
+ # (like the Try repo).
+ Optional('skip-untrusted', default=False): bool,
}],
# artifacts to extract from the task image after completion
Optional('artifacts'): [{
# type of artifact -- simple file, or recursive directory
'type': Any('file', 'directory'),
# task image path from which to read artifact
@@ -580,16 +584,17 @@ def index_builder(name):
index_builders[name] = func
return func
return wrap
@payload_builder('docker-worker')
def build_docker_worker_payload(config, task, task_def):
worker = task['worker']
+ level = int(config.params['level'])
image = worker['docker-image']
if isinstance(image, dict):
if 'in-tree' in image:
docker_image_task = 'build-docker-image-' + image['in-tree']
task.setdefault('dependencies', {})['docker-image'] = docker_image_task
image = {
@@ -679,17 +684,24 @@ def build_docker_worker_payload(config,
run_task = payload.get('command', [''])[0].endswith('run-task')
if run_task:
suffix = '-%s' % _run_task_suffix()
else:
suffix = ''
+ skip_untrusted = config.params['project'] == 'try' or level == 1
+
for cache in worker['caches']:
+ # Some caches aren't enabled in environments where we can't
+ # guarantee certain behavior. Filter those out.
+ if cache.get('skip-untrusted') and skip_untrusted:
+ continue
+
name = '%s%s' % (cache['name'], suffix)
caches[name] = cache['mount-point']
task_def['scopes'].append('docker-worker:cache:%s' % name)
# Assertion: only run-task is interested in this.
if run_task:
payload['env']['TASKCLUSTER_CACHES'] = ';'.join(sorted(
caches.values()))
@@ -697,17 +709,17 @@ def build_docker_worker_payload(config,
payload['cache'] = caches
if features:
payload['features'] = features
if capabilities:
payload['capabilities'] = capabilities
# coalesce / superseding
- if 'coalesce-name' in task and int(config.params['level']) > 1:
+ if 'coalesce-name' in task and level > 1:
key = COALESCE_KEY.format(
project=config.params['project'],
name=task['coalesce-name'])
payload['supersederUrl'] = "https://coalesce.mozilla-releng.net/v1/list/" + key
@payload_builder('generic-worker')
def build_generic_worker_payload(config, task, task_def):