Bug 1333255: replace uses of index_paths with optimizations; r=jonasfj
MozReview-Commit-ID: 1w8HVv94xoD
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -2,17 +2,16 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
from . import transform
-from taskgraph.util.docker import INDEX_PREFIX
from taskgraph.transforms.base import TransformSequence, TransformConfig
from taskgraph.util.python_path import find_object
logger = logging.getLogger(__name__)
def transform_inputs(inputs, kind, path, config, params, loaded_tasks):
"""
@@ -35,18 +34,10 @@ def load_tasks(kind, path, config, param
transform.get_inputs(kind, path, config, params, loaded_tasks),
kind, path, config, params, loaded_tasks)
class DockerImageTask(transform.TransformTask):
@classmethod
def from_json(cls, task_dict):
- # Generating index_paths for optimization
- imgMeta = task_dict['task']['extra']['imageMeta']
- image_name = imgMeta['imageName']
- context_hash = imgMeta['contextHash']
- index_paths = ['{}.level-{}.{}.hash.{}'.format(
- INDEX_PREFIX, level, image_name, context_hash)
- for level in reversed(range(int(imgMeta['level']), 4))]
- task_dict['index_paths'] = index_paths
docker_image_task = cls(kind='docker-image', task=task_dict)
return docker_image_task
--- a/taskcluster/taskgraph/test/test_taskgraph.py
+++ b/taskcluster/taskgraph/test/test_taskgraph.py
@@ -6,50 +6,46 @@ from __future__ import absolute_import,
import unittest
from ..graph import Graph
from ..task.docker_image import DockerImageTask
from ..task.transform import TransformTask
from ..taskgraph import TaskGraph
from mozunit import main
-from taskgraph.util.docker import INDEX_PREFIX
class TestTaskGraph(unittest.TestCase):
def test_from_json(self):
task = {
"routes": [],
"extra": {
"imageMeta": {
"contextHash": "<hash>",
"imageName": "<image>",
"level": "1"
}
}
}
- index_paths = ["{}.level-{}.<image>.hash.<hash>".format(INDEX_PREFIX, level)
- for level in reversed(range(1, 4))]
graph = TaskGraph(tasks={
'a': TransformTask(
kind='fancy',
task={
'label': 'a',
'attributes': {},
'dependencies': {},
'when': {},
'task': {'task': 'def'},
}),
'b': DockerImageTask(kind='docker-image',
task={
'label': 'b',
'attributes': {},
'task': task,
- 'index_paths': index_paths,
}),
}, graph=Graph(nodes={'a', 'b'}, edges=set()))
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph.tasks['a'], new_graph.tasks['a'])
self.assertEqual(graph, new_graph)
if __name__ == '__main__':
--- a/taskcluster/taskgraph/transforms/docker_image.py
+++ b/taskcluster/taskgraph/transforms/docker_image.py
@@ -56,18 +56,18 @@ def fill_template(config, tasks):
context_hash=context_hash,
))
# As an optimization, if the context hash exists for a high level, that image
# task ID will be used. The reasoning behind this is that eventually everything ends
# up on level 3 at some point if most tasks use this as a common image
# for a given context hash, a worker within Taskcluster does not need to contain
# the same image per branch.
- index_paths = ['{}.level-{}.{}.hash.{}'.format(
- INDEX_PREFIX, level, image_name, context_hash)
+ optimizations = [['index-search', '{}.level-{}.{}.hash.{}'.format(
+ INDEX_PREFIX, level, image_name, context_hash)]
for level in reversed(range(int(config.params['level']), 4))]
# include some information that is useful in reconstructing this task
# from JSON
extra = {
'imageMeta': {
'level': config.params['level'],
'contextHash': context_hash,
@@ -76,17 +76,17 @@ def fill_template(config, tasks):
}
taskdesc = {
'label': 'build-docker-image-' + image_name,
'description': description,
'attributes': {'image_name': image_name},
'expires-after': '1 year',
'routes': routes,
- 'index-paths': index_paths,
+ 'optimizations': optimizations,
'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'],
'extra': extra,
'treeherder': {
'symbol': job_symbol,
'platform': 'taskcluster-images/opt',
'kind': 'other',
'tier': 1,
},
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -36,36 +36,36 @@ toolchain_run_schema = Schema({
),
# Paths/patterns pointing to files that influence the outcome of a
# toolchain build.
Optional('resources'): [basestring],
})
-def add_index_paths(config, run, taskdesc):
+def add_optimizations(config, run, taskdesc):
files = list(run.get('resources', []))
# This file
files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
# The script
files.append('taskcluster/scripts/misc/{}'.format(run['script']))
label = taskdesc['label']
subs = {
'name': label.replace('toolchain-', '').split('/')[0],
'digest': hash_paths(GECKO, files),
}
- index_paths = taskdesc.setdefault('index-paths', [])
+ optimizations = taskdesc.setdefault('optimizations', [])
# We'll try to find a cached version of the toolchain at levels above
# and including the current level, starting at the highest level.
for level in reversed(range(int(config.params['level']), 4)):
subs['level'] = level
- index_paths.append(TOOLCHAIN_INDEX.format(**subs))
+ optimizations.append(['index-search', TOOLCHAIN_INDEX.format(**subs)])
# ... and cache at the lowest level.
taskdesc.setdefault('routes', []).append(
'index.{}'.format(TOOLCHAIN_INDEX.format(**subs)))
@run_job_using("docker-worker", "toolchain-script", schema=toolchain_run_schema)
def docker_worker_toolchain(config, job, taskdesc):
@@ -119,17 +119,17 @@ def docker_worker_toolchain(config, job,
'--',
'bash',
'-c',
'cd /home/worker && '
'./workspace/build/src/taskcluster/scripts/misc/{}'.format(
run['script'])
]
- add_index_paths(config, run, taskdesc)
+ add_optimizations(config, run, taskdesc)
@run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema)
def windows_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
@@ -168,9 +168,9 @@ def windows_toolchain(config, job, taskd
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
' '.join(hg_command),
# do something intelligent.
r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
]
- add_index_paths(config, run, taskdesc)
+ add_optimizations(config, run, taskdesc)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -47,20 +47,16 @@ task_description_schema = Schema({
# (e.g., "14 days"). Defaults are set based on the project.
Optional('expires-after'): basestring,
Optional('deadline-after'): basestring,
# custom routes for this task; the default treeherder routes will be added
# automatically
Optional('routes'): [basestring],
- # The index paths where this task may be cached. Transforms are expected to
- # fill these automatically when wanted.
- Optional('index-paths'): [basestring],
-
# custom scopes for this task; any scopes required for the worker will be
# added automatically
Optional('scopes'): [basestring],
# Tags
Optional('tags'): {basestring: object},
# custom "task.extra" content
@@ -794,17 +790,16 @@ def add_files_changed(config, tasks):
yield task
@transforms.add
def setup_optimizations(config, tasks):
for task in tasks:
optimizations = task.setdefault('optimizations', [])
- optimizations.extend([['index-search', idx] for idx in task.get('index-paths', [])])
optimizations.append(['seta'])
if 'when' in task and 'files-changed' in task['when']:
optimizations.append(['files-changed', task['when']['files-changed']])
yield task
@transforms.add
def build_task(config, tasks):