Bug 1288567 - Use context hash of tar file; r?dustin
Now that tar file generation is deterministic, we can use the hash
of the created archive rather than the hash of the files that are
(presumably) in the archive.
This temporarily breaks consistent hashing by using independent
hashing mechanisms. This will be cleaned up in a subsequent commit.
MozReview-Commit-ID: CWooVGfDKZO
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -59,38 +59,40 @@ class DockerImageTask(base.Task):
'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
.format(repo=params['head_repository'], rev=params['head_rev']),
}
tasks = []
templates = Templates(path)
for image_name in config['images']:
context_path = os.path.join('testing', 'docker', image_name)
- context_hash = generate_context_hash(context_path)
image_parameters = dict(parameters)
- image_parameters['context_hash'] = context_hash
image_parameters['context_path'] = context_path
image_parameters['artifact_path'] = 'public/image.tar'
image_parameters['image_name'] = image_name
image_artifact_path = \
"public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
if os.environ.get('TASK_ID'):
destination = os.path.join(
os.environ['HOME'],
"artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name))
image_parameters['context_url'] = ARTIFACT_URL.format(
os.environ['TASK_ID'], image_artifact_path)
- cls.create_context_tar(context_path, destination, image_name)
+ context_hash = cls.create_context_tar(context_path, destination,
+ image_name)
else:
# skip context generation since this isn't a decision task
# TODO: generate context tarballs using subdirectory clones in
# the image-building task so we don't have to worry about this.
image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path
+ context_hash = generate_context_hash(context_path)
+
+ image_parameters['context_hash'] = context_hash
image_task = templates.load('image.yml', image_parameters)
attributes = {'image_name': image_name}
# As an optimization, if the context hash exists for mozilla-central, that image
# task ID will be used. The reasoning behind this is that eventually everything ends
# up on mozilla-central at some point if most tasks use this as a common image
@@ -128,22 +130,25 @@ class DockerImageTask(base.Task):
return True, existing_task['taskId']
except urllib2.HTTPError:
pass
return False, None
@classmethod
def create_context_tar(cls, context_dir, destination, image_name):
- 'Creates a tar file of a particular context directory.'
+ """Creates a tar file of a particular context directory.
+
+ Returns the SHA-256 hex digest of the created file.
+ """
destination = os.path.abspath(destination)
if not os.path.exists(os.path.dirname(destination)):
os.makedirs(os.path.dirname(destination))
- create_context_tar(context_dir, destination, image_name)
+ return create_context_tar(context_dir, destination, image_name)
@classmethod
def from_json(cls, task_dict):
# Generating index_paths for optimization
routes = task_dict['task']['routes']
index_paths = []
for route in routes:
index_path_regex = re.compile(INDEX_REGEX)