Bug 1288567 - Use context hash of tar file; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Fri, 22 Jul 2016 10:24:08 -0700
changeset 392503 293026da2349c6d42d5d98d30c1e0f5fdf7e294c
parent 392502 557cb3c6ff4869c8e7a575e00da6dea6e800babb
child 392504 3357e35462bca6ab9dcdd03309387e02e2f181bb
push id24042
push userbmo:gps@mozilla.com
push dateMon, 25 Jul 2016 18:25:42 +0000
reviewersdustin
bugs1288567
milestone50.0a1
Bug 1288567 - Use context hash of tar file; r?dustin Now that tar file generation is deterministic, we can use the hash of the created archive rather than the hash of the files that are (presumably) in the archive. This temporarily breaks consistent hashing by using independent hashing mechanisms. This will be cleaned up in a subsequent commit. MozReview-Commit-ID: CWooVGfDKZO
taskcluster/taskgraph/task/docker_image.py
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -59,38 +59,40 @@ class DockerImageTask(base.Task):
             'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
                       .format(repo=params['head_repository'], rev=params['head_rev']),
         }
 
         tasks = []
         templates = Templates(path)
         for image_name in config['images']:
             context_path = os.path.join('testing', 'docker', image_name)
-            context_hash = generate_context_hash(context_path)
 
             image_parameters = dict(parameters)
-            image_parameters['context_hash'] = context_hash
             image_parameters['context_path'] = context_path
             image_parameters['artifact_path'] = 'public/image.tar'
             image_parameters['image_name'] = image_name
 
             image_artifact_path = \
                 "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
             if os.environ.get('TASK_ID'):
                 destination = os.path.join(
                     os.environ['HOME'],
                     "artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name))
                 image_parameters['context_url'] = ARTIFACT_URL.format(
                     os.environ['TASK_ID'], image_artifact_path)
-                cls.create_context_tar(context_path, destination, image_name)
+                context_hash = cls.create_context_tar(context_path, destination,
+                                                      image_name)
             else:
                 # skip context generation since this isn't a decision task
                 # TODO: generate context tarballs using subdirectory clones in
                 # the image-building task so we don't have to worry about this.
                 image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path
+                context_hash = generate_context_hash(context_path)
+
+            image_parameters['context_hash'] = context_hash
 
             image_task = templates.load('image.yml', image_parameters)
 
             attributes = {'image_name': image_name}
 
             # As an optimization, if the context hash exists for mozilla-central, that image
             # task ID will be used.  The reasoning behind this is that eventually everything ends
             # up on mozilla-central at some point if most tasks use this as a common image
@@ -128,22 +130,25 @@ class DockerImageTask(base.Task):
                 return True, existing_task['taskId']
             except urllib2.HTTPError:
                 pass
 
         return False, None
 
     @classmethod
     def create_context_tar(cls, context_dir, destination, image_name):
-        'Creates a tar file of a particular context directory.'
+        """Creates a tar file of a particular context directory.
+
+        Returns the SHA-256 hex digest of the created file.
+        """
         destination = os.path.abspath(destination)
         if not os.path.exists(os.path.dirname(destination)):
             os.makedirs(os.path.dirname(destination))
 
-        create_context_tar(context_dir, destination, image_name)
+        return create_context_tar(context_dir, destination, image_name)
 
     @classmethod
     def from_json(cls, task_dict):
         # Generating index_paths for optimization
         routes = task_dict['task']['routes']
         index_paths = []
         for route in routes:
             index_path_regex = re.compile(INDEX_REGEX)