Bug 1427326 - Allow Docker images tasks to depend on package tasks. r=dustin draft
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 29 Dec 2017 13:03:19 +0900
changeset 716017 ccaadc35403a2c4ecd0c11dff8d18c1881f000b4
parent 716016 4d8eeec6f879aeb870bd242f17507d83390590ee
child 716018 b1575eb74030311d023981010c58176be6f78efc
push id94300
push userbmo:mh+mozilla@glandium.org
push dateThu, 04 Jan 2018 22:59:03 +0000
reviewersdustin
bugs1427326
milestone59.0a1
Bug 1427326 - Allow Docker images tasks to depend on package tasks. r=dustin
taskcluster/taskgraph/transforms/docker_image.py
--- a/taskcluster/taskgraph/transforms/docker_image.py
+++ b/taskcluster/taskgraph/transforms/docker_image.py
@@ -36,34 +36,60 @@ docker_image_schema = Schema({
     Optional('job-from'): basestring,
 
     # Arguments to use for the Dockerfile.
     Optional('args'): {basestring: basestring},
 
     # Name of the docker image definition under taskcluster/docker, when
     # different from the docker image name.
     Optional('definition'): basestring,
+
+    # List of package tasks this docker image depends on.
+    Optional('packages'): [basestring],
 })
 
 
 @transforms.add
 def validate(config, tasks):
     for task in tasks:
         yield validate_schema(
             docker_image_schema, task,
             "In docker image {!r}:".format(task.get('name', 'unknown')))
 
 
 @transforms.add
 def fill_template(config, tasks):
+    available_packages = {}
+    for task in config.kind_dependencies_tasks:
+        if task.kind != 'packages':
+            continue
+        name = task.label.replace('packages-', '')
+        for route in task.task.get('routes', []):
+            if route.startswith('index.') and '.hash.' in route:
+                available_packages[name] = route
+                break
     for task in tasks:
         image_name = task.pop('name')
         job_symbol = task.pop('symbol')
         args = task.pop('args', {})
         definition = task.pop('definition', image_name)
+        packages = task.pop('packages', [])
+
+        for p in packages:
+            if p not in available_packages:
+                raise Exception('Missing package job for {}-{}: {}'.format(
+                    config.kind, image_name, p))
+
+        # Generating the context hash relies on arguments being set, so we
+        # set this now, although it's not the final value (it's a
+        # task-reference value, see further below). We add the package routes
+        # containing a hash to get the overall docker image hash, so changes
+        # to packages will be reflected in the docker image hash.
+        args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p)
+                                                 for p in packages)
 
         context_path = os.path.join('taskcluster', 'docker', definition)
         context_hash = generate_context_hash(
             GECKO, context_path, image_name, args)
 
         description = 'Build the docker image {} for use by dependent tasks'.format(
             image_name)
 
@@ -126,18 +152,30 @@ def fill_template(config, tasks):
                 'chain-of-trust': True,
                 'docker-in-docker': True,
                 'taskcluster-proxy': True,
                 'max-run-time': 7200,
             },
         }
 
         for k, v in args.items():
-            taskdesc['worker']['env'][k] = v
+            if k == 'DOCKER_IMAGE_PACKAGES':
+                taskdesc['worker']['env'][k] = {'task-reference': v}
+            else:
+                taskdesc['worker']['env'][k] = v
 
+        if packages:
+            deps = taskdesc.setdefault('dependencies', {})
+            digest_data = [context_hash]
+            for p in sorted(packages):
+                deps[p] = 'packages-{}'.format(p)
+                digest_data.append(available_packages[p])
+            kwargs = {'digest_data': digest_data}
+        else:
+            kwargs = {'digest': context_hash}
         add_optimization(
             config, taskdesc,
             cache_type="docker-images.v1",
             cache_name=image_name,
-            digest=context_hash,
+            **kwargs
         )
 
         yield taskdesc