Bug 1373326 - Reformat repackage tasks taskgraph to support new input style. r=dustin draft
authorJustin Wood <Callek@gmail.com>
Fri, 16 Jun 2017 15:34:43 -0400
changeset 597684 0507e659995a25bfa90995476ec5e24751dae57f
parent 597683 a575562ba62e276ccc15f59292f3070e203bc73c
child 597685 3651dd368f032ae9f17cba42010902f850a64700
push id64993
push userCallek@gmail.com
push dateTue, 20 Jun 2017 20:14:15 +0000
reviewersdustin
bugs1373326
milestone56.0a1
Bug 1373326 - Reformat repackage tasks taskgraph to support new input style. r=dustin (For Landing more OSX Nightly Support from date to central) MozReview-Commit-ID: F8Dl1502D4L
taskcluster/ci/repackage-l10n/kind.yml
taskcluster/ci/repackage/kind.yml
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/repackage_l10n.py
--- a/taskcluster/ci/repackage-l10n/kind.yml
+++ b/taskcluster/ci/repackage-l10n/kind.yml
@@ -2,15 +2,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
    - taskgraph.transforms.repackage_l10n:transforms
    - taskgraph.transforms.repackage:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - nightly-l10n-signing
 
 only-for-build-platforms:
   - macosx64-nightly/opt
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
    - taskgraph.transforms.repackage:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - build-signing
 
 only-for-build-platforms:
   - macosx64-nightly/opt
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -90,17 +90,17 @@ job_description_schema = Schema({
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
         yield validate_schema(job_description_schema, job,
-                              "In job {!r}:".format(job['name']))
+                              "In job {!r}:".format(job.get('name', job.get('label'))))
 
 
 @transforms.add
 def rewrite_when_to_optimization(config, jobs):
     for job in jobs:
         when = job.pop('when', {})
         files_changed = when.get('files-changed')
         if not files_changed:
@@ -128,17 +128,17 @@ def make_task_description(config, jobs):
     """Given a build description, create a task description"""
     # import plugin modules first, before iterating over jobs
     import_all()
     for job in jobs:
         if 'label' not in job:
             if 'name' not in job:
                 raise Exception("job has neither a name nor a label")
             job['label'] = '{}-{}'.format(config.kind, job['name'])
-        if job['name']:
+        if job.get('name'):
             del job['name']
 
         impl, os = worker_type_implementation(job['worker-type'])
         worker = job.setdefault('worker', {})
         assert 'implementation' not in worker
         worker['implementation'] = impl
         if os:
             worker['os'] = os
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -7,31 +7,38 @@ worker implementation they operate on, a
 consistency.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
 
 
-def docker_worker_add_workspace_cache(config, job, taskdesc):
+def docker_worker_add_workspace_cache(config, job, taskdesc, extra=None):
     """Add the workspace cache based on the build platform/type and level,
-    except on try where workspace caches are not used."""
+    except on try where workspace caches are not used.
+
+    extra, is an optional kwarg passed in that supports extending the cache
+    key name to avoid undesired conflicts with other caches."""
     if config.params['project'] == 'try':
         return
 
     taskdesc['worker'].setdefault('caches', []).append({
         'type': 'persistent',
         'name': 'level-{}-{}-build-{}-{}-workspace'.format(
             config.params['level'], config.params['project'],
             taskdesc['attributes']['build_platform'],
             taskdesc['attributes']['build_type'],
         ),
         'mount-point': "/home/worker/workspace",
     })
+    if extra:
+        taskdesc['worker']['caches'][-1]['name'] += '-{}'.format(
+            extra
+        )
 
 
 def docker_worker_add_tc_vcs_cache(config, job, taskdesc):
     taskdesc['worker'].setdefault('caches', []).append({
         'type': 'persistent',
         'name': 'level-{}-{}-tc-vcs'.format(
             config.params['level'], config.params['project']),
         'mount-point': "/home/worker/.tc-vcs",
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -44,16 +44,20 @@ mozharness_run_schema = Schema({
 
     # any additional options (without leading --) to be passed to mozharness;
     # not supported on Windows
     Optional('options'): [basestring],
 
     # --custom-build-variant-cfg value (not supported on Windows)
     Optional('custom-build-variant-cfg'): basestring,
 
+    # Extra metadata to use toward the workspace caching.
+    # Only supported on docker-worker
+    Optional('extra-workspace-cache-key'): basestring,
+
     # If not false, tooltool downloads will be enabled via relengAPIProxy
     # for either just public files, or all files.  Not supported on Windows
     Required('tooltool-downloads', default=False): Any(
         False,
         'public',
         'internal',
     ),
 
@@ -89,17 +93,18 @@ def mozharness_on_docker_worker_setup(co
 
     # running via mozharness assumes desktop-build (which contains build.sh)
     taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
 
     worker['relengapi-proxy'] = False  # but maybe enabled for tooltool below
     worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
 
     docker_worker_add_public_artifacts(config, job, taskdesc)
-    docker_worker_add_workspace_cache(config, job, taskdesc)
+    docker_worker_add_workspace_cache(config, job, taskdesc,
+                                      extra=run.get('extra-workspace-cache-key'))
     support_vcs_checkout(config, job, taskdesc)
 
     env = worker.setdefault('env', {})
     env.update({
         'MOZHARNESS_CONFIG': ' '.join(run['config']),
         'MOZHARNESS_SCRIPT': run['script'],
         'MH_BRANCH': config.params['project'],
         'MH_BUILD_POOL': 'taskcluster',
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -62,111 +62,104 @@ def validate(config, jobs):
 @transforms.add
 def make_repackage_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
         label = job.get('label',
                         dep_job.label.replace("signing-", "repackage-"))
         job['label'] = label
-        cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
-        cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
 
         yield job
 
 
 @transforms.add
-def make_task_description(config, jobs):
+def make_job_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
+        dependencies = {dep_job.attributes.get('kind'): dep_job.label}
+        if len(dep_job.dependencies) > 1:
+            raise NotImplementedError(
+                "Can't repackage a signing task with multiple dependencies")
 
         treeherder = job.get('treeherder', {})
         treeherder.setdefault('symbol', 'tc(Nr)')
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
+        signing_task = None
+        for dependency in dependencies.keys():
+            if 'signing' in dependency:
+                signing_task = dependency
+        signing_task_ref = "<{}>".format(signing_task)
 
         attributes = {
             'nightly': dep_job.attributes.get('nightly', False),
             'build_platform': dep_job.attributes.get('build_platform'),
             'build_type': dep_job.attributes.get('build_type'),
         }
+        if job.get('locale'):
+            attributes['locale'] = job['locale']
 
-        command = ['/home/worker/bin/run-task',
-                   # Various caches/volumes are default owned by root:root.
-                   '--chown-recursive', '/home/worker/workspace',
-                   '--chown-recursive', '/home/worker/tooltool-cache',
-                   '--vcs-checkout', '/home/worker/workspace/build/src',
-                   '--tools-checkout', '/home/worker/workspace/build/tools',
-                   '--',
-                   '/home/worker/workspace/build/src/taskcluster/scripts/builder/repackage.sh'
-                   ]
+        level = config.params['level']
 
-        dependencies = {dep_job.attributes.get('kind'): dep_job.label}
-        if job.get('locale'):
-            input_string = 'https://queue.taskcluster.net/v1/task/<nightly-l10n-signing>/' + \
-                'artifacts/public/build/{}/target.tar.gz'
-            input_string = input_string.format(job['locale'])
+        task_env = {}
+        locale_output_path = ""
+        if attributes['build_platform'].startswith('macosx'):
+            if job.get('locale'):
+                input_string = 'https://queue.taskcluster.net/v1/task/' + \
+                    '{}/artifacts/public/build/{}/target.tar.gz'
+                input_string = input_string.format(signing_task_ref, job['locale'])
+                locale_output_path = "{}/".format(job['locale'])
+            else:
+                input_string = 'https://queue.taskcluster.net/v1/task/' + \
+                    '{}/artifacts/public/build/target.tar.gz'.format(signing_task_ref)
+            task_env.update(
+                SIGNED_INPUT={'task-reference': input_string},
+            )
+            mozharness_config = ['repackage/osx_signed.py']
+            output_files = [{
+                'type': 'file',
+                'path': '/home/worker/workspace/build/artifacts/target.dmg',
+                'name': 'public/build/{}target.dmg'.format(locale_output_path),
+            }]
         else:
-            input_string = 'https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/' + \
-                'public/build/target.tar.gz'
-        signed_input = {'task-reference': input_string}
-        level = config.params['level']
+            raise Exception("Unexpected build platform for repackage")
+
+        run = {
+            'using': 'mozharness',
+            'script': 'mozharness/scripts/repackage.py',
+            'config': mozharness_config,
+            'job-script': 'taskcluster/scripts/builder/repackage.sh',
+            'actions': ['download_input', 'setup', 'repackage'],
+            'extra-workspace-cache-key': 'repackage',
+        }
+
+        if attributes["build_platform"].startswith('macosx'):
+            worker = {
+                'docker-image': {"in-tree": "desktop-build"},
+                'artifacts': output_files,
+                'env': task_env,
+                'chain-of-trust': True,
+                'max-run-time': 3600
+            }
+            run["tooltool-downloads"] = 'internal'
+            worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
+            cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
+            cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
 
         task = {
             'label': job['label'],
             'description': "{} Repackage".format(
                 dep_job.task["metadata"]["description"]),
-            'worker-type': 'aws-provisioner-v1/gecko-%s-b-macosx64' % level,
+            'worker-type': worker_type,
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'routes': job.get('routes', []),
             'extra': job.get('extra', {}),
-            'scopes':
-            ['docker-worker:relengapi-proxy:tooltool.download.internal',
-             'secrets:get:project/taskcluster/gecko/hgfingerprint',
-             'docker-worker:relengapi-proxy:tooltool.download.public',
-             'project:releng:signing:format:dmg'],
-            'worker': {'implementation': 'docker-worker',
-                       'os': 'linux',
-                       'docker-image': {"in-tree": "desktop-build"},
-                       'caches': [{
-                                   'type': 'persistent',
-                                   'name': 'tooltool-cache',
-                                   'mount-point': '/home/worker/tooltool-cache',
-                                 }, {
-                                   'type': 'persistent',
-                                   'name': 'level-%s-checkouts-v1' % level,
-                                   'mount-point': '/home/worker/checkouts',
-                                 }],
-                       'artifacts': [{
-                                       'type': 'file',
-                                       'path': '/home/worker/workspace/build/upload/target.dmg',
-                                       'name': 'public/build/target.dmg',
-                                     }],
-                       'env': {
-                               'SIGNED_INPUT': signed_input,
-                               'MOZ_BUILD_DATE': config.params['moz_build_date'],
-                               'MH_BUILD_POOL': 'taskcluster',
-                               'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
-                               'GECKO_HEAD_REV': config.params['head_rev'],
-                               'MH_BRANCH': config.params['project'],
-                               'MOZ_SCM_LEVEL': level,
-                               'MOZHARNESS_ACTIONS': 'download_input setup repackage',
-                               'NEED_XVFB': 'true',
-                               'GECKO_BASE_REPOSITORY': config.params['base_repository'],
-                               'TOOLTOOL_CACHE': '/home/worker/tooltool-cache',
-                               'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
-                               'MOZHARNESS_CONFIG': 'repackage/osx_signed.py',
-                               'USE_SCCACHE': '1',
-                               'MOZHARNESS_SCRIPT': 'mozharness/scripts/repackage.py'
-                               },
-                       'command': command,
-                       'chain-of-trust': True,
-                       'relengapi-proxy': True,
-                       'max-run-time': 3600
-                       }
+            'worker': worker,
+            'run': run,
         }
         yield task
--- a/taskcluster/taskgraph/transforms/repackage_l10n.py
+++ b/taskcluster/taskgraph/transforms/repackage_l10n.py
@@ -12,20 +12,22 @@ from taskgraph.transforms.base import Tr
 transforms = TransformSequence()
 
 
 @transforms.add
 def split_locales(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
         for locale in dep_job.attributes.get('chunk_locales', []):
-            job['locale'] = locale
-
             label = dep_job.label.replace("signing-", "repackage-{}-".format(locale))
             label = "repackage-{}-{}/{}".format(locale,
                                                 dep_job.attributes['build_platform'],
                                                 dep_job.attributes['build_type'],)
-            job['label'] = label
 
             treeherder = job.get('treeherder', {})
             treeherder['symbol'] = 'tc-L10n-Rpk({})'.format(locale)
-            job['treeherder'] = treeherder
-            yield job
+
+            yield {
+                'locale': locale,
+                'label': label,
+                'treeherder': treeherder,
+                'dependent-task': dep_job,
+            }