Bug 1314847 - Sign mar files as part of signingscriptworker -- f?kmoir
authorJustin Wood <Callek@gmail.com>
Wed, 02 Nov 2016 21:47:16 -0400
changeset 434827 4a1231655fbb9bd971f09da60475d0ae26668640
parent 434826 10b21d912f60b8de5c655799a27e9229c8761bb1
child 436825 c372afc478a5a0a7f6c7bd07689902b4c4f1cc1b
push id34840
push userCallek@gmail.com
push dateMon, 07 Nov 2016 14:08:04 +0000
bugs1314847
milestone52.0a1
Bug 1314847 - Sign mar files as part of signingscriptworker -- f?kmoir Explicitly skips beetmoving the mar files for now. MozReview-Commit-ID: AKcKurIjC5a
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/build_signing.py
taskcluster/taskgraph/transforms/nightly_l10n_signing.py
taskcluster/taskgraph/transforms/signing.py
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -49,16 +49,26 @@ def validate(config, jobs):
     for job in jobs:
         label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
         yield validate_schema(
             beetmover_description_schema, job,
             "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
 
 
 @transforms.add
+def skip_mar_sign_moving(config, jobs):
+    "We should skip beetmoving mar files until we are ready for that piece"
+    for job in jobs:
+        dep_job = job['dependent-task']
+        if "signing-mar-" in dep_job.label:
+            continue  # by not yielding we are not creating a task
+        yield job
+
+
+@transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
         treeherder = job.get('treeherder', {})
         treeherder.setdefault('symbol', 'tc(BM)')
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
--- a/taskcluster/taskgraph/transforms/build_signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -3,35 +3,61 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.treeherder import join_symbol
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/{}'
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def make_signing_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
-        job['label'] = dep_job.label.replace("build-", "signing-")
-
         artifacts = []
         if 'android' in dep_job.attributes.get('build_platform'):
-            artifacts = ['public/build/target.apk', 'public/build/en-US/target.apk']
+            job_specs = [
+                {
+                    'artifacts': ['public/build/target.apk',
+                                  'public/build/en-US/target.apk'],
+                    'format': 'jar',
+                },
+            ]
         else:
-            artifacts = ['public/build/target.tar.bz2']
+            job_specs = [
+                {
+                    'artifacts': ['public/build/target.tar.bz2',
+                                  'public/build/target.checksums'],
+                    'format': 'gpg',
+                }, {
+                    'artifacts': ['public/build/target.complete.mar'],
+                    'format': 'mar',
+                }
+            ]
         unsigned_artifacts = []
-        for artifact in artifacts:
-            url = {"task-reference": ARTIFACT_URL.format('build', artifact)}
-            unsigned_artifacts.append(url)
+        for spec in job_specs:
+            fmt = spec["format"]
+            for artifact in artifacts:
+                url = {"task-reference": ARTIFACT_URL.format('build', artifact)}
+                unsigned_artifacts.append(url)
+
+            job['unsigned-artifacts'] = unsigned_artifacts
+            job['signing-format'] = "gpg" if "linux" in dep_job.label else "jar"
 
-        job['unsigned-artifacts'] = unsigned_artifacts
-        job['signing-format'] = "gpg" if "linux" in dep_job.label else "jar"
+            label = dep_job.label.replace("build-", "signing-{}-".format(fmt))
+            job['label'] = label
 
-        yield job
+            # add the format character to the TH symbol
+            symbol = 'Ns{}'.format(fmt.title()[:1])
+            group = 'tc'
+
+            job['treeherder'] = {
+                'symbol': join_symbol(group, symbol),
+            }
+            yield job
--- a/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
+++ b/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
@@ -11,49 +11,58 @@ from taskgraph.transforms.base import Tr
 from taskgraph.util.treeherder import join_symbol
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/public/build/{}'
 
 transforms = TransformSequence()
 
 
 @transforms.add
-def add_signing_artifacts(config, jobs):
+def make_signing_description(config, jobs):
     for job in jobs:
+        job['depname'] = 'unsigned-repack'
+
         dep_job = job['dependent-task']
         dep_platform = dep_job.attributes.get('build_platform')
 
         job['unsigned-artifacts'] = []
-        extension = '.apk' if 'android' in dep_platform else '.tar.bz2'
-        for locale in dep_job.attributes.get('chunk_locales', []):
-            filename = '{}/target{}'.format(locale, extension)
-            job['unsigned-artifacts'].append({
-                'task-reference': ARTIFACT_URL.format('unsigned-repack',
-                                                      filename)
-                })
-            if 'tar.bz2' == filename[-7:]:
-                # Add the checksums file to be signed for linux
-                checksums_file = filename[:-7] + "checksums"
-                job['unsigned-artifacts'].append({
-                    'task-reference': ARTIFACT_URL.format('unsigned-repack',
-                                                          checksums_file)
-                    })
-        yield job
-
+        if 'android' in dep_platform:
+            job_specs = [
+                {
+                    'extensions': ['.apk'],
+                    'format': 'jar',
+                },
+            ]
+        else:
+            job_specs = [
+                {
+                    'extensions': ['.tar.bz2', '.checksums'],
+                    'format': 'gpg',
+                }, {
+                    'extensions': ['complete.mar'],
+                    'format': 'mar',
+                }
+            ]
+        for spec in job_specs:
+            fmt = spec['format']
+            for locale in dep_job.attributes.get('chunk_locales', []):
+                for ext in spec['extensions']:
+                    filename = '{}/target{}'.format(locale, ext)
+                    job['unsigned-artifacts'].append({
+                        'task-reference': ARTIFACT_URL.format('unsigned-repack',
+                                                              filename)
+                        })
 
-@transforms.add
-def make_signing_description(config, jobs):
-    for job in jobs:
-        dep_job = job['dependent-task']
+            job['signing-format'] = fmt
 
-        job['label'] = dep_job.label.replace("nightly-l10n-", "signing-l10n-")
+            label = dep_job.label.replace("nightly-l10n-",
+                                          "signing-l10n-{}-".format(fmt))
+            job['label'] = label
 
-        job['depname'] = 'unsigned-repack'
-        job['signing-format'] = "gpg" if "linux" in dep_job.label else "jar"
+            # add the chunk number to the TH symbol
+            symbol = 'Ns{}{}'.format(dep_job.attributes.get('l10n_chunk'),
+                                     fmt.title()[:1])
+            group = 'tc-L10n'
 
-        # add the chunk number to the TH symbol
-        symbol = 'Ns{}'.format(dep_job.attributes.get('l10n_chunk'))
-        group = 'tc-L10n'
-
-        job['treeherder'] = {
-            'symbol': join_symbol(group, symbol),
-        }
-        yield job
+            job['treeherder'] = {
+                'symbol': join_symbol(group, symbol),
+            }
+            yield job
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -76,18 +76,19 @@ def make_task_description(config, jobs):
         treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
         treeherder.setdefault('tier', 2)
         treeherder.setdefault('kind', 'build')
 
         label = job.get('label', "{}-signing".format(dep_job))
 
         task = {
             'label': label,
-            'description': "{} Signing".format(
-                dep_job.task["metadata"]["description"]),
+            'description': "{} Signing ({})".format(
+                dep_job.task["metadata"]["description"],
+                job['signing-format']),
             'worker-type': "scriptworker-prov-v1/signing-linux-v1",
             'worker': {'implementation': 'scriptworker-signing',
                        'unsigned-artifacts': job['unsigned-artifacts']},
             'scopes': ["project:releng:signing:cert:nightly-signing",
                        signing_format_scope],
             'dependencies': {job['depname']: dep_job.label},
             'attributes': {
                 'nightly': dep_job.attributes.get('nightly', False),