Bug 1457010 - gpg sign partner repacks, r?aki draft
authorNick Thomas <nthomas@mozilla.com>
Thu, 26 Apr 2018 10:52:03 +1200
changeset 788269 ee0f0bd263484954c9b4d82f31a94d03a5bfa8f8
parent 788123 ee36b000e3adae7acc7976372eeed1f1ca8db6ef
push id107932
push usernthomas@mozilla.com
push dateThu, 26 Apr 2018 03:07:14 +0000
reviewersaki
bugs1457010
milestone61.0a1
Bug 1457010 - gpg sign partner repacks, r?aki This adds repackage-signing on mac and linux, depending on repackage and the chunking-dummy kinds respectively, and repackage-signing is extended to create gpg signatures. The signing_dependencies are no longer added because the beetmover_repackage_partner.py transform is going to set that up manually, and it avoids duplicate targets which the schema blocks. Beetmover can depend now on repackage-signing for all platforms, and no longer has any indirect dependencies to worry about, but does need to know about copying the .asc files as upstream artifacts. MozReview-Commit-ID: JcIdXQ2B7Rg
taskcluster/ci/release-eme-free-repack-beetmover/kind.yml
taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml
taskcluster/ci/release-partner-repack-beetmover/kind.yml
taskcluster/ci/release-partner-repack-repackage-signing/kind.yml
taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
taskcluster/taskgraph/transforms/repackage_signing_partner.py
--- a/taskcluster/ci/release-eme-free-repack-beetmover/kind.yml
+++ b/taskcluster/ci/release-eme-free-repack-beetmover/kind.yml
@@ -6,18 +6,17 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.name_sanity:transforms
    - taskgraph.transforms.beetmover_repackage_partner:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - release-eme-free-repack-repackage  # Mac
-   - release-eme-free-repack-repackage-signing  # Windows
+   - release-eme-free-repack-repackage-signing
 
 only-for-build-platforms:
    - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
 
 job-template:
    shipping-phase: promote
--- a/taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml
+++ b/taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml
@@ -11,8 +11,9 @@ transforms:
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - release-eme-free-repack-repackage
 
 only-for-build-platforms:
    - win32-nightly/opt
    - win64-nightly/opt
+   - macosx64-nightly/opt
--- a/taskcluster/ci/release-partner-repack-beetmover/kind.yml
+++ b/taskcluster/ci/release-partner-repack-beetmover/kind.yml
@@ -6,19 +6,17 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.name_sanity:transforms
    - taskgraph.transforms.beetmover_repackage_partner:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - release-partner-repack-chunking-dummy  # Linux
-   - release-partner-repack-repackage  # Mac
-   - release-partner-repack-repackage-signing  # Windows
+   - release-partner-repack-repackage-signing
 
 only-for-build-platforms:
    - linux-nightly/opt
    - linux64-nightly/opt
    - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
 
--- a/taskcluster/ci/release-partner-repack-repackage-signing/kind.yml
+++ b/taskcluster/ci/release-partner-repack-repackage-signing/kind.yml
@@ -6,13 +6,17 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.name_sanity:transforms
    - taskgraph.transforms.repackage_signing_partner:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - release-partner-repack-repackage
+   - release-partner-repack-chunking-dummy  # Linux
+   - release-partner-repack-repackage  # Windows, Mac
 
 only-for-build-platforms:
+   - linux-nightly/opt
+   - linux64-nightly/opt
+   - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
--- a/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
@@ -75,43 +75,16 @@ def validate(config, jobs):
         label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
         validate_schema(
             beetmover_description_schema, job,
             "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
         yield job
 
 
 @transforms.add
-def skip_for_indirect_dependencies(config, jobs):
-    for job in jobs:
-        dep_job = job['dependent-task']
-        build_platform = dep_job.attributes.get("build_platform")
-        if not build_platform:
-            raise Exception("Cannot find build platform!")
-
-        # Partner and EME free beetmover tasks have multiple upstreams defined
-        # because some platforms don't run some parts of the sign -> repack ->
-        # repack sign chain. We only want to run beetmover for the last part of
-        # that chain that runs for any given platform.
-        # For Linux, it is the eme-free/partner repack build tasks.
-        # For Mac, it is repackage.
-        # For Windows, it is repackage-signing.
-        if "win" in build_platform:
-            if "repackage" not in dep_job.label:
-                continue
-            elif "signing" not in dep_job.label:
-                continue
-        if "macosx" in build_platform:
-            if "repackage" not in dep_job.label:
-                continue
-
-        yield job
-
-
-@transforms.add
 def resolve_keys(config, jobs):
     for job in jobs:
         resolve_keyed_by(
             job, 'partner-bucket-scope', item_name=job['label'], project=config.params['project']
         )
         yield job
 
 
@@ -145,20 +118,19 @@ def make_task_description(config, jobs):
         base_label = "release-partner-repack"
         if "eme" in config.kind:
             base_label = "release-eme-free-repack"
         dependencies["build"] = "{}-{}".format(base_label, build_platform)
         if "macosx" in build_platform or "win" in build_platform:
             dependencies["repackage"] = "{}-repackage-{}-{}".format(
                 base_label, build_platform, repack_id.replace('/', '-')
             )
-        if "win" in build_platform:
-            dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format(
-                base_label, build_platform, repack_id.replace('/', '-')
-            )
+        dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format(
+             base_label, build_platform, repack_id.replace('/', '-')
+        )
 
         attributes = copy_attributes_from_dependent_job(dep_job)
 
         task = {
             'label': label,
             'description': description,
             'dependencies': dependencies,
             'attributes': attributes,
@@ -216,30 +188,48 @@ def generate_upstream_artifacts(job, bui
 
     if "linux" in platform:
         upstream_artifacts.append({
             "taskId": {"task-reference": build_task_ref},
             "taskType": "build",
             "paths": ["{}/{}/target.tar.bz2".format(artifact_prefix, repack_id)],
             "locale": partner_path,
         })
+        upstream_artifacts.append({
+            "taskId": {"task-reference": repackage_signing_task_ref},
+            "taskType": "repackage",
+            "paths": ["{}/{}/target.tar.bz2.asc".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
     elif "macosx" in platform:
         upstream_artifacts.append({
             "taskId": {"task-reference": repackage_task_ref},
             "taskType": "repackage",
             "paths": ["{}/{}/target.dmg".format(artifact_prefix, repack_id)],
             "locale": partner_path,
         })
+        upstream_artifacts.append({
+            "taskId": {"task-reference": repackage_signing_task_ref},
+            "taskType": "repackage",
+            "paths": ["{}/{}/target.dmg.asc".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
     elif "win" in platform:
         upstream_artifacts.append({
             "taskId": {"task-reference": repackage_signing_task_ref},
             "taskType": "repackage",
             "paths": ["{}/{}/target.installer.exe".format(artifact_prefix, repack_id)],
             "locale": partner_path,
         })
+        upstream_artifacts.append({
+            "taskId": {"task-reference": repackage_signing_task_ref},
+            "taskType": "repackage",
+            "paths": ["{}/{}/target.installer.exe.asc".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
 
     if not upstream_artifacts:
         raise Exception("Couldn't find any upstream artifacts.")
 
     return upstream_artifacts
 
 
 @transforms.add
--- a/taskcluster/taskgraph/transforms/repackage_signing_partner.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing_partner.py
@@ -48,56 +48,78 @@ def validate(config, jobs):
 
 
 @transforms.add
 def make_repackage_signing_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
         repack_id = dep_job.task['extra']['repack_id']
         attributes = dep_job.attributes
+        build_platform = dep_job.attributes.get('build_platform')
+        is_nightly = dep_job.attributes.get('nightly')
 
+        # Mac & windows
         label = dep_job.label.replace("repackage-", "repackage-signing-")
+        # Linux
+        label = label.replace("chunking-dummy-", "repackage-signing-")
         description = (
             "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
             "{build_platform}/{build_type}'".format(
                 repack_id=repack_id,
                 build_platform=attributes.get('build_platform'),
                 build_type=attributes.get('build_type')
             )
         )
 
-        dependencies = {"repackage": dep_job.label}
+        if 'linux' in build_platform:
+            # we want the repack job, via the dependencies for the the chunking-dummy dep_job
+            for dep in dep_job.dependencies.values():
+                if dep.startswith('release-partner-repack'):
+                    dependencies = {"repack": dep}
+                    break
+        else:
+            # we have a genuine repackage job as our parent
+            dependencies = {"repackage": dep_job.label}
 
-        signing_dependencies = dep_job.dependencies
-        # This is so we get the build task etc in our dependencies to
-        # have better beetmover support.
-        dependencies.update({k: v for k, v in signing_dependencies.items()
-                             if k != 'docker-image'})
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['repackage_type'] = 'repackage-signing'
 
-        build_platform = dep_job.attributes.get('build_platform')
-        is_nightly = dep_job.attributes.get('nightly')
         signing_cert_scope = get_signing_cert_scope_per_platform(
             build_platform, is_nightly, config
         )
-        scopes = [signing_cert_scope]
-
-        if 'win' not in build_platform:
-            raise Exception("Repackage signing is not supported for non-Windows partner repacks.")
+        scopes = [signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg')]
 
-        upstream_artifacts = [{
-            "taskId": {"task-reference": "<repackage>"},
-            "taskType": "repackage",
-            "paths": [
-                get_artifact_path(dep_job, "{}/target.installer.exe".format(repack_id)),
-            ],
-            "formats": ["sha2signcode"]
-        }]
-        scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
+        if 'win' in build_platform:
+            upstream_artifacts = [{
+                "taskId": {"task-reference": "<repackage>"},
+                "taskType": "repackage",
+                "paths": [
+                    get_artifact_path(dep_job, "{}/target.installer.exe".format(repack_id)),
+                ],
+                "formats": ["sha2signcode", "gpg"]
+            }]
+            scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
+        elif 'mac' in build_platform:
+            upstream_artifacts = [{
+                "taskId": {"task-reference": "<repackage>"},
+                "taskType": "repackage",
+                "paths": [
+                    get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)),
+                ],
+                "formats": ["gpg"]
+            }]
+        elif 'linux' in build_platform:
+            upstream_artifacts = [{
+                "taskId": {"task-reference": "<repack>"},
+                "taskType": "repackage",
+                "paths": [
+                    get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)),
+                ],
+                "formats": ["gpg"]
+            }]
 
         task = {
             'label': label,
             'description': description,
             # 'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,