bug 1423081 - source readme. r=callek draft
authorAki Sasaki <asasaki@mozilla.com>
Tue, 05 Dec 2017 19:10:01 -0800
changeset 711350 c929fd7a219a5d4cc2dd570cc50dff0322df0467
parent 711349 450dee0302ebddd4e1a171f4b25f5d4d93578f29
child 711351 f3bc17f72774922d1e9b3786dbf46f63b3a6e42b
push id93059
push userasasaki@mozilla.com
push dateWed, 13 Dec 2017 21:20:02 +0000
reviewerscallek
bugs1423081, 749312
milestone59.0a1
bug 1423081 - source readme. r=callek In bug 749312, we were given permission to create a source readme instead of a source tarball. This will save us cycles, disk, and human configuration time. We still need to address the missing balrog_props.json for beetmover-source for that task to turn green. MozReview-Commit-ID: wnyPoNXCsH
taskcluster/ci/beetmover-source/kind.yml
taskcluster/ci/release-source-signing/kind.yml
taskcluster/ci/release-source/kind.yml
taskcluster/ci/release-source/source.yml
taskcluster/taskgraph/loader/build_signing.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_source.py
taskcluster/taskgraph/transforms/build_signing.py
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/util/signed_artifacts.py
testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
testing/mozharness/scripts/fx_source.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-source/kind.yml
@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.beetmover:transforms
+   - taskgraph.transforms.beetmover_source:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-source-signing
+
+job-template:
+   shipping-phase: promote
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-source-signing/kind.yml
@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.build_signing:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.build_signing:transforms
+   - taskgraph.transforms.signing:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-source
+
+job-template:
+   shipping-phase: promote
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-source/kind.yml
@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+   - toolchain
+
+transforms:
+   - taskgraph.transforms.build:transforms
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.build_lints:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+jobs-from:
+   - source.yml
+
+job-defaults:
+   shipping-phase: promote
+   notifications:
+      completed:
+         subject: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         message: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         plugins:
+            by-project:
+               mozilla-beta: ["log_collect"]
+               mozilla-release: ["log_collect"]
+               default: []
+
+      failed:
+         subject: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         message: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         plugins:
+            by-project:
+               mozilla-beta: ["log_collect", "ses"]
+               mozilla-release: ["log_collect", "ses"]
+               default: ["ses"]
+         emails:
+            by-project:
+               mozilla-beta: ["release-automation-notifications@mozilla.com"]
+               mozilla-release: ["release-automation-notifications@mozilla.com"]
+               try: ["{task_def[metadata][owner]}"]
+               maple: ["release+tcstaging@mozilla.com"]
+               default: []
+
+      exception:
+         subject: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         message: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         plugins:
+            by-project:
+               mozilla-beta: ["log_collect", "ses"]
+               mozilla-release: ["log_collect", "ses"]
+               default: ["ses"]
+         emails:
+            by-project:
+               mozilla-beta: ["release-automation-notifications@mozilla.com"]
+               mozilla-release: ["release-automation-notifications@mozilla.com"]
+               try: ["{task_def[metadata][owner]}"]
+               maple: ["release+tcstaging@mozilla.com"]
+               default: []
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-source/source.yml
@@ -0,0 +1,62 @@
+linux64-source/opt:
+    description: "Firefox source builder"
+    index:
+        product: firefox
+        job-name: linux64-source-opt
+    treeherder:
+        platform: linux64/opt
+        symbol: tc(Src)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [build]
+        config:
+            - builds/releng_sub_linux_configs/64_source.py
+        options: []
+        script: "mozharness/scripts/fx_source.py"
+    run-on-projects: []
+    shipping-product: firefox
+
+linux64-fennec-source/opt:
+    description: "Fennec source builder"
+    index:
+        product: fennec
+        job-name: linux64-fennec-source-opt
+    treeherder:
+        platform: android-api-16/opt
+        symbol: tc(Src)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [build]
+        config:
+            - builds/releng_sub_linux_configs/64_source.py
+        options: []
+        script: "mozharness/scripts/fx_source.py"
+    run-on-projects: []
+    shipping-product: fennec
+
+linux64-devedition-source/opt:
+    description: "Devedition source builder"
+    index:
+        product: devedition
+        job-name: linux64-devedition-source-opt
+    treeherder:
+        platform: linux64-devedition-source/opt
+        symbol: tc(Src)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [build]
+        config:
+            - builds/releng_sub_linux_configs/64_source.py
+        options: []
+        script: "mozharness/scripts/fx_source.py"
+    run-on-projects: []
+    shipping-product: devedition
--- a/taskcluster/taskgraph/loader/build_signing.py
+++ b/taskcluster/taskgraph/loader/build_signing.py
@@ -7,16 +7,19 @@ from __future__ import absolute_import, 
 from taskgraph.loader.single_dep import loader as base_loader
 
 # XXX: This logic should rely in kind.yml. This hasn't been done in the original
 # patch because it required some heavy changes in single_dep.
 LABELS_WHICH_SHOULD_SIGN_CI_BUILDS = (
     'build-win32/debug', 'build-win32/opt', 'build-win32/pgo',
     'build-win64/debug', 'build-win64/opt', 'build-win64/pgo',
     'build-win32-devedition/opt', 'build-win64-devedition/opt',
+    'release-source-linux64-source/opt',
+    'release-source-linux64-fennec-source/opt',
+    'release-source-linux64-devedition-source/opt',
 )
 
 
 def loader(kind, path, config, params, loaded_tasks):
     jobs = base_loader(kind, path, config, params, loaded_tasks)
 
     for job in jobs:
         dependent_task = job['dependent-task']
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -141,16 +141,22 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
     ],
     'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
     ],
+    'linux64-source': [
+    ],
+    'linux64-devedition-source': [
+    ],
+    'linux64-fennec-source': [
+    ],
     'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-aarch64-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-16-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-x86-old-id-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-16-old-id-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
@@ -203,16 +209,28 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
+    'linux64-source': [
+        "SOURCE",
+        "SOURCE.asc",
+    ],
+    'linux64-devedition-source': [
+        "SOURCE",
+        "SOURCE.asc",
+    ],
+    'linux64-fennec-source': [
+        "SOURCE",
+        "SOURCE.asc",
+    ],
     'android-x86-nightly': ["en-US/target.apk"],
     'android-aarch64-nightly': ["en-US/target.apk"],
     'android-api-16-nightly': ["en-US/target.apk"],
     'android-x86-old-id-nightly': ["en-US/target.apk"],
     'android-api-16-old-id-nightly': ["en-US/target.apk"],
     'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.dmg",
         "target.dmg.asc",
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_source.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover-source task to also append `build` as dependency
+"""
+from __future__ import absolute_import
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def tweak_beetmover_source_dependencies_and_upstream_artifacts(config, jobs):
+    for job in jobs:
+        # HACK1: instead of grabbing SOURCE file from `release-source` task, we
+        # instead take it along with SOURCE.asc directly from the
+        # `release-source-signing`.
+        #
+        # HACK2: This way, we can just overwrite the `build`
+        # dependency, which at this point still is `release-source` task, with
+        # the actual Nightly en-US linux64 build which contains the
+        # `balrog_props` file we're interested in.
+        #
+        # XXX: this hack should go away by either:
+        # * rewriting beetmover transforms to allow more flexibility in deps
+        # * ditch balrog_props in beetmover and rely on in-tree task payload
+
+        if job['attributes']['shipping_product'] == 'firefox':
+            job['dependencies']['build'] = u'build-linux64-nightly/opt'
+        elif job['attributes']['shipping_product'] == 'fennec':
+            job['dependencies']['build'] = u'build-android-api-16-nightly/opt'
+        elif job['attributes']['shipping_product'] == 'devedition':
+            job['dependencies']['build'] = u'build-linux64-devedition-nightly/opt'
+        else:
+            raise NotImplemented(
+                "Unknown shipping_product {} for beetmover_source!".format(
+                    job['attributes']['shipping_product']
+                )
+            )
+        upstream_artifacts = job['worker']['upstream-artifacts']
+        for artifact in upstream_artifacts:
+            if artifact['taskType'] == 'build':
+                artifact['paths'].append(u'public/build/balrog_props.json')
+                break
+
+        yield job
--- a/taskcluster/taskgraph/transforms/build_signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -39,17 +39,18 @@ def add_signed_routes(config, jobs):
 def define_upstream_artifacts(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
         build_platform = dep_job.attributes.get('build_platform')
 
         artifacts_specifications = generate_specifications_of_artifacts_to_sign(
             build_platform,
             dep_job.attributes.get('nightly'),
-            keep_locale_template=False
+            keep_locale_template=False,
+            kind=config.kind,
         )
 
         if 'android' in build_platform:
             # We're in the job that creates both multilocale and en-US APKs
             artifacts_specifications[0]['artifacts'].append('public/build/en-US/target.apk')
 
         job['upstream-artifacts'] = [{
             'taskId': {'task-reference': '<build>'},
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -26,16 +26,17 @@ JOB_NAME_WHITELIST = set([
     'android-findbugs',
     'android-lint',
     'android-test',
     'android-x86-old-id-opt',
     'android-x86-opt',
     'browser-haz-debug',
     'linux-debug',
     'linux-devedition',
+    'linux-devedition-opt',
     'linux-devedition-nightly-repackage',
     'linux-devedition-nightly-repackage-signing',
     'linux-nightly-repackage',
     'linux-nightly-repackage-signing',
     'linux-opt',
     'linux-pgo',
     'linux-rusttests-opt',
     'linux-rusttests-debug',
@@ -47,34 +48,39 @@ JOB_NAME_WHITELIST = set([
     'linux64-base-toolchains-debug',
     'linux64-base-toolchains-opt',
     'linux64-fuzzing-asan-opt',
     'linux64-fuzzing-debug',
     'linux64-ccov-opt',
     'linux64-clang-tidy',
     'linux64-debug',
     'linux64-devedition',
+    'linux64-devedition-opt',
     'linux64-devedition-nightly-repackage',
     'linux64-devedition-nightly-repackage-signing',
     'linux64-jsdcov-opt',
     'linux64-nightly-repackage',
     'linux64-nightly-repackage-signing',
     'linux64-noopt-debug',
     'linux64-opt',
     'linux64-pgo',
     'linux64-rusttests-opt',
     'linux64-rusttests-debug',
     'linux64-searchfox-debug',
     'linux64-st-an-debug',
     'linux64-st-an-opt',
     'linux64-valgrind-opt',
     'linux64-dmd-opt',
+    'linux64-source-opt',
+    'linux64-devedition-source-opt',
+    'linux64-fennec-source-opt',
     'macosx64-add-on-devel',
     'macosx64-clang-tidy',
     'macosx64-debug',
+    'macosx64-devedition-opt',
     'macosx64-nightly-repackage',
     'macosx64-nightly-repackage-signing',
     'macosx64-noopt-debug',
     'macosx64-opt',
     'macosx64-devedition-nightly-repackage',
     'macosx64-devedition-nightly-repackage-signing',
     'macosx64-st-an-debug',
     'macosx64-st-an-opt',
--- a/taskcluster/taskgraph/util/signed_artifacts.py
+++ b/taskcluster/taskgraph/util/signed_artifacts.py
@@ -4,19 +4,26 @@
 """
 Defines artifacts to sign before repackage.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 
 def generate_specifications_of_artifacts_to_sign(
-    build_platform, is_nightly=False, keep_locale_template=True
+    build_platform, is_nightly=False, keep_locale_template=True, kind=None
 ):
-    if 'android' in build_platform:
+    if kind == 'release-source-signing':
+        artifacts_specifications = [{
+            'artifacts': [
+                'public/build/SOURCE'
+            ],
+            'formats': ['gpg'],
+        }]
+    elif 'android' in build_platform:
         artifacts_specifications = [{
             'artifacts': [
                 'public/build/{locale}/target.apk',
             ],
             'formats': ['jar'],
         }]
     # XXX: Mars aren't signed here (on any platform) because internals will be
     # signed at after this stage of the release
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
@@ -1,19 +1,21 @@
+import os
+
 config = {
-    'default_actions': [
-        'clobber',
-        'clone-tools',
-        'checkout-sources',
-        'setup-mock',
-        'package-source',
-        'generate-source-signing-manifest',
-    ],
-    'stage_platform': 'source',  # Not used, but required by the script
-    'buildbot_json_path': 'buildprops.json',
-    'app_ini_path': 'FAKE',  # Not used, but required by the script
-    'env': {
-        'MOZ_OBJDIR': '%(abs_obj_dir)s',
-        'TINDERBOX_OUTPUT': '1',
-        'LC_ALL': 'C',
-    },
-    'mozconfig_variant': 'source',
+    "tmpl": """The binaries in this directory are made available to you under the Mozilla Public License v.2
+(MPL 2):
+http://www.mozilla.org/MPL/2.0/
+
+The source code used to build these binaries is specified by the following unique URL:
+{repo}/rev/{revision}
+
+    zip: {repo}/archive/{revision}.zip
+    tar.gz: {repo}/archive/{revision}.tar.gz
+    tar.bz2: {repo}/archive/{revision}.tar.bz2
+
+Instructions for downloading and building this source code can be found here:
+https://developer.mozilla.org/en/Mozilla_Source_Code_%28Mercurial%29
+""",
+    "out_path": "artifacts/SOURCE",
+    "repo": os.environ["GECKO_HEAD_REPOSITORY"],
+    "revision": os.environ["GECKO_HEAD_REV"],
 }
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/fx_source.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""fx_source.py.
+
+Create a source readme.
+
+"""
+import sys
+import os
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.script import BaseScript
+from mozharness.base.log import FATAL
+
+
+class FxSource(BaseScript):
+    config_options = [
+        [["--repo"], {
+            "dest": "repo",
+            "help": "the URL to the source repo",
+        }],
+        [["--revision"], {
+            "dest": "revision",
+            "help": "the revision of the source",
+        }],
+        [["--disable-mock"], {
+            "dest": "disable_mock",
+            "action": "store_true",
+            "help": "dummy option",
+        }],
+        [["--scm-level"], {
+            "dest": "scm_level",
+            "help": "dummy option",
+        }],
+        [["--branch"], {
+            "dest": "branch",
+            "help": "dummy option",
+        }],
+        [["--build-pool"], {
+            "dest": "build_pool",
+            "help": "dummy option",
+        }],
+    ]
+
+    def __init__(self):
+        buildscript_kwargs = {
+            'config_options': self.config_options,
+            'all_actions': [
+                'build',
+            ],
+            'require_config_file': True,
+        }
+        super(FxSource, self).__init__(**buildscript_kwargs)
+
+    def build(self):
+        c = self.config
+        contents = c['tmpl'].format(**c)
+        self.write_to_file(c['out_path'], contents, create_parent_dir=True,
+                           error_level=FATAL)
+
+
+if __name__ == '__main__':
+    fx_source = FxSource()
+    fx_source.run_and_exit()