--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -17,17 +17,16 @@ from taskgraph.transforms.task import ta
from voluptuous import Any, Required, Optional
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
- "balrog_props.json",
"target.common.tests.zip",
"target.cppunittest.tests.zip",
"target.crashreporter-symbols.zip",
"target.json",
"target.mochitest.tests.zip",
"target.mozinfo.json",
"target.reftest.tests.zip",
"target.talos.tests.zip",
@@ -50,17 +49,16 @@ from voluptuous import Any, Required, Op
"update/target.complete.mar",
]
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
"target.langpack.xpi",
- "balrog_props.json",
]
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
"target.complete.mar",
]
@@ -87,17 +85,16 @@ from voluptuous import Any, Required, Op
"en-US/robocop.apk",
"en-US/target.jsshell.zip",
]
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
_MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
- "balrog_props.json",
"target.common.tests.zip",
"target.cppunittest.tests.zip",
"target.json",
"target.mochitest.tests.zip",
"target.mozinfo.json",
"target.reftest.tests.zip",
"target.talos.tests.zip",
"target.awsy.tests.zip",
@@ -184,17 +181,17 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
],
'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'linux64-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'linux-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
'android-x86-old-id-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
'android-aarch64-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
- 'android-api-16-nightly-l10n': ["balrog_props.json"],
+ 'android-api-16-nightly-l10n': [],
'android-api-16-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
'android-api-16-old-id-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'macosx64-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'win32-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'win32-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'win64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'win64-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
@@ -442,16 +439,39 @@ def generate_upstream_artifacts(signing_
"paths": ["{}/{}".format(artifact_prefix, p)
for p in signing_mapping[multi_platform]],
"locale": "multi",
}])
return upstream_artifacts
+def craft_release_properties(config, job):
+ params = config.params
+ build_platform = job['attributes']['build_platform']
+ build_platform = build_platform.replace('-nightly', '')
+ if 'fennec-source' in build_platform:
+ # XXX This case is hardcoded to match the current implementation in beetmover
+ build_platform = 'android-api-16'
+ else:
+ build_platform = build_platform.replace('-source', '')
+
+ app_name = 'Fennec' if 'android' in job['label'] or 'fennec' in job['label'] else 'Firefox'
+
+ return {
+ # XXX Even DevEdition is called Firefox
+ 'app-name': app_name,
+ 'app-version': str(params['app_version']),
+ 'branch': params['project'],
+ 'build-id': str(params['moz_build_date']),
+ 'hash-type': 'sha512',
+ 'platform': build_platform,
+ }
+
+
@transforms.add
def make_task_worker(config, jobs):
for job in jobs:
valid_beetmover_job = (len(job["dependencies"]) == 2 and
any(['signing' in j for j in job['dependencies']]))
if not valid_beetmover_job:
raise NotImplementedError("Beetmover must have two dependencies.")
@@ -462,19 +482,22 @@ def make_task_worker(config, jobs):
for dependency in job["dependencies"].keys():
if 'signing' in dependency:
signing_task = dependency
else:
build_task = dependency
signing_task_ref = "<" + str(signing_task) + ">"
build_task_ref = "<" + str(build_task) + ">"
- upstream_artifacts = generate_upstream_artifacts(
- signing_task_ref, build_task_ref, platform, locale
- )
- worker = {'implementation': 'beetmover',
- 'upstream-artifacts': upstream_artifacts}
+ worker = {
+ 'implementation': 'beetmover',
+ 'release-properties': craft_release_properties(config, job),
+ 'upstream-artifacts': generate_upstream_artifacts(
+ signing_task_ref, build_task_ref, platform, locale
+ )
+ }
+
if locale:
worker["locale"] = locale
job["worker"] = worker
yield job
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the checksums signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -127,21 +128,16 @@ def generate_upstream_artifacts(refs, pl
"public/target.checksums.asc",
]
upstream_artifacts = [{
"taskId": {"task-reference": refs["signing"]},
"taskType": "signing",
"paths": common_paths,
"locale": locale or "en-US",
- }, {
- "taskId": {"task-reference": refs["beetmover"]},
- "taskType": "beetmover",
- "paths": ["public/balrog_props.json"],
- "locale": locale or "en-US",
}]
return upstream_artifacts
@transforms.add
def make_beetmover_checksums_worker(config, jobs):
for job in jobs:
@@ -160,18 +156,21 @@ def make_beetmover_checksums_worker(conf
if dependency.startswith("beetmover"):
refs['beetmover'] = "<{}>".format(dependency)
else:
refs['signing'] = "<{}>".format(dependency)
if None in refs.values():
raise NotImplementedError(
"Beetmover checksums must have a beetmover and signing dependency!")
- upstream_artifacts = generate_upstream_artifacts(refs,
- platform, locale)
+ worker = {
+ 'implementation': 'beetmover',
+ 'release-properties': craft_release_properties(config, job),
+ 'upstream-artifacts': generate_upstream_artifacts(
+ refs, platform, locale
+ ),
+ }
- worker = {'implementation': 'beetmover',
- 'upstream-artifacts': upstream_artifacts}
if locale:
worker["locale"] = locale
job["worker"] = worker
yield job
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partials import (get_balrog_platform_name,
get_partials_artifacts,
get_partials_artifact_map)
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_phase)
@@ -32,17 +33,16 @@ logger = logging.getLogger(__name__)
'win32-devedition-nightly',
]
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
- "balrog_props.json",
"target.common.tests.zip",
"target.cppunittest.tests.zip",
"target.crashreporter-symbols.zip",
"target.json",
"target.mochitest.tests.zip",
"target.mozinfo.json",
"target.reftest.tests.zip",
"target.talos.tests.zip",
@@ -58,17 +58,16 @@ logger = logging.getLogger(__name__)
]
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
"target.langpack.xpi",
- "balrog_props.json",
]
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
r'^(linux(|64)|macosx64)(|-devedition)-nightly$':
@@ -366,23 +365,25 @@ def make_task_worker(config, jobs):
build_signing_task = dependency
else:
build_task = "build"
build_task_ref = "<" + str(build_task) + ">"
build_signing_task_ref = "<" + str(build_signing_task) + ">"
repackage_task_ref = "<" + str(repackage_task) + ">"
repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
- upstream_artifacts = generate_upstream_artifacts(
- build_task_ref, build_signing_task_ref, repackage_task_ref,
- repackage_signing_task_ref, platform, locale
- )
- worker = {'implementation': 'beetmover',
- 'upstream-artifacts': upstream_artifacts}
+ worker = {
+ 'implementation': 'beetmover',
+ 'release-properties': craft_release_properties(config, job),
+ 'upstream-artifacts': generate_upstream_artifacts(
+ build_task_ref, build_signing_task_ref, repackage_task_ref,
+ repackage_signing_task_ref, platform, locale
+ ),
+ }
if locale:
worker["locale"] = locale
job["worker"] = worker
yield job
@transforms.add
--- a/taskcluster/taskgraph/transforms/beetmover_source.py
+++ b/taskcluster/taskgraph/transforms/beetmover_source.py
@@ -7,42 +7,24 @@ Transform the beetmover-source task to a
from __future__ import absolute_import
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
@transforms.add
-def tweak_beetmover_source_dependencies_and_upstream_artifacts(config, jobs):
+def remove_build_dependency_in_beetmover_source(config, jobs):
for job in jobs:
- # HACK1: instead of grabbing SOURCE file from `release-source` task, we
- # instead take it along with SOURCE.asc directly from the
- # `release-source-signing`.
- #
- # HACK2: This way, we can just overwrite the `build`
- # dependency, which at this point still is `release-source` task, with
- # the actual Nightly en-US linux64 build which contains the
- # `balrog_props` file we're interested in.
- #
- # XXX: this hack should go away by either:
- # * rewriting beetmover transforms to allow more flexibility in deps
- # * ditch balrog_props in beetmover and rely on in-tree task payload
+ # XXX: We delete the build dependency because, unlike the other beetmover
+ # tasks, source doesn't depend on any build task at all. This hack should
+ # go away when we rewrite beetmover transforms to allow more flexibility in deps
+ del job['dependencies']['build']
- if job['attributes']['shipping_product'] == 'firefox':
- job['dependencies']['build'] = u'build-linux64-nightly/opt'
- elif job['attributes']['shipping_product'] == 'fennec':
- job['dependencies']['build'] = u'build-android-api-16-nightly/opt'
- elif job['attributes']['shipping_product'] == 'devedition':
- job['dependencies']['build'] = u'build-linux64-devedition-nightly/opt'
- else:
- raise NotImplemented(
- "Unknown shipping_product {} for beetmover_source!".format(
- job['attributes']['shipping_product']
- )
- )
- upstream_artifacts = job['worker']['upstream-artifacts']
- for artifact in upstream_artifacts:
- if artifact['taskType'] == 'build':
- artifact['paths'].append(u'public/build/balrog_props.json')
- break
+ all_upstream_artifacts = job['worker']['upstream-artifacts']
+ upstream_artifacts_without_build = [
+ upstream_artifact
+ for upstream_artifact in all_upstream_artifacts
+ if upstream_artifact['taskId']['task-reference'] != '<build>'
+ ]
+ job['worker']['upstream-artifacts'] = upstream_artifacts_without_build
yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -508,16 +508,25 @@ task_description_schema = Schema({
Required('implementation'): 'beetmover',
# the maximum time to run, in seconds
Required('max-run-time'): int,
# locale key, if this is a locale beetmover job
Optional('locale'): basestring,
+ Required('release-properties'): {
+ 'app-name': basestring,
+ 'app-version': basestring,
+ 'branch': basestring,
+ 'build-id': basestring,
+ 'hash-type': basestring,
+ 'platform': basestring,
+ },
+
# list of artifact URLs for the artifacts that should be beetmoved
Required('upstream-artifacts'): [{
# taskId of the task with the artifact
Required('taskId'): taskref_or_string,
# type of signing task (for CoT)
Required('taskType'): basestring,
@@ -1008,21 +1017,30 @@ def build_binary_transparency_payload(co
),
}
@payload_builder('beetmover')
def build_beetmover_payload(config, task, task_def):
worker = task['worker']
release_config = get_release_config(config)
+ release_properties = worker['release-properties']
task_def['payload'] = {
'maxRunTime': worker['max-run-time'],
+ 'releaseProperties': {
+ 'appName': release_properties['app-name'],
+ 'appVersion': release_properties['app-version'],
+ 'branch': release_properties['branch'],
+ 'buildid': release_properties['build-id'],
+ 'hashType': release_properties['hash-type'],
+ 'platform': release_properties['platform'],
+ },
'upload_date': config.params['build_date'],
- 'upstreamArtifacts': worker['upstream-artifacts']
+ 'upstreamArtifacts': worker['upstream-artifacts'],
}
if worker.get('locale'):
task_def['payload']['locale'] = worker['locale']
if release_config:
task_def['payload'].update(release_config)
@payload_builder('beetmover-cdns')