--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -10,18 +10,16 @@ from __future__ import absolute_import,
from taskgraph.transforms.base import (
validate_schema,
TransformSequence
)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Schema, Any, Required, Optional
-ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/public'
-
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
@@ -81,28 +79,35 @@ def make_task_description(config, jobs):
}
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
label = job.get('label', "balrog-{}".format(dep_job.label))
- parent_task_artifacts_url = {
- "task-reference": ARTIFACT_URL.format('<beetmover>')
- }
+ upstream_artifacts = [{
+ "taskId": {"task-reference": "<beetmover>"},
+ "taskType": "beetmover",
+ "paths": [
+ "public/manifest.json"
+ ],
+ }]
task = {
'label': label,
'description': "{} Balrog".format(
dep_job.task["metadata"]["description"]),
# do we have to define worker type somewhere?
'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
- 'worker': {'implementation': 'balrog',
- 'task_artifact_url': parent_task_artifacts_url},
- 'scopes': [],
+ 'worker': {
+ 'implementation': 'balrog',
+ 'upstream-artifacts': upstream_artifacts,
+ },
+ # bump this to nightly / release when applicable+permitted
+ 'scopes': ["project:releng:balrog:dep"],
'dependencies': {'beetmover': dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -10,16 +10,137 @@ from __future__ import absolute_import,
from taskgraph.transforms.base import (
validate_schema,
TransformSequence
)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Schema, Any, Required, Optional
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+ "balrog_props.json",
+ "target.common.tests.zip",
+ "target.cppunittest.tests.zip",
+ "target.crashreporter-symbols.zip",
+ "target.json",
+ "target.mochitest.tests.zip",
+ "target.mozinfo.json",
+ "target.reftest.tests.zip",
+ "target.talos.tests.zip",
+ "target.test_packages.json",
+ "target.txt",
+ "target.web-platform.tests.zip",
+ "target.xpcshell.tests.zip",
+ "target_info.txt",
+ "target.jsshell.zip",
+ "mozharness.zip",
+ "target.langpack.xpi",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+ "update/target.complete.mar",
+ "target.checksums",
+ "target.checksums.asc",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
+ "target.langpack.xpi",
+ "balrog_props.json",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
+ "update/target.complete.mar",
+ "target.checksums",
+ "target.checksums.asc",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+ "en-US/target.common.tests.zip",
+ "en-US/target.cppunittest.tests.zip",
+ "en-US/target.json",
+ "en-US/target.mochitest.tests.zip",
+ "en-US/target.mozinfo.json",
+ "en-US/target.reftest.tests.zip",
+ "en-US/target.talos.tests.zip",
+ "en-US/target.test_packages.json",
+ "en-US/target.txt",
+ "en-US/target.web-platform.tests.zip",
+ "en-US/target.xpcshell.tests.zip",
+ "en-US/target_info.txt",
+ "en-US/bouncer.apk",
+ "en-US/mozharness.zip",
+ "en-US/robocop.apk",
+ "en-US/target.jsshell.zip",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
+ "balrog_props.json",
+ "target.common.tests.zip",
+ "target.cppunittest.tests.zip",
+ "target.json",
+ "target.mochitest.tests.zip",
+ "target.mozinfo.json",
+ "target.reftest.tests.zip",
+ "target.talos.tests.zip",
+ "target.test_packages.json",
+ "target.txt",
+ "target.web-platform.tests.zip",
+ "target.xpcshell.tests.zip",
+ "target_info.txt",
+ "bouncer.apk",
+ "mozharness.zip",
+ "robocop.apk",
+ "target.jsshell.zip",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+ "en-US/target.apk",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_SIGNED_MULTI = [
+ "target.apk",
+]
+
+
+UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
+ 'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+ "target.sdk.tar.bz2"
+ ],
+ 'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+ "target.sdk.tar.bz2"
+ ],
+ 'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+ 'android-api-15-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+ 'macosx64-nightly': [],
+
+ 'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+ 'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+ 'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
+ 'android-api-15-nightly-l10n': ["balrog_props.json"],
+ 'android-api-15-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
+ 'macosx64-nightly-l10n': [],
+}
+UPSTREAM_ARTIFACT_SIGNED_PATHS = {
+ 'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+ "target.sdk.tar.bz2"
+ ],
+ 'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+ "target.sdk.tar.bz2"
+ ],
+ 'android-x86-nightly': ["en-US/target.apk"],
+ 'android-api-15-nightly': ["en-US/target.apk"],
+ 'macosx64-nightly': [],
+
+ 'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+ "target.tar.bz2",
+ "target.tar.bz2.asc",
+ ],
+ 'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+ "target.tar.bz2",
+ "target.tar.bz2.asc",
+ ],
+ 'android-x86-nightly-multi': ["target.apk"],
+ 'android-api-15-nightly-l10n': ["target.apk"],
+ 'android-api-15-nightly-multi': ["target.apk"],
+ 'macosx64-nightly-l10n': [],
+}
+
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
@@ -65,63 +186,148 @@ def make_task_description(config, jobs):
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 2)
treeherder.setdefault('kind', 'build')
label = job.get('label', "beetmover-{}".format(dep_job.label))
- # if dependent task is build, taskid_to_beetmove == taskid_of_manifest
- # and update_manifest = False
- # if dependent task is signed, the taskid_to_beetmove is the signed
- # build and update_manifest = True
- dependent_kind = str(job['dependent-task'].kind)
- # both artifacts are the build artifacts if not the signing task
- taskid_of_manifest = "<" + str(dependent_kind) + ">"
- taskid_to_beetmove = taskid_of_manifest
- update_manifest = False
- dependencies = {job['dependent-task'].kind: dep_job.label}
+ dependent_kind = str(dep_job.kind)
+ dependencies = {dependent_kind: dep_job.label}
# taskid_of_manifest always refers to the unsigned task
if "signing" in dependent_kind:
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"can't beetmove a signing task with multiple dependencies")
- dep_name = dep_job.dependencies.keys()[0]
- taskid_of_manifest = "<" + str(dep_name) + ">"
- update_manifest = True
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
treeherder.setdefault('symbol', 'tc(BM-S)')
else:
treeherder.setdefault('symbol', 'tc(BM)')
- worker = {'implementation': 'beetmover',
- 'taskid_to_beetmove': {"task-reference":
- taskid_to_beetmove},
- 'taskid_of_manifest': {"task-reference":
- taskid_of_manifest},
- 'update_manifest': update_manifest}
-
attributes = {
'nightly': dep_job.attributes.get('nightly', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
}
if job.get('locale'):
- worker['locale'] = job['locale']
attributes['locale'] = job['locale']
task = {
'label': label,
'description': "{} Beetmover".format(
dep_job.task["metadata"]["description"]),
# do we have to define worker type somewhere?
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
- 'worker': worker,
- 'scopes': [],
+ # bump this to nightly / release when applicable+permitted
+ 'scopes': ["project:releng:beetmover:dep"],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
+
+
+def generate_upstream_artifacts(taskid_to_beetmove, platform, locale=None, signing=False):
+ task_type = "build"
+ mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
+ if signing:
+ task_type = "signing"
+ mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
+
+ artifact_prefix = 'public/build'
+ if locale:
+ artifact_prefix = 'public/build/{}'.format(locale)
+ platform = "{}-l10n".format(platform)
+
+ upstream_artifacts = [{
+ "taskId": {"task-reference": taskid_to_beetmove},
+ "taskType": task_type,
+ "paths": ["{}/{}".format(artifact_prefix, p) for p in mapping[platform]],
+ "locale": locale or "en-US",
+ }]
+ if not locale and "android" in platform:
+ # edge case to support 'multi' locale paths
+ multi_platform = "{}-multi".format(platform)
+ upstream_artifacts.append({
+ "taskId": {"task-reference": taskid_to_beetmove},
+ "taskType": task_type,
+ "paths": ["{}/{}".format(artifact_prefix, p) for p in mapping[multi_platform]],
+ "locale": "multi",
+ })
+
+ return upstream_artifacts
+
+
+def generate_signing_upstream_artifacts(taskid_to_beetmove, taskid_of_manifest, platform,
+ locale=None):
+ upstream_artifacts = generate_upstream_artifacts(taskid_to_beetmove, platform, locale,
+ signing=True)
+ if locale:
+ artifact_prefix = 'public/build/{}'.format(locale)
+ else:
+ artifact_prefix = 'public/build'
+ manifest_path = "{}/balrog_props.json".format(artifact_prefix)
+ upstream_artifacts.append({
+ "taskId": {"task-reference": taskid_of_manifest},
+ "taskType": "signing",
+ "paths": [manifest_path],
+ "locale": locale or "en-US",
+ })
+
+ return upstream_artifacts
+
+
+def generate_build_upstream_artifacts(taskid_to_beetmove, platform, locale=None):
+ upstream_artifacts = generate_upstream_artifacts(taskid_to_beetmove, platform, locale,
+ signing=False)
+ return upstream_artifacts
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+ for job in jobs:
+ valid_beetmover_signing_job = (len(job["dependencies"]) == 2 and
+ any(['signing' in j for j in job['dependencies']]))
+ valid_beetmover_build_job = len(job["dependencies"]) == 1
+ if not valid_beetmover_build_job and not valid_beetmover_signing_job:
+ raise NotImplementedError(
+ "beetmover tasks must have either 1 or 2 dependencies. "
+ "If 2, one of those must be a signing task"
+ )
+
+ build_kind = None
+ signing_kind = None
+ locale = job["attributes"].get("locale")
+ platform = job["attributes"]["build_platform"]
+ for dependency in job["dependencies"].keys():
+ if 'signing' in dependency:
+ signing_kind = dependency
+ else:
+ build_kind = dependency
+
+ if signing_kind:
+ taskid_to_beetmove = "<" + str(signing_kind) + ">"
+ taskid_of_manifest = "<" + str(build_kind) + ">"
+ update_manifest = True
+ upstream_artifacts = generate_signing_upstream_artifacts(
+ taskid_to_beetmove, taskid_of_manifest, platform, locale
+ )
+ else:
+ taskid_to_beetmove = "<" + str(build_kind) + ">"
+ update_manifest = False
+ upstream_artifacts = generate_build_upstream_artifacts(
+ taskid_to_beetmove, platform, locale
+ )
+
+ worker = {'implementation': 'beetmover',
+ 'update_manifest': update_manifest,
+ 'upstream-artifacts': upstream_artifacts}
+
+ if locale:
+ worker["locale"] = locale
+
+ job["worker"] = worker
+
+ yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -287,31 +287,50 @@ task_description_schema = Schema({
}],
}, {
Required('implementation'): 'beetmover',
# the maximum time to spend signing, in seconds
Required('max-run-time', default=600): int,
# taskid of task with artifacts to beetmove
- Required('taskid_to_beetmove'): taskref_or_string,
-
- # taskid of task with artifacts to beetmove
- Required('taskid_of_manifest'): taskref_or_string,
-
# beetmover template key
Required('update_manifest'): bool,
# locale key, if this is a locale beetmover job
Optional('locale'): basestring,
+
+ # list of artifact URLs for the artifacts that should be beetmoved
+ Required('upstream-artifacts'): [{
+ # taskId of the task with the artifact
+ Required('taskId'): taskref_or_string,
+
+ # type of signing task (for CoT)
+ Required('taskType'): basestring,
+
+ # Paths to the artifacts to sign
+ Required('paths'): [basestring],
+
+ # locale is used to map upload path and allow for duplicate simple names
+ Required('locale'): basestring,
+ }],
}, {
Required('implementation'): 'balrog',
- # taskid of the signed beetmoved task
- Required('task_artifact_url'): taskref_or_string,
+ # list of artifact URLs for the artifacts that should be beetmoved
+ Required('upstream-artifacts'): [{
+ # taskId of the task with the artifact
+ Required('taskId'): taskref_or_string,
+
+ # type of signing task (for CoT)
+ Required('taskType'): basestring,
+
+ # Paths to the artifacts to sign
+ Required('paths'): [basestring],
+ }],
}),
# The "when" section contains descriptions of the circumstances
# under which this task can be "optimized", that is, left out of the
# task graph because it is unnecessary.
Optional('when'): Any({
# This task only needs to be run if a file matching one of the given
# patterns has changed in the push. The patterns use the mozpack
@@ -510,32 +529,31 @@ def build_scriptworker_signing_payload(c
@payload_builder('beetmover')
def build_beetmover_payload(config, task, task_def):
worker = task['worker']
task_def['payload'] = {
'maxRunTime': worker['max-run-time'],
'upload_date': config.params['build_date'],
- 'taskid_to_beetmove': worker['taskid_to_beetmove'],
- 'taskid_of_manifest': worker['taskid_of_manifest'],
'update_manifest': worker['update_manifest'],
+ 'upstreamArtifacts': worker['upstream-artifacts']
}
if worker.get('locale'):
task_def['payload']['locale'] = worker['locale']
@payload_builder('balrog')
def build_balrog_payload(config, task, task_def):
worker = task['worker']
task_def['payload'] = {
- 'parent_task_artifacts_url': worker['task_artifact_url'],
# signing cert is unused, but required by balrogworker (Bug 1282187 c#7)
'signing_cert': "dep",
+ 'upstreamArtifacts': worker['upstream-artifacts']
}
@payload_builder('macosx-engine')
def build_macosx_engine_payload(config, task, task_def):
worker = task['worker']
artifacts = map(lambda artifact: {
'name': artifact['name'],