Bug 1171738 - Migrate Linux64, Linux32, and Android l10n desktop nightly repacks to TaskCluster. f=dustin
Make sure EN_US_BINARY_URL is settable by environment to allow taskcluster to pass in a specific location.
Also for now depend on *unsigned* builds so that android does not report the apk as corrupt during repack steps.
MozReview-Commit-ID: H7LzRTktoiN
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -0,0 +1,124 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+ - taskgraph.transforms.l10n:transforms
+ - taskgraph.transforms.build_attrs:transforms
+ - taskgraph.transforms.job:transforms
+ - taskgraph.transforms.task:transforms
+
+job-defaults:
+ index:
+ product: firefox
+ treeherder:
+ kind: build
+ tier: 2
+ symbol: tc-L10n(N)
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ docker-image: {in-tree: desktop-build}
+ max-run-time: 36000
+ # don't run anywhere by default, but still used from nightlies
+ run-on-projects: []
+ run:
+ # NOTE: this should really be a different "using" since it's using
+ # build-l10n.sh instead of build-linux.sh. Preferably, build-linux.sh
+ # and the mozharness run implementation should be modified to support
+ # the functionality that l10n needs
+ using: mozharness
+ job-script: taskcluster/scripts/builder/build-l10n.sh
+ chunks: 6
+
+jobs:
+ # Names correspond to the <platform> nightly label (as used in signing)
+ # In order to automatically set the dependency.
+ linux-nightly/opt:
+ description: "Localization"
+ attributes:
+ build_platform: linux64-nightly
+ nightly: true
+ index:
+ job-name:
+ gecko-v2: linux64-nightly-l10n-opt
+ treeherder:
+ platform: linux/opt
+ worker:
+ env:
+ EN_US_PACKAGE_NAME: target.tar.bz2
+ EN_US_BINARY_URL:
+ task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build
+ run:
+ script: mozharness/scripts/desktop_l10n.py
+ actions: [clone-locales list-locales setup repack summary]
+ config:
+ - single_locale/tc_linux64.py
+ options:
+ - environment-config=single_locale/production.py
+ - branch-config=single_locale/{project}.py
+ - platform-config=single_locale/linux64.py
+ tooltool-downloads: public
+ need-xvfb: true
+
+ linux64-nightly/opt:
+ description: "Localization"
+ attributes:
+ build_platform: linux64-nightly
+ nightly: true
+ index:
+ job-name:
+ gecko-v2: linux64-nightly-l10n-opt
+ treeherder:
+ platform: linux64/opt
+ worker:
+ env:
+ EN_US_PACKAGE_NAME: target.tar.bz2
+ EN_US_BINARY_URL:
+ task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build
+ run:
+ script: mozharness/scripts/desktop_l10n.py
+ actions: [clone-locales list-locales setup repack summary]
+ config:
+ - single_locale/tc_linux64.py
+ options:
+ - environment-config=single_locale/production.py
+ - branch-config=single_locale/{project}.py
+ - platform-config=single_locale/linux64.py
+ tooltool-downloads: public
+ need-xvfb: true
+
+ android-api-15-nightly/opt:
+ description: "Single Locale Repack"
+ attributes:
+ nightly: true
+ build_platform: android-api-15-nightly
+ index:
+ product: mobile
+ job-name:
+ gecko-v2: android-nightly-l10n-opt
+ treeherder:
+ platform: android-api-15/opt
+ worker-type: aws-provisioner-v1/android-api-15
+ worker:
+ max-run-time: 18000
+ env:
+ EN_US_PACKAGE_NAME: target.apk
+ EN_US_BINARY_URL:
+ task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build/en-US
+ run:
+ using: mozharness
+ script: mozharness/scripts/mobile_l10n.py
+ actions: [clone-locales list-locales setup repack upload-repacks summary]
+ config:
+ - single_locale/{project}_android-api-15.py
+ - single_locale/tc_android-api-15.py
+ tooltool-downloads: internal
+ need-xvfb: true
+
+kind-dependencies:
+ - build-signing
--- a/taskcluster/taskgraph/transforms/build_attrs.py
+++ b/taskcluster/taskgraph/transforms/build_attrs.py
@@ -20,14 +20,14 @@ def set_build_attributes(config, jobs):
build_platform, build_type = job['name'].split('/')
# pgo builds are represented as a different platform, type opt
if build_type == 'pgo':
build_platform = build_platform + '-pgo'
build_type = 'opt'
attributes = job.setdefault('attributes', {})
- attributes.update({
- 'build_platform': build_platform,
- 'build_type': build_type,
- })
+ attributes.update({'build_type': build_type})
+ if 'build_platform' not in attributes:
+ # Allow some jobs to specify build_platform differently
+ attributes.update({'build_platform': build_platform})
yield job
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -18,29 +18,31 @@ from __future__ import absolute_import,
JOB_NAME_WHITELIST = set([
'android-api-15-debug',
'android-api-15-gradle-dependencies-opt',
'android-api-15-gradle-opt',
'android-api-15-opt',
'android-api-15-nightly-opt',
'android-api-15-partner-sample1-opt',
'android-l10n-opt',
+ 'android-nightly-l10n-opt',
'android-x86-opt',
'aries-debug',
'aries-eng-opt',
'browser-haz-debug',
'linux32-l10n-opt',
'linux64-artifact-opt',
'linux64-asan-debug',
'linux64-asan-opt',
'linux64-ccov-opt',
'linux64-debug',
'linux64-jsdcov-opt',
'linux64-l10n-opt',
'linux64-nightly-opt',
+ 'linux64-nightly-l10n-opt',
'linux64-opt',
'linux64-pgo',
'linux64-st-an-opt',
'linux64-valgrind-opt',
'linux-debug',
'linux-opt',
'linux-pgo',
'linux-nightly-opt',
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -2,22 +2,57 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Do transforms specific to l10n kind
"""
from __future__ import absolute_import, print_function, unicode_literals
+import copy
+
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
@transforms.add
+def setup_nightly_dependancy(config, jobs):
+ """ Sets up a task dependency to the signing job this relates to """
+ for job in jobs:
+ if 'nightly' not in config.kind:
+ yield job
+ continue # do not add a dep unless we're a nightly
+ dep_label = "build-{}".format(job['name'])
+ job['dependencies'] = {'unsigned-build': dep_label}
+ yield job
+
+
+@transforms.add
+def chunkify(config, jobs):
+ """ Utilizes chunking for l10n stuff """
+ for job in jobs:
+ chunks = job.get('chunks')
+ if 'chunks' in job:
+ del job['chunks']
+ if chunks:
+ for this_chunk in range(1, chunks + 1):
+ chunked = copy.deepcopy(job)
+ chunked['name'] = chunked['name'].replace(
+ '/', '-{}/'.format(this_chunk), 1
+ )
+ chunked['run']['options'] = chunked['run'].get('options', [])
+ chunked['run']['options'].extend(["total-chunks={}".format(chunks),
+ "this-chunk={}".format(this_chunk)])
+ yield chunked
+ else:
+ yield job
+
+
+@transforms.add
def mh_config_replace_project(config, jobs):
""" Replaces {project} in mh config entries with the current project """
# XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
for job in jobs:
if not job['run'].get('using') == 'mozharness':
# Nothing to do, not mozharness
yield job
continue
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -272,16 +272,17 @@ GROUP_NAMES = {
'tc-R': 'Reftests executed by TaskCluster',
'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
'tc-W': 'Web platform tests executed by TaskCluster',
'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
'tc-X': 'Xpcshell tests executed by TaskCluster',
'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
'tc-Sim': 'Mulet simulator runs',
+ 'tc-L10n': 'Localised Repacks executed by Taskcluster',
'Aries': 'Aries Device Image',
'Nexus 5-L': 'Nexus 5-L Device Image',
'Cc': 'Toolchain builds',
'SM-tc': 'Spidermonkey builds',
}
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
BUILDBOT_ROUTE_TEMPLATES = [
copy from testing/mozharness/configs/single_locale/try.py
copy to testing/mozharness/configs/single_locale/date.py
--- a/testing/mozharness/configs/single_locale/try.py
+++ b/testing/mozharness/configs/single_locale/date.py
@@ -1,43 +1,29 @@
config = {
- "nightly_build": False,
- "branch": "try",
+ "nightly_build": True,
+ "branch": "date",
"en_us_binary_url": "http://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central",
- "update_channel": "nightly",
- "latest_mar_dir": '/pub/firefox/nightly/latest-mozilla-central-l10n',
- "update_gecko_source_to_enUS": False,
+ "update_channel": "nightly-date",
+ "latest_mar_dir": '/pub/firefox/nightly/latest-date-l10n',
# l10n
"hg_l10n_base": "https://hg.mozilla.org/l10n-central",
# mar
"mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
# repositories
- "mozilla_dir": "try",
+ "mozilla_dir": "date",
"repos": [{
"vcs": "hg",
"repo": "https://hg.mozilla.org/build/tools",
"branch": "default",
"dest": "tools",
}, {
"vcs": "hg",
- "repo": "https://hg.mozilla.org/try",
+ "repo": "https://hg.mozilla.org/projects/date",
"revision": "%(revision)s",
- "dest": "try",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-central",
- "clone_by_revision": True,
- "clone_with_purge": True,
+ "dest": "date",
}],
# purge options
'is_automation': True,
- "upload_env": {
- "UPLOAD_USER": "trybld",
- # ssh_key_dir is defined per platform: it is "~/.ssh" for every platform
- # except when mock is in use, in this case, ssh_key_dir is
- # /home/mock_mozilla/.ssh
- "UPLOAD_SSH_KEY": "%(ssh_key_dir)s/trybld_dsa",
- "UPLOAD_HOST": "upload.trybld.productdelivery.%(upload_environment)s.mozaws.net",
- "POST_UPLOAD_CMD": "post_upload.py --who %(who)s --builddir %(branch)s-%(platform)s --tinderbox-builds-dir %(who)s-%(revision)s -p %(stage_product)s -i %(buildid)s --revision %(revision)s --release-to-try-builds %(post_upload_extra)s",
- "UPLOAD_TO_TEMP": "1"
- },
}
copy from testing/mozharness/configs/single_locale/try_android-api-15.py
copy to testing/mozharness/configs/single_locale/date_android-api-15.py
--- a/testing/mozharness/configs/single_locale/try_android-api-15.py
+++ b/testing/mozharness/configs/single_locale/date_android-api-15.py
@@ -1,71 +1,69 @@
-BRANCH = "try"
+import os
+
+BRANCH = "date"
MOZILLA_DIR = BRANCH
EN_US_BINARY_URL = "http://archive.mozilla.org/pub/" \
- "mobile/nightly/latest-mozilla-central-android-api-15/en-US"
+ "mobile/nightly/latest-date-android-api-15/en-US"
config = {
- "branch": "try",
+ "branch": BRANCH,
"log_name": "single_locale",
"objdir": "obj-l10n",
"is_automation": True,
"buildbot_json_path": "buildprops.json",
"force_clobber": True,
"clobberer_url": "https://api.pub.build.mozilla.org/clobberer/lastclobber",
"locales_file": "%s/mobile/android/locales/all-locales" % MOZILLA_DIR,
"locales_dir": "mobile/android/locales",
"ignore_locales": ["en-US"],
- "nightly_build": False,
+ "nightly_build": True,
'balrog_credentials_file': 'oauth.txt',
"tools_repo": "https://hg.mozilla.org/build/tools",
"tooltool_config": {
"manifest": "mobile/android/config/tooltool-manifests/android/releng.manifest",
"output_dir": "%(abs_work_dir)s/" + MOZILLA_DIR,
},
"exes": {
'tooltool.py': '/builds/tooltool.py',
},
- "update_gecko_source_to_enUS": False,
"repos": [{
"vcs": "hg",
"repo": "https://hg.mozilla.org/build/tools",
"branch": "default",
"dest": "tools",
}, {
"vcs": "hg",
- "repo": "https://hg.mozilla.org/try",
+ "repo": "https://hg.mozilla.org/projects/date",
"revision": "%(revision)s",
- "dest": "try",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-central",
- "clone_by_revision": True,
- "clone_with_purge": True,
+ "dest": MOZILLA_DIR,
}],
"hg_l10n_base": "https://hg.mozilla.org/l10n-central",
"hg_l10n_tag": "default",
'vcs_share_base': "/builds/hg-shared",
"l10n_dir": "l10n-central",
"repack_env": {
# so ugly, bug 951238
"LD_LIBRARY_PATH": "/lib:/tools/gcc-4.7.2-0moz1/lib:/tools/gcc-4.7.2-0moz1/lib64",
"MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": EN_US_BINARY_URL,
+ "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
"LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "MOZ_UPDATE_CHANNEL": "try", # XXX Invalid
+ "MOZ_UPDATE_CHANNEL": "nightly-date",
},
"upload_branch": "%s-android-api-15" % BRANCH,
"ssh_key_dir": "~/.ssh",
"merge_locales": True,
"mozilla_dir": MOZILLA_DIR,
"mozconfig": "%s/mobile/android/config/mozconfigs/android-api-15/l10n-nightly" % MOZILLA_DIR,
"signature_verification_script": "tools/release/signing/verify-android-signature.sh",
"stage_product": "mobile",
- "platform": "android", # XXX Validate
- "build_type": "api-15-opt", # XXX Validate
+ "platform": "android",
+ "build_type": "api-15-opt",
# Balrog
"build_target": "Android_arm-eabi-gcc3",
# Mock
"mock_target": "mozilla-centos6-x86_64-android",
"mock_packages": ['autoconf213', 'python', 'zip', 'mozilla-python27-mercurial', 'git', 'ccache',
'glibc-static', 'libstdc++-static', 'perl-Test-Simple', 'perl-Config-General',
--- a/testing/mozharness/configs/single_locale/tc_linux32.py
+++ b/testing/mozharness/configs/single_locale/tc_linux32.py
@@ -1,18 +1,20 @@
import os
+EN_US_BINARY_URL = "%(en_us_binary_url)s"
+
config = {
"locales_file": "src/browser/locales/all-locales",
"tools_repo": "https://hg.mozilla.org/build/tools",
"mozconfig": "src/browser/config/mozconfigs/linux64/l10n-mozconfig",
"bootstrap_env": {
"NO_MERCURIAL_SETUP_CHECK": "1",
"MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
+ "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
"LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
"MOZ_UPDATE_CHANNEL": "%(update_channel)s",
"DIST": "%(abs_objdir)s",
"LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
"L10NBASEDIR": "../../l10n",
"MOZ_MAKE_COMPLETE_MAR": "1",
'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
},
--- a/testing/mozharness/configs/single_locale/tc_linux64.py
+++ b/testing/mozharness/configs/single_locale/tc_linux64.py
@@ -1,18 +1,20 @@
import os
+EN_US_BINARY_URL = "%(en_us_binary_url)s"
+
config = {
"locales_file": "src/browser/locales/all-locales",
"tools_repo": "https://hg.mozilla.org/build/tools",
"mozconfig": "src/browser/config/mozconfigs/linux64/l10n-mozconfig",
"bootstrap_env": {
"NO_MERCURIAL_SETUP_CHECK": "1",
"MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
+ "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
"LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
"MOZ_UPDATE_CHANNEL": "%(update_channel)s",
"DIST": "%(abs_objdir)s",
"LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
"L10NBASEDIR": "../../l10n",
"MOZ_MAKE_COMPLETE_MAR": "1",
'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
},