Bug 1171738 - Migrate Linux64, Linux32, and Android l10n desktop nightly repacks to TaskCluster. f=dustin draft
authorJustin Wood <Callek@gmail.com>
Mon, 10 Oct 2016 10:34:07 -0400
changeset 426086 4b9e1dab252037b3a3aa1359d39f30cebf316f5c
parent 426085 811fe85b6452df5c1c1aa1cc162b5cc363470034
child 534091 877fe66bd086a0dc35f062e26afdf8a4465b2408
push id32615
push userCallek@gmail.com
push dateMon, 17 Oct 2016 19:34:46 +0000
bugs1171738
milestone52.0a1
Bug 1171738 - Migrate Linux64, Linux32, and Android l10n desktop nightly repacks to TaskCluster. f=dustin Make sure EN_US_BINARY_URL is settable by environment to allow taskcluster to pass in a specific location. Also for now depend on *unsigned* builds so that android does not report the apk as corrupt during repack steps. MozReview-Commit-ID: H7LzRTktoiN
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/taskgraph/transforms/build_attrs.py
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/transforms/l10n.py
taskcluster/taskgraph/transforms/task.py
testing/mozharness/configs/single_locale/date.py
testing/mozharness/configs/single_locale/date_android-api-15.py
testing/mozharness/configs/single_locale/tc_linux32.py
testing/mozharness/configs/single_locale/tc_linux64.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -0,0 +1,124 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.l10n:transforms
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+    index:
+        product: firefox
+    treeherder:
+        kind: build
+        tier: 2
+        symbol: tc-L10n(N)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 36000
+    # don't run anywhere by default, but still used from nightlies
+    run-on-projects: []
+    run:
+        # NOTE: this should really be a different "using" since it's using
+        # build-l10n.sh instead of build-linux.sh.  Preferably, build-linux.sh
+        # and the mozharness run implementation should be modified to support
+        # the functionality that l10n needs
+        using: mozharness
+        job-script: taskcluster/scripts/builder/build-l10n.sh
+    chunks: 6
+
+jobs:
+    # Names correspond to the <platform> nightly label (as used in signing)
+    # In order to automatically set the dependency.
+    linux-nightly/opt:
+        description: "Localization"
+        attributes:
+            build_platform: linux64-nightly
+            nightly: true
+        index:
+            job-name:
+                gecko-v2: linux64-nightly-l10n-opt
+        treeherder:
+            platform: linux/opt
+        worker:
+            env:
+                EN_US_PACKAGE_NAME: target.tar.bz2
+                EN_US_BINARY_URL:
+                    task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build
+        run:
+            script: mozharness/scripts/desktop_l10n.py
+            actions: [clone-locales list-locales setup repack summary]
+            config:
+                - single_locale/tc_linux64.py
+            options:
+                - environment-config=single_locale/production.py
+                - branch-config=single_locale/{project}.py
+                - platform-config=single_locale/linux64.py
+            tooltool-downloads: public
+            need-xvfb: true
+
+    linux64-nightly/opt:
+        description: "Localization"
+        attributes:
+            build_platform: linux64-nightly
+            nightly: true
+        index:
+            job-name:
+                gecko-v2: linux64-nightly-l10n-opt
+        treeherder:
+            platform: linux64/opt
+        worker:
+            env:
+                EN_US_PACKAGE_NAME: target.tar.bz2
+                EN_US_BINARY_URL:
+                    task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build
+        run:
+            script: mozharness/scripts/desktop_l10n.py
+            actions: [clone-locales list-locales setup repack summary]
+            config:
+                - single_locale/tc_linux64.py
+            options:
+                - environment-config=single_locale/production.py
+                - branch-config=single_locale/{project}.py
+                - platform-config=single_locale/linux64.py
+            tooltool-downloads: public
+            need-xvfb: true
+
+    android-api-15-nightly/opt:
+        description: "Single Locale Repack"
+        attributes:
+            nightly: true
+            build_platform: android-api-15-nightly
+        index:
+            product: mobile
+            job-name:
+                gecko-v2: android-nightly-l10n-opt
+        treeherder:
+            platform: android-api-15/opt
+        worker-type: aws-provisioner-v1/android-api-15
+        worker:
+            max-run-time: 18000
+            env:
+                EN_US_PACKAGE_NAME: target.apk
+                EN_US_BINARY_URL:
+                    task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/public/build/en-US
+        run:
+            using: mozharness
+            script: mozharness/scripts/mobile_l10n.py
+            actions: [clone-locales list-locales setup repack upload-repacks summary]
+            config:
+                - single_locale/{project}_android-api-15.py
+                - single_locale/tc_android-api-15.py
+            tooltool-downloads: internal
+            need-xvfb: true
+
+kind-dependencies:
+  - build-signing
--- a/taskcluster/taskgraph/transforms/build_attrs.py
+++ b/taskcluster/taskgraph/transforms/build_attrs.py
@@ -20,14 +20,14 @@ def set_build_attributes(config, jobs):
         build_platform, build_type = job['name'].split('/')
 
         # pgo builds are represented as a different platform, type opt
         if build_type == 'pgo':
             build_platform = build_platform + '-pgo'
             build_type = 'opt'
 
         attributes = job.setdefault('attributes', {})
-        attributes.update({
-            'build_platform': build_platform,
-            'build_type': build_type,
-        })
+        attributes.update({'build_type': build_type})
+        if 'build_platform' not in attributes:
+            # Allow some jobs to specify build_platform differently
+            attributes.update({'build_platform': build_platform})
 
         yield job
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -18,29 +18,31 @@ from __future__ import absolute_import, 
 JOB_NAME_WHITELIST = set([
     'android-api-15-debug',
     'android-api-15-gradle-dependencies-opt',
     'android-api-15-gradle-opt',
     'android-api-15-opt',
     'android-api-15-nightly-opt',
     'android-api-15-partner-sample1-opt',
     'android-l10n-opt',
+    'android-nightly-l10n-opt',
     'android-x86-opt',
     'aries-debug',
     'aries-eng-opt',
     'browser-haz-debug',
     'linux32-l10n-opt',
     'linux64-artifact-opt',
     'linux64-asan-debug',
     'linux64-asan-opt',
     'linux64-ccov-opt',
     'linux64-debug',
     'linux64-jsdcov-opt',
     'linux64-l10n-opt',
     'linux64-nightly-opt',
+    'linux64-nightly-l10n-opt',
     'linux64-opt',
     'linux64-pgo',
     'linux64-st-an-opt',
     'linux64-valgrind-opt',
     'linux-debug',
     'linux-opt',
     'linux-pgo',
     'linux-nightly-opt',
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -2,22 +2,57 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Do transforms specific to l10n kind
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import copy
+
 from taskgraph.transforms.base import TransformSequence
 
 transforms = TransformSequence()
 
 
 @transforms.add
+def setup_nightly_dependancy(config, jobs):
+    """ Sets up a task dependency to the signing job this relates to """
+    for job in jobs:
+        if 'nightly' not in config.kind:
+            yield job
+            continue  # do not add a dep unless we're a nightly
+        dep_label = "build-{}".format(job['name'])
+        job['dependencies'] = {'unsigned-build': dep_label}
+        yield job
+
+
+@transforms.add
+def chunkify(config, jobs):
+    """ Utilizes chunking for l10n stuff """
+    for job in jobs:
+        chunks = job.get('chunks')
+        if 'chunks' in job:
+            del job['chunks']
+        if chunks:
+            for this_chunk in range(1, chunks + 1):
+                chunked = copy.deepcopy(job)
+                chunked['name'] = chunked['name'].replace(
+                    '/', '-{}/'.format(this_chunk), 1
+                )
+                chunked['run']['options'] = chunked['run'].get('options', [])
+                chunked['run']['options'].extend(["total-chunks={}".format(chunks),
+                                                  "this-chunk={}".format(this_chunk)])
+                yield chunked
+        else:
+            yield job
+
+
+@transforms.add
 def mh_config_replace_project(config, jobs):
     """ Replaces {project} in mh config entries with the current project """
     # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
     for job in jobs:
         if not job['run'].get('using') == 'mozharness':
             # Nothing to do, not mozharness
             yield job
             continue
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -272,16 +272,17 @@ GROUP_NAMES = {
     'tc-R': 'Reftests executed by TaskCluster',
     'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-Sim': 'Mulet simulator runs',
+    'tc-L10n': 'Localised Repacks executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 BUILDBOT_ROUTE_TEMPLATES = [
copy from testing/mozharness/configs/single_locale/try.py
copy to testing/mozharness/configs/single_locale/date.py
--- a/testing/mozharness/configs/single_locale/try.py
+++ b/testing/mozharness/configs/single_locale/date.py
@@ -1,43 +1,29 @@
 config = {
-    "nightly_build": False,
-    "branch": "try",
+    "nightly_build": True,
+    "branch": "date",
     "en_us_binary_url": "http://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central",
-    "update_channel": "nightly",
-    "latest_mar_dir": '/pub/firefox/nightly/latest-mozilla-central-l10n',
-    "update_gecko_source_to_enUS": False,
+    "update_channel": "nightly-date",
+    "latest_mar_dir": '/pub/firefox/nightly/latest-date-l10n',
 
     # l10n
     "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
 
     # mar
     "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
 
     # repositories
-    "mozilla_dir": "try",
+    "mozilla_dir": "date",
     "repos": [{
         "vcs": "hg",
         "repo": "https://hg.mozilla.org/build/tools",
         "branch": "default",
         "dest": "tools",
     }, {
         "vcs": "hg",
-        "repo": "https://hg.mozilla.org/try",
+        "repo": "https://hg.mozilla.org/projects/date",
         "revision": "%(revision)s",
-        "dest": "try",
-        "clone_upstream_url": "https://hg.mozilla.org/mozilla-central",
-        "clone_by_revision": True,
-        "clone_with_purge": True,
+        "dest": "date",
     }],
     # purge options
     'is_automation': True,
-    "upload_env": {
-        "UPLOAD_USER": "trybld",
-        # ssh_key_dir is defined per platform: it is "~/.ssh" for every platform
-        # except when mock is in use, in this case, ssh_key_dir is
-        # /home/mock_mozilla/.ssh
-        "UPLOAD_SSH_KEY": "%(ssh_key_dir)s/trybld_dsa",
-        "UPLOAD_HOST": "upload.trybld.productdelivery.%(upload_environment)s.mozaws.net",
-        "POST_UPLOAD_CMD": "post_upload.py --who %(who)s --builddir %(branch)s-%(platform)s --tinderbox-builds-dir %(who)s-%(revision)s -p %(stage_product)s -i %(buildid)s --revision %(revision)s --release-to-try-builds %(post_upload_extra)s",
-        "UPLOAD_TO_TEMP": "1"
-    },
 }
copy from testing/mozharness/configs/single_locale/try_android-api-15.py
copy to testing/mozharness/configs/single_locale/date_android-api-15.py
--- a/testing/mozharness/configs/single_locale/try_android-api-15.py
+++ b/testing/mozharness/configs/single_locale/date_android-api-15.py
@@ -1,71 +1,69 @@
-BRANCH = "try"
+import os
+
+BRANCH = "date"
 MOZILLA_DIR = BRANCH
 EN_US_BINARY_URL = "http://archive.mozilla.org/pub/" \
-                   "mobile/nightly/latest-mozilla-central-android-api-15/en-US"
+                   "mobile/nightly/latest-date-android-api-15/en-US"
 
 config = {
-    "branch": "try",
+    "branch": BRANCH,
     "log_name": "single_locale",
     "objdir": "obj-l10n",
     "is_automation": True,
     "buildbot_json_path": "buildprops.json",
     "force_clobber": True,
     "clobberer_url": "https://api.pub.build.mozilla.org/clobberer/lastclobber",
     "locales_file": "%s/mobile/android/locales/all-locales" % MOZILLA_DIR,
     "locales_dir": "mobile/android/locales",
     "ignore_locales": ["en-US"],
-    "nightly_build": False,
+    "nightly_build": True,
     'balrog_credentials_file': 'oauth.txt',
     "tools_repo": "https://hg.mozilla.org/build/tools",
     "tooltool_config": {
         "manifest": "mobile/android/config/tooltool-manifests/android/releng.manifest",
         "output_dir": "%(abs_work_dir)s/" + MOZILLA_DIR,
     },
     "exes": {
         'tooltool.py': '/builds/tooltool.py',
     },
-    "update_gecko_source_to_enUS": False,
     "repos": [{
         "vcs": "hg",
         "repo": "https://hg.mozilla.org/build/tools",
         "branch": "default",
         "dest": "tools",
     }, {
         "vcs": "hg",
-        "repo": "https://hg.mozilla.org/try",
+        "repo": "https://hg.mozilla.org/projects/date",
         "revision": "%(revision)s",
-        "dest": "try",
-        "clone_upstream_url": "https://hg.mozilla.org/mozilla-central",
-        "clone_by_revision": True,
-        "clone_with_purge": True,
+        "dest": MOZILLA_DIR,
     }],
     "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
     "hg_l10n_tag": "default",
     'vcs_share_base': "/builds/hg-shared",
 
     "l10n_dir": "l10n-central",
     "repack_env": {
         # so ugly, bug 951238
         "LD_LIBRARY_PATH": "/lib:/tools/gcc-4.7.2-0moz1/lib:/tools/gcc-4.7.2-0moz1/lib64",
         "MOZ_OBJDIR": "obj-l10n",
-        "EN_US_BINARY_URL": EN_US_BINARY_URL,
+        "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
-        "MOZ_UPDATE_CHANNEL": "try", # XXX Invalid
+        "MOZ_UPDATE_CHANNEL": "nightly-date",
     },
     "upload_branch": "%s-android-api-15" % BRANCH,
     "ssh_key_dir": "~/.ssh",
     "merge_locales": True,
     "mozilla_dir": MOZILLA_DIR,
     "mozconfig": "%s/mobile/android/config/mozconfigs/android-api-15/l10n-nightly" % MOZILLA_DIR,
     "signature_verification_script": "tools/release/signing/verify-android-signature.sh",
     "stage_product": "mobile",
-    "platform": "android", # XXX Validate
-    "build_type": "api-15-opt", # XXX Validate
+    "platform": "android",
+    "build_type": "api-15-opt",
 
     # Balrog
     "build_target": "Android_arm-eabi-gcc3",
 
     # Mock
     "mock_target": "mozilla-centos6-x86_64-android",
     "mock_packages": ['autoconf213', 'python', 'zip', 'mozilla-python27-mercurial', 'git', 'ccache',
                       'glibc-static', 'libstdc++-static', 'perl-Test-Simple', 'perl-Config-General',
--- a/testing/mozharness/configs/single_locale/tc_linux32.py
+++ b/testing/mozharness/configs/single_locale/tc_linux32.py
@@ -1,18 +1,20 @@
 import os
 
+EN_US_BINARY_URL = "%(en_us_binary_url)s"
+
 config = {
     "locales_file": "src/browser/locales/all-locales",
     "tools_repo": "https://hg.mozilla.org/build/tools",
     "mozconfig": "src/browser/config/mozconfigs/linux64/l10n-mozconfig",
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "obj-l10n",
-        "EN_US_BINARY_URL": "%(en_us_binary_url)s",
+        "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
         "L10NBASEDIR": "../../l10n",
         "MOZ_MAKE_COMPLETE_MAR": "1",
         'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
     },
--- a/testing/mozharness/configs/single_locale/tc_linux64.py
+++ b/testing/mozharness/configs/single_locale/tc_linux64.py
@@ -1,18 +1,20 @@
 import os
 
+EN_US_BINARY_URL = "%(en_us_binary_url)s"
+
 config = {
     "locales_file": "src/browser/locales/all-locales",
     "tools_repo": "https://hg.mozilla.org/build/tools",
     "mozconfig": "src/browser/config/mozconfigs/linux64/l10n-mozconfig",
     "bootstrap_env": {
         "NO_MERCURIAL_SETUP_CHECK": "1",
         "MOZ_OBJDIR": "obj-l10n",
-        "EN_US_BINARY_URL": "%(en_us_binary_url)s",
+        "EN_US_BINARY_URL": os.environ.get("EN_US_BINARY_URL", EN_US_BINARY_URL),
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
         "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
         "DIST": "%(abs_objdir)s",
         "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
         "L10NBASEDIR": "../../l10n",
         "MOZ_MAKE_COMPLETE_MAR": "1",
         'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
     },