--- a/taskcluster/ci/android-test/kind.yml
+++ b/taskcluster/ci/android-test/kind.yml
@@ -1,12 +1,12 @@
implementation: taskgraph.task.test:TestTask
kind-dependencies:
- - legacy
+ - build
transforms:
- taskgraph.transforms.tests.test_description:validate
- taskgraph.transforms.tests.android_test:transforms
- taskgraph.transforms.tests.all_kinds:transforms
- taskgraph.transforms.tests.test_description:validate
- taskgraph.transforms.tests.make_task_description:transforms
- taskgraph.transforms.task:transformsnew file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/android-partner.yml
@@ -0,0 +1,32 @@
+android-partner-sample1/opt:
+ description: "Android armv7 API 15+ partner sample 1"
+ index:
+ product: mobile
+ job-name: android-api-15-partner-sample1-opt
+ treeherder:
+ platform: android-4-0-armv7-api15-partner1/opt
+ symbol: tc(B)
+ tier: 2
+ run-on-projects:
+ - try
+ worker-type: aws-provisioner-v1/android-api-15
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ env:
+ EXTRA_CHECKOUT_REPOSITORIES: "PARTNER\n"
+ PARTNER_BASE_REPOSITORY: "https://github.com/mozilla/fennec-distribution-sample"
+ PARTNER_DEST_DIR: "/home/worker/workspace/build/partner"
+ PARTNER_HEAD_REPOSITORY: "https://github.com/mozilla/fennec-distribution-sample"
+ PARTNER_HEAD_REV: "master"
+ run:
+ using: mozharness
+ actions: [get-secrets build multi-l10n update]
+ config:
+ - builds/releng_base_android_64_builds.py
+ - disable_signing.py
+ - platform_supports_post_upload_to_latest.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ custom-build-variant-cfg: api-15-partner-sample1
+ tooltool-downloads: internal
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/android.yml
@@ -0,0 +1,102 @@
+android-api-15/debug:
+ description: "Android 4.0 API15+ Debug"
+ index:
+ product: mobile
+ job-name:
+ buildbot: android-api-15-debug
+ gecko-v1: android-api-15.debug
+ gecko-v2: android-api-15-debug
+ treeherder:
+ platform: android-4-0-armv7-api15/debug
+ symbol: tc(B)
+ worker-type: aws-provisioner-v1/android-api-15
+ worker:
+ implementation: docker-worker
+ max-run-time: 7200
+ run:
+ using: mozharness
+ actions: [get-secrets build multi-l10n update]
+ config:
+ - builds/releng_base_android_64_builds.py
+ - disable_signing.py
+ - platform_supports_post_upload_to_latest.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ custom-build-variant-cfg: api-15-debug
+ tooltool-downloads: internal
+
+android-x86/opt:
+ description: "Android 4.2 x86 Opt"
+ index:
+ product: mobile
+ job-name: android-x86-opt
+ treeherder:
+ platform: android-4-2-x86/opt
+ symbol: tc(B)
+ tier: 1
+ worker-type: aws-provisioner-v1/android-api-15
+ worker:
+ implementation: docker-worker
+ max-run-time: 7200
+ run:
+ using: mozharness
+ actions: [get-secrets build multi-l10n update]
+ config:
+ - builds/releng_base_android_64_builds.py
+ - disable_signing.py
+ - platform_supports_post_upload_to_latest.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ custom-build-variant-cfg: x86
+ tooltool-downloads: internal
+
+android-api-15/opt:
+ description: "Android 4.0 API15+ Opt"
+ index:
+ product: mobile
+ job-name: android-api-15-opt
+ treeherder:
+ platform: android-4-0-armv7-api15/opt
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/android-api-15
+ worker:
+ implementation: docker-worker
+ max-run-time: 7200
+ run:
+ using: mozharness
+ actions: [get-secrets build multi-l10n update]
+ config:
+ - builds/releng_base_android_64_builds.py
+ - disable_signing.py
+ - platform_supports_post_upload_to_latest.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ custom-build-variant-cfg: api-15
+ tooltool-downloads: internal
+
+android-api-15-gradle/opt:
+ description: "Android 4.0 API15+ (Gradle) Opt"
+ index:
+ product: mobile
+ job-name: android-api-15-gradle-opt
+ treeherder:
+ platform: android-4-0-armv7-api15/opt
+ symbol: tc(Bg)
+ tier: 2
+ worker-type: aws-provisioner-v1/android-api-15
+ worker:
+ implementation: docker-worker
+ max-run-time: 7200
+ env:
+ # Bug 1292762 - Set GRADLE_USER_HOME to avoid sdk-manager-plugin intermittent
+ GRADLE_USER_HOME: /home/worker/workspace/build/src/dotgradle
+ run:
+ using: mozharness
+ actions: [get-secrets build multi-l10n update]
+ config:
+ - builds/releng_base_android_64_builds.py
+ - disable_signing.py
+ - platform_supports_post_upload_to_latest.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ custom-build-variant-cfg: api-15-gradle
+ tooltool-downloads: internal
+
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/kind.yml
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+ - taskgraph.transforms.build:transforms
+ - taskgraph.transforms.build_attrs:transforms
+ - taskgraph.transforms.job:transforms
+ - taskgraph.transforms.task:transforms
+
+jobs-from:
+ - android-partner.yml
+ - android.yml
+ - linux.yml
+ - macosx.yml
+ - mulet.yml
+ - windows.yml
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/linux.yml
@@ -0,0 +1,176 @@
+linux64/opt:
+ description: "Linux64 Opt"
+ index:
+ product: firefox
+ job-name: linux64-opt
+ treeherder:
+ platform: linux64/opt
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ config:
+ - builds/releng_base_linux_64_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ tooltool-downloads: public
+ need-xvfb: true
+
+linux64/pgo:
+ description: "Linux64 Opt PGO"
+ index:
+ product: firefox
+ job-name: linux64-pgo-opt
+ treeherder:
+ platform: linux64/pgo
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ coalesce-name: linux64-pgo
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ options: [enable-pgo]
+ config:
+ - builds/releng_base_linux_64_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ tooltool-downloads: public
+ need-xvfb: true
+
+linux64/debug:
+ description: "Linux64 Debug"
+ index:
+ product: firefox
+ job-name:
+ buildbot: linux64-debug
+ gecko-v1: linux64.debug
+ gecko-v2: linux64-debug
+ treeherder:
+ platform: linux64/debug
+ symbol: tc(B)
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ config:
+ - builds/releng_base_linux_64_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ custom-build-variant-cfg: debug
+ tooltool-downloads: public
+ need-xvfb: true
+
+linux/opt:
+ description: "Linux32 Opt"
+ index:
+ product: firefox
+ job-name: linux-opt
+ treeherder:
+ platform: linux32/opt
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ coalesce-name: opt_linux32
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ config:
+ - builds/releng_base_linux_32_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ tooltool-downloads: public
+ need-xvfb: true
+
+linux/debug:
+ description: "Linux32 Debug"
+ index:
+ product: firefox
+ job-name: linux-debug
+ treeherder:
+ platform: linux32/debug
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ coalesce-name: dbg_linux32
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ config:
+ - builds/releng_base_linux_32_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ custom-build-variant-cfg: debug
+ tooltool-downloads: public
+ need-xvfb: true
+
+linux64-asan/opt:
+ description: "Linux64 Opt ASAN"
+ index:
+ product: firefox
+ job-name: linux64-asan-opt
+ treeherder:
+ platform: linux64/asan
+ symbol: tc(Bo)
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ config:
+ - builds/releng_base_linux_64_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ custom-build-variant-cfg: asan-tc
+ tooltool-downloads: public
+ need-xvfb: true
+
+linux64-asan/debug:
+ description: "Linux64 Debug ASAN"
+ index:
+ product: firefox
+ job-name: linux64-asan-debug
+ treeherder:
+ platform: linux64/asan
+ symbol: tc(Bd)
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ run:
+ using: mozharness
+ actions: [get-secrets build check-test generate-build-stats update]
+ config:
+ - builds/releng_base_linux_64_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ custom-build-variant-cfg: asan-tc-and-debug
+ tooltool-downloads: public
+ need-xvfb: true
+new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/macosx.yml
@@ -0,0 +1,46 @@
+macosx64/debug:
+ description: "MacOS X x64 Cross-compile"
+ index:
+ product: firefox
+ job-name: macosx64-debug
+ treeherder:
+ platform: osx-10-7/debug
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/dbg-macosx64
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ run:
+ using: mozharness
+ actions: [get-secrets build generate-build-stats update]
+ config:
+ - builds/releng_base_mac_64_cross_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ custom-build-variant-cfg: cross-debug
+ tooltool-downloads: internal
+
+macosx64/opt:
+ description: "MacOS X x64 Cross-compile"
+ index:
+ product: firefox
+ job-name: macosx64-opt
+ treeherder:
+ platform: osx-10-7/opt
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/opt-macosx64
+ worker:
+ implementation: docker-worker
+ max-run-time: 36000
+ run:
+ using: mozharness
+ actions: [get-secrets build generate-build-stats update]
+ config:
+ - builds/releng_base_mac_64_cross_builds.py
+ - balrog/production.py
+ script: "mozharness/scripts/fx_desktop_build.py"
+ secrets: true
+ tooltool-downloads: internal
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/mulet.yml
@@ -0,0 +1,43 @@
+linux64-mulet/debug:
+ description: "Linux64 Mulet Debug"
+ index:
+ product: b2g
+ job-name:
+ buildbot: linux64-mulet
+ gecko-v1: mulet.dbg
+ gecko-v2: mulet-dbg
+ treeherder:
+ platform: mulet-linux64/debug # See bug 1286086
+ symbol: B
+ tier: 3
+ worker-type: aws-provisioner-v1/mulet-debug
+ worker:
+ implementation: docker-worker
+ max-run-time: 3600
+ run:
+ using: mach-via-build-mulet-linux.sh
+ mozconfig: b2g/dev/config/mozconfigs/linux64/mulet_dbg
+ tooltool-manifest: b2g/dev/config/tooltool-manifests/linux64/releng.manifest
+
+linux64-mulet/opt:
+ description: "Linux64 Mulet Opt"
+ index:
+ product: b2g
+ job-name:
+ buildbot: linux64-mulet
+ gecko-v1: mulet.opt
+ gecko-v2: mulet-opt
+ treeherder:
+ platform: mulet-linux64/opt # ?!?%
+ symbol: B
+ tier: 3
+ worker-type: aws-provisioner-v1/mulet-opt
+ worker:
+ implementation: docker-worker
+ max-run-time: 3600
+ run:
+ using: mach-via-build-mulet-linux.sh
+ mozconfig: b2g/dev/config/mozconfigs/linux64/mulet
+ tooltool-manifest: b2g/dev/config/tooltool-manifests/linux64/releng.manifest
+
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/windows.yml
@@ -0,0 +1,80 @@
+win32/debug:
+ description: "Win32 Debug"
+ index:
+ product: firefox
+ job-name:
+ gecko-v2: win32-debug
+ treeherder:
+ platform: windowsxp/debug
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+ worker:
+ implementation: generic-worker
+ max-run-time: 7200
+ run:
+ using: mozharness
+ script: mozharness/scripts/fx_desktop_build.py
+ config:
+ - builds/taskcluster_firefox_win32_debug.py
+
+win32/opt:
+ description: "Win32 Opt"
+ index:
+ product: firefox
+ job-name:
+ gecko-v2: win32-opt
+ treeherder:
+ platform: windowsxp/opt
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+ worker:
+ implementation: generic-worker
+ max-run-time: 7200
+ run:
+ using: mozharness
+ script: mozharness/scripts/fx_desktop_build.py
+ config:
+ - builds/taskcluster_firefox_win32_opt.py
+
+win64/debug:
+ description: "Win64 Debug"
+ index:
+ product: firefox
+ job-name:
+ gecko-v2: win64-debug
+ treeherder:
+ platform: windows8-64/debug
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+ worker:
+ implementation: generic-worker
+ max-run-time: 7200
+ run:
+ using: mozharness
+ script: mozharness/scripts/fx_desktop_build.py
+ config:
+ - builds/taskcluster_firefox_win64_debug.py
+
+win64/opt:
+ description: "Win64 Opt"
+ index:
+ product: firefox
+ job-name:
+ gecko-v2: win64-opt
+ treeherder:
+ platform: windows8-64/opt
+ symbol: tc(B)
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+ worker:
+ implementation: generic-worker
+ max-run-time: 7200
+ run:
+ using: mozharness
+ script: mozharness/scripts/fx_desktop_build.py
+ config:
+ - builds/taskcluster_firefox_win64_opt.py
+--- a/taskcluster/ci/desktop-test/kind.yml
+++ b/taskcluster/ci/desktop-test/kind.yml
@@ -1,12 +1,12 @@
implementation: taskgraph.task.test:TestTask
kind-dependencies:
- - legacy
+ - build
transforms:
- taskgraph.transforms.tests.test_description:validate
- taskgraph.transforms.tests.desktop_test:transforms
- taskgraph.transforms.tests.all_kinds:transforms
- taskgraph.transforms.tests.test_description:validate
- taskgraph.transforms.tests.make_task_description:transforms
- taskgraph.transforms.task:transforms--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -1,17 +1,20 @@
Task Kinds
==========
This section lists and documents the available task kinds.
-Builds
+build
------
-Builds are currently implemented by the ``legacy`` kind.
+Builds are tasks that produce an installer or other output that can be run by
+users or automated tests. This is more restrictive than most definitions of
+"build" in a Mozilla context: it does not include tasks that run build-like
+actions for static analysis or to produce instrumented artifacts.
Tests
-----
Test tasks for Gecko products are divided into several kinds, but share a
common implementation. The process goes like this, based on a set of YAML
files named in ``kind.yml``:
--- a/taskcluster/taskgraph/task/test.py
+++ b/taskcluster/taskgraph/task/test.py
@@ -57,19 +57,16 @@ class TestTask(transform.TransformTask):
def get_builds_by_platform(cls, dep_kind, loaded_tasks):
"""Find the build tasks on which tests will depend, keyed by
platform/type. Returns a dictionary mapping build platform to task
label."""
builds_by_platform = {}
for task in loaded_tasks:
if task.kind != dep_kind:
continue
- # remove this check when builds are no longer legacy
- if task.attributes['legacy_kind'] != 'build':
- continue
build_platform = task.attributes.get('build_platform')
build_type = task.attributes.get('build_type')
if not build_platform or not build_type:
continue
platform = "{}/{}".format(build_platform, build_type)
if platform in builds_by_platform:
raise Exception("multiple build jobs for " + platform)new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/build.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply some defaults and minor modifications to the jobs defined in the build
+kind.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_defaults(config, jobs):
+ """Set defaults, including those that differ per worker implementation"""
+ for job in jobs:
+ job['treeherder'].setdefault('kind', 'build')
+ job['treeherder'].setdefault('tier', 1)
+ if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
+ job['worker'].setdefault('docker-image', {'in-tree': 'desktop-build'})
+ yield jobnew file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/build_attrs.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_build_attributes(config, jobs):
+ """
+ Set the build_platform and build_type attributes based on the job name.
+ Although not all jobs using this transform are actual "builds", the try
+ option syntax treats them as such, and this arranges the attributes
+ appropriately for that purpose.
+ """
+ for job in jobs:
+ build_platform, build_type = job['name'].split('/')
+
+ # pgo builds are represented as a different platform, type opt
+ if build_type == 'pgo':
+ build_platform = build_platform + '-pgo'
+ build_type = 'opt'
+
+ attributes = job.setdefault('attributes', {})
+ attributes.update({
+ 'build_platform': build_platform,
+ 'build_type': build_type,
+ })
+
+ yield jobnew file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Common support for various job types. These functions are all named after the
+worker implementation they operate on, and take the same three parameters, for
+consistency.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
+
+
+def docker_worker_add_workspace_cache(config, job, taskdesc):
+ """Add the workspace cache based on the build platform/type and level,
+ except on try where workspace caches are not used."""
+ if config.params['project'] == 'try':
+ return
+
+ taskdesc['worker'].setdefault('caches', []).append({
+ 'type': 'persistent',
+ 'name': 'level-{}-{}-build-{}-{}-workspace'.format(
+ config.params['level'], config.params['project'],
+ taskdesc['attributes']['build_platform'],
+ taskdesc['attributes']['build_type'],
+ ),
+ 'mount-point': "/home/worker/workspace",
+ })
+
+
+def docker_worker_add_tc_vcs_cache(config, job, taskdesc):
+ taskdesc['worker'].setdefault('caches', []).append({
+ 'type': 'persistent',
+ 'name': 'level-{}-{}-tc-vcs'.format(
+ config.params['level'], config.params['project']),
+ 'mount-point': "/home/worker/.tc-vcs",
+ })
+
+
+def docker_worker_add_public_artifacts(config, job, taskdesc):
+ taskdesc['worker'].setdefault('artifacts', []).append({
+ 'name': 'public/build',
+ 'path': '/home/worker/artifacts/',
+ 'type': 'directory',
+ })
+
+
+def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc):
+ """Add the GECKO_BASE_* and GECKO_HEAD_* env vars to the worker."""
+ env = taskdesc['worker'].setdefault('env', {})
+ env.update({
+ 'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+ 'GECKO_HEAD_REF': config.params['head_rev'],
+ 'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+ 'GECKO_HEAD_REV': config.params['head_rev'],
+ })
+
+
+def docker_worker_setup_secrets(config, job, taskdesc):
+ """Set up access to secrets via taskcluster-proxy. The value of
+ run['secrets'] should be a boolean or a list of secret names that
+ can be accessed."""
+ if not job['run'].get('secrets'):
+ return
+
+ taskdesc['worker']['taskcluster-proxy'] = True
+ secrets = job['run']['secrets']
+ if secrets is True:
+ secrets = ['*']
+ for sec in secrets:
+ taskdesc['scopes'].append(SECRET_SCOPE.format(
+ job['treeherder']['kind'], config.params['level'], sec))new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -0,0 +1,195 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+
+Support for running jobs via mozharness. Ideally, most stuff gets run this
+way, and certainly anything using mozharness should use this approach.
+
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required, Optional, Any
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+ docker_worker_add_workspace_cache,
+ docker_worker_add_tc_vcs_cache,
+ docker_worker_add_gecko_vcs_env_vars,
+ docker_worker_setup_secrets,
+ docker_worker_add_public_artifacts
+)
+
+COALESCE_KEY = 'builds.{project}.{name}'
+
+mozharness_run_schema = Schema({
+ Required('using'): 'mozharness',
+
+ # the mozharness script used to run this task, relative to the testing/
+ # directory and using forward slashes even on Windows
+ Required('script'): basestring,
+
+ # the config files required for the task, relative to
+ # testing/mozharness/configs and using forward slashes even on Windows
+ Required('config'): [basestring],
+
+ # any additional actions to pass to the mozharness command; not supported
+ # on Windows
+ Optional('actions'): [basestring],
+
+ # any additional options (without leading --) to be passed to mozharness;
+ # not supported on Windows
+ Optional('options'): [basestring],
+
+ # --custom-build-variant-cfg value (not supported on Windows)
+ Optional('custom-build-variant-cfg'): basestring,
+
+ # If not false, tooltool downloads will be enabled via relengAPIProxy
+ # for either just public files, or all files. Not supported on Windows
+ Required('tooltool-downloads', default=False): Any(
+ False,
+ 'public',
+ 'internal',
+ ),
+
+ # The set of secret names to which the task has access; these are prefixed
+ # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
+ # this will enable any worker features required and set the task's scopes
+ # appropriately. `true` here means ['*'], all secrets. Not supported on
+ # Windows
+ Required('secrets', default=False): Any(bool, [basestring]),
+
+ # If true, taskcluster proxy will be enabled; note that it may also be enabled
+ # automatically e.g., for secrets support. Not supported on Windows.
+ Required('taskcluster-proxy', default=False): bool,
+
+ # If true, the build scripts will start Xvfb. Not supported on Windows.
+ Required('need-xvfb', default=False): bool,
+
+ # If false, indicate that builds should skip producing artifacts. Not
+ # supported on Windows.
+ Required('keep-artifacts', default=True): bool,
+})
+
+
+@run_job_using("docker-worker", "mozharness", schema=mozharness_run_schema)
+def mozharness_on_docker_worker_setup(config, job, taskdesc):
+ run = job['run']
+
+ worker = taskdesc['worker']
+ worker['implementation'] = job['worker']['implementation']
+
+ # running via mozharness assumes desktop-build (which contains build.sh)
+ taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
+
+ worker['relengapi-proxy'] = False # but maybe enabled for tooltool below
+ worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
+
+ docker_worker_add_public_artifacts(config, job, taskdesc)
+ docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+ docker_worker_add_workspace_cache(config, job, taskdesc)
+ docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+ env = worker.setdefault('env', {})
+ env.update({
+ 'MOZHARNESS_CONFIG': ' '.join(run['config']),
+ 'MOZHARNESS_SCRIPT': run['script'],
+ 'MH_BRANCH': config.params['project'],
+ 'MH_BUILD_POOL': 'taskcluster',
+ 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+ 'MOZ_SCM_LEVEL': config.params['level'],
+ })
+
+ if 'actions' in run:
+ env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
+
+ if 'options' in run:
+ env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
+
+ if 'custom-build-variant-cfg' in run:
+ env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
+
+ # if we're not keeping artifacts, set some env variables to empty values
+ # that will cause the build process to skip copying the results to the
+ # artifacts directory. This will have no effect for operations that are
+ # not builds.
+ if not run['keep-artifacts']:
+ env['DIST_TARGET_UPLOADS'] = ''
+ env['DIST_UPLOADS'] = ''
+
+ # Xvfb
+ if run['need-xvfb']:
+ env['NEED_XVFB'] = 'true'
+
+ # tooltool downloads
+ if run['tooltool-downloads']:
+ worker['relengapi-proxy'] = True
+ worker['caches'].append({
+ 'type': 'persistent',
+ 'name': 'tooltool-cache',
+ 'mount-point': '/home/worker/tooltool-cache',
+ })
+ taskdesc['scopes'].extend([
+ 'docker-worker:relengapi-proxy:tooltool.download.public',
+ ])
+ if run['tooltool-downloads'] == 'internal':
+ taskdesc['scopes'].append(
+ 'docker-worker:relengapi-proxy:tooltool.download.internal')
+ env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+ env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+ env['TOOLTOOL_REV'] = 'master'
+
+ docker_worker_setup_secrets(config, job, taskdesc)
+
+ worker['command'] = ["/bin/bash", "bin/build.sh"]
+
+
+# We use the generic worker to run tasks on Windows
+@run_job_using("generic-worker", "mozharness", schema=mozharness_run_schema)
+def mozharness_on_windows(config, job, taskdesc):
+ run = job['run']
+
+ # fail if invalid run options are included
+ invalid = []
+ for prop in ['actions', 'options', 'custom-build-variant-cfg',
+ 'tooltool-downloads', 'secrets', 'taskcluster-proxy',
+ 'need-xvfb']:
+ if prop in run and run[prop]:
+ invalid.append(prop)
+ if not run.get('keep-artifacts', True):
+ invalid.append('keep-artifacts')
+ if invalid:
+ raise Exception("Jobs run using mozharness on Windows do not support properties " +
+ ', '.join(invalid))
+
+ worker = taskdesc['worker']
+
+ worker['artifacts'] = [{
+ 'path': r'public\build',
+ 'type': 'directory',
+ }]
+
+ docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+ env = worker['env']
+ env.update({
+ 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+ 'MOZ_SCM_LEVEL': config.params['level'],
+ 'TOOLTOOL_REPO': 'https://github.com/mozilla/build-tooltool',
+ 'TOOLTOOL_REV': 'master',
+ })
+
+ mh_command = [r'c:\mozilla-build\python\python.exe']
+ mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')]))
+ for cfg in run['config']:
+ mh_command.append('--config ' + cfg.replace('/', '\\'))
+ mh_command.append('--branch ' + config.params['project'])
+ mh_command.append(r'--skip-buildbot-actions --work-dir %cd:Z:=z:%\build')
+ worker['command'] = [
+ r'mkdir .\build\src',
+ r'hg share c:\builds\hg-shared\mozilla-central .\build\src',
+ r'hg pull -u -R .\build\src --rev %GECKO_HEAD_REV% %GECKO_HEAD_REPOSITORY%',
+ ' '.join(mh_command),
+ ]new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/mulet.py
@@ -0,0 +1,119 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running mulet tasks via build-mulet-linux.sh
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+ docker_worker_add_workspace_cache,
+ docker_worker_add_tc_vcs_cache,
+ docker_worker_add_gecko_vcs_env_vars,
+ docker_worker_add_public_artifacts
+)
+
+COALESCE_KEY = 'builds.{project}.{name}'
+
+build_mulet_linux_schema = Schema({
+ Required('using'): 'mach-via-build-mulet-linux.sh',
+
+ # The pathname of the mozconfig to use
+ Required('mozconfig'): basestring,
+
+ # The tooltool manifest to use
+ Required('tooltool-manifest'): basestring,
+})
+
+
+@run_job_using("docker-worker", "mach-via-build-mulet-linux.sh", schema=build_mulet_linux_schema)
+def docker_worker_make_via_build_mulet_linux_sh(config, job, taskdesc):
+ run = job['run']
+ worker = taskdesc.get('worker')
+
+ # assumes the builder image (which contains the gecko checkout command)
+ taskdesc['worker']['docker-image'] = {"in-tree": "builder"}
+
+ worker['taskcluster-proxy'] = False
+
+ docker_worker_add_public_artifacts(config, job, taskdesc)
+ docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+ docker_worker_add_workspace_cache(config, job, taskdesc)
+ docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+ env = worker.setdefault('env', {})
+ env.update({
+ 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+ 'MOZ_SCM_LEVEL': config.params['level'],
+ })
+
+ env['MOZCONFIG'] = run['mozconfig']
+ env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
+
+ # tooltool downloads
+ worker['relengapi-proxy'] = True
+ worker['caches'].append({
+ 'type': 'persistent',
+ 'name': 'tooltool-cache',
+ # N.B. different from build.sh
+ # TODO(taskdiff): grepping suggests this isn't used..
+ 'mount-point': '/home/worker/tools/tooltool-cache',
+ })
+ taskdesc['scopes'].extend([
+ 'docker-worker:relengapi-proxy:tooltool.download.public',
+ ])
+ env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+ env['TOOLTOOL_REV'] = 'master'
+
+ worker['command'] = [
+ "/bin/bash",
+ "-c",
+ "checkout-gecko workspace"
+ " && cd ./workspace/gecko/taskcluster/scripts/builder"
+ " && buildbot_step 'Build' ./build-mulet-linux.sh $HOME/workspace",
+ ]
+
+mulet_simulator_schema = Schema({
+ Required('using'): 'mulet-simulator',
+
+ # The shell command to run with `bash -exc`. This will have parameters
+ # substituted for {..} and will be enclosed in a {task-reference: ..} block
+ # so it can refer to the parent task as <build>
+ Required('shell-command'): basestring,
+})
+
+
+@run_job_using("docker-worker", "mulet-simulator", schema=mulet_simulator_schema)
+def docker_worker_mulet_simulator(config, job, taskdesc):
+ run = job['run']
+ worker = taskdesc.get('worker')
+
+ # assumes the builder image (which contains the gecko checkout command)
+ taskdesc['worker']['docker-image'] = {"in-tree": "builder"}
+
+ worker['taskcluster-proxy'] = False
+
+ docker_worker_add_public_artifacts(config, job, taskdesc)
+ docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+ taskdesc.setdefault('routes', []).extend([
+ 'index.gecko.v1.{project}.latest.simulator.opt'.format(**config.params),
+ ])
+
+ # TODO(taskdiff): has the scope for this cache, but not the cache
+ taskdesc.setdefault('scopes', []).extend([
+ 'docker-worker:cache:level-{level}-{project}-tc-vcs'.format(**config.params),
+ ])
+
+ shell_command = run['shell-command'].format(**config.params)
+
+ worker['command'] = [
+ "/bin/bash",
+ "-exc",
+ {'task-reference': shell_command},
+ ]--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -18,16 +18,17 @@ TRY_DELIMITER = 'try:'
# mappings from the single char alias to a longer more recognizable form.
BUILD_TYPE_ALIASES = {
'o': 'opt',
'd': 'debug'
}
# consider anything in this whitelist of kinds to be governed by -b/-p
BUILD_KINDS = set([
+ 'build',
])
# anything in this list is governed by -j
JOB_KINDS = set([
])
# mapping from shortcut name (usable with -u) to a boolean function identifying