Bug 1421163: Allow passing configuration to mozharness directly from taskcluster configuration; r?jlund
MozReview-Commit-ID: 1ZeI89hHppD
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -44,17 +44,18 @@ linux64-dmd/opt:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
run:
using: mozharness
actions: [get-secrets build check-test update]
config:
- builds/releng_base_firefox.py
- builds/releng_base_linux_64_builds.py
script: "mozharness/scripts/fx_desktop_build.py"
- custom-build-variant-cfg: dmd
+ extra-config:
+ mozconfig_variant: 'opt-dmd'
secrets: true
tooltool-downloads: public
need-xvfb: true
run-on-projects: []
toolchains:
- linux64-clang
- linux64-gcc
- linux64-rust
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -77,17 +77,18 @@ macosx64-dmd/opt:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
run:
using: mozharness
actions: [get-secrets build update]
config:
- builds/releng_base_firefox.py
- builds/releng_base_mac_64_cross_builds.py
script: "mozharness/scripts/fx_desktop_build.py"
- custom-build-variant-cfg: dmd
+ extra-config:
+ mozconfig_variant: 'opt-dmd'
secrets: true
tooltool-downloads: internal
run-on-projects: []
toolchains:
- linux64-cctools-port
- linux64-clang
- linux64-hfsplus
- linux64-libdmg
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -72,17 +72,18 @@ win32-dmd/opt:
using: mozharness
options: [append-env-variables-from-configs]
script: mozharness/scripts/fx_desktop_build.py
config:
- builds/releng_base_firefox.py
- builds/taskcluster_base_windows.py
- builds/taskcluster_base_win32.py
- builds/taskcluster_sub_win32/opt.py
- custom-build-variant-cfg: dmd
+ extra-config:
+ mozconfig_variant: 'opt-dmd'
run-on-projects: []
toolchains:
- win32-clang-cl
- win32-rust
- win64-sccache
win32/pgo:
description: "Win32 Opt PGO"
@@ -186,17 +187,18 @@ win64-dmd/opt:
using: mozharness
options: [append-env-variables-from-configs]
script: mozharness/scripts/fx_desktop_build.py
config:
- builds/releng_base_firefox.py
- builds/taskcluster_base_windows.py
- builds/taskcluster_base_win64.py
- builds/taskcluster_sub_win64/opt.py
- custom-build-variant-cfg: dmd
+ extra-config:
+ mozconfig_variant: 'opt-dmd'
run-on-projects: []
toolchains:
- win64-clang-cl
- win64-rust
- win64-sccache
win32-nightly/opt:
description: "Win32 Nightly"
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -4,16 +4,17 @@
"""
Support for running jobs via mozharness. Ideally, most stuff gets run this
way, and certainly anything using mozharness should use this approach.
"""
from __future__ import absolute_import, print_function, unicode_literals
+import json
import slugid
from textwrap import dedent
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional, Any
from taskgraph.transforms.job import run_job_using
@@ -47,16 +48,19 @@ mozharness_run_schema = Schema({
Optional('actions'): [basestring],
# any additional options (without leading --) to be passed to mozharness
Optional('options'): [basestring],
# --custom-build-variant-cfg value
Optional('custom-build-variant-cfg'): basestring,
+ # Extra configuration options to pass to mozharness.
+ Optional('extra-config'): dict,
+
# Extra metadata to use toward the workspace caching.
# Only supported on docker-worker
Optional('extra-workspace-cache-key'): basestring,
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads', default=False): Any(
False,
@@ -146,16 +150,19 @@ def mozharness_on_docker_worker_setup(co
env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
if 'config-paths' in run:
env['MOZHARNESS_CONFIG_PATHS'] = ' '.join(run['config-paths'])
if 'custom-build-variant-cfg' in run:
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
+ if 'extra-config' in run:
+ env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(run['extra-config'])
+
if 'job-script' in run:
env['JOB_SCRIPT'] = run['job-script']
if 'try' in config.params['project']:
env['TRY_COMMIT_MSG'] = config.params['message']
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
@@ -224,16 +231,19 @@ def mozharness_on_generic_worker(config,
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_AUTOMATION': '1',
})
if run['use-simple-package']:
env.update({'MOZ_SIMPLE_PACKAGE_NAME': 'target'})
+ if 'extra-config' in run:
+ env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(run['extra-config'])
+
# The windows generic worker uses batch files to pass environment variables
# to commands. Setting a variable to empty in a batch file unsets, so if
# there is no `TRY_COMMIT_MESSAGE`, pass a space instead, so that
# mozharness doesn't try to find the commit message on its own.
if 'try' in config.params['project']:
env['TRY_COMMIT_MSG'] = config.params['message'] or 'no commit message'
if not job['attributes']['build_platform'].startswith('win'):
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- 'mozconfig_variant': 'opt-dmd',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- 'mozconfig_variant': 'opt-dmd',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- 'mozconfig_variant': 'opt-dmd',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- 'mozconfig_variant': 'opt-dmd',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- 'mozconfig_variant': 'opt-dmd',
-}
--- a/testing/mozharness/mozharness/base/config.py
+++ b/testing/mozharness/mozharness/base/config.py
@@ -459,16 +459,21 @@ class BaseConfig(object):
)
except Exception:
if cf in options.opt_config_files:
print(
"WARNING: optional config file not found %s" % cf
)
else:
raise
+
+ if 'EXTRA_MOZHARNESS_CONFIG' in os.environ:
+ env_config = json.loads(os.environ['EXTRA_MOZHARNESS_CONFIG'])
+ all_cfg_files_and_dicts.append(("[EXTRA_MOZHARENSS_CONFIG]", env_config))
+
return all_cfg_files_and_dicts
def parse_args(self, args=None):
"""Parse command line arguments in a generic way.
Return the parser object after adding the basic options, so
child objects can manipulate it.
"""
self.command_line = ' '.join(sys.argv)