Bug 1421163: Allow passing configuration to mozharness directly from taskcluster configuration; r?jlund draft
authorTom Prince <mozilla@hocat.ca>
Mon, 27 Nov 2017 21:33:13 -0700
changeset 704048 265a4c93b233ccfa8919bf9f0789c6050dda9db5
parent 703864 59f1687a8e254329636154093f0a9b4388f9a9a9
child 704053 c586d9bb67ad18eb23101d36392ca5372d08282a
push id91049
push userbmo:mozilla@hocat.ca
push dateTue, 28 Nov 2017 06:06:00 +0000
reviewersjlund
bugs1421163
milestone59.0a1
Bug 1421163: Allow passing configuration to mozharness directly from taskcluster configuration; r?jlund MozReview-Commit-ID: 1ZeI89hHppD
taskcluster/ci/build/linux.yml
taskcluster/ci/build/macosx.yml
taskcluster/ci/build/windows.yml
taskcluster/taskgraph/transforms/job/mozharness.py
testing/mozharness/configs/builds/releng_sub_linux_configs/32_dmd.py
testing/mozharness/configs/builds/releng_sub_linux_configs/64_dmd.py
testing/mozharness/configs/builds/releng_sub_mac_configs/64_dmd.py
testing/mozharness/configs/builds/releng_sub_windows_configs/32_dmd.py
testing/mozharness/configs/builds/releng_sub_windows_configs/64_dmd.py
testing/mozharness/mozharness/base/config.py
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -44,17 +44,18 @@ linux64-dmd/opt:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
         script: "mozharness/scripts/fx_desktop_build.py"
-        custom-build-variant-cfg: dmd
+        extra-config:
+            mozconfig_variant: 'opt-dmd'
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
     run-on-projects: []
     toolchains:
         - linux64-clang
         - linux64-gcc
         - linux64-rust
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -77,17 +77,18 @@ macosx64-dmd/opt:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
     run:
         using: mozharness
         actions: [get-secrets build update]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_mac_64_cross_builds.py
         script: "mozharness/scripts/fx_desktop_build.py"
-        custom-build-variant-cfg: dmd
+        extra-config:
+            mozconfig_variant: 'opt-dmd'
         secrets: true
         tooltool-downloads: internal
     run-on-projects: []
     toolchains:
         - linux64-cctools-port
         - linux64-clang
         - linux64-hfsplus
         - linux64-libdmg
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -72,17 +72,18 @@ win32-dmd/opt:
         using: mozharness
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/opt.py
-        custom-build-variant-cfg: dmd
+        extra-config:
+            mozconfig_variant: 'opt-dmd'
     run-on-projects: []
     toolchains:
         - win32-clang-cl
         - win32-rust
         - win64-sccache
 
 win32/pgo:
     description: "Win32 Opt PGO"
@@ -186,17 +187,18 @@ win64-dmd/opt:
         using: mozharness
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win64.py
             - builds/taskcluster_sub_win64/opt.py
-        custom-build-variant-cfg: dmd
+        extra-config:
+            mozconfig_variant: 'opt-dmd'
     run-on-projects: []
     toolchains:
         - win64-clang-cl
         - win64-rust
         - win64-sccache
 
 win32-nightly/opt:
     description: "Win32 Nightly"
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -4,16 +4,17 @@
 """
 
 Support for running jobs via mozharness.  Ideally, most stuff gets run this
 way, and certainly anything using mozharness should use this approach.
 
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
+import json
 import slugid
 
 from textwrap import dedent
 
 from taskgraph.util.schema import Schema
 from voluptuous import Required, Optional, Any
 
 from taskgraph.transforms.job import run_job_using
@@ -47,16 +48,19 @@ mozharness_run_schema = Schema({
     Optional('actions'): [basestring],
 
     # any additional options (without leading --) to be passed to mozharness
     Optional('options'): [basestring],
 
     # --custom-build-variant-cfg value
     Optional('custom-build-variant-cfg'): basestring,
 
+    # Extra configuration options to pass to mozharness.
+    Optional('extra-config'): dict,
+
     # Extra metadata to use toward the workspace caching.
     # Only supported on docker-worker
     Optional('extra-workspace-cache-key'): basestring,
 
     # If not false, tooltool downloads will be enabled via relengAPIProxy
     # for either just public files, or all files.  Not supported on Windows
     Required('tooltool-downloads', default=False): Any(
         False,
@@ -146,16 +150,19 @@ def mozharness_on_docker_worker_setup(co
         env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
 
     if 'config-paths' in run:
         env['MOZHARNESS_CONFIG_PATHS'] = ' '.join(run['config-paths'])
 
     if 'custom-build-variant-cfg' in run:
         env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
 
+    if 'extra-config' in run:
+        env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(run['extra-config'])
+
     if 'job-script' in run:
         env['JOB_SCRIPT'] = run['job-script']
 
     if 'try' in config.params['project']:
         env['TRY_COMMIT_MSG'] = config.params['message']
 
     # if we're not keeping artifacts, set some env variables to empty values
     # that will cause the build process to skip copying the results to the
@@ -224,16 +231,19 @@ def mozharness_on_generic_worker(config,
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'MOZ_AUTOMATION': '1',
     })
     if run['use-simple-package']:
         env.update({'MOZ_SIMPLE_PACKAGE_NAME': 'target'})
 
+    if 'extra-config' in run:
+        env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(run['extra-config'])
+
     # The windows generic worker uses batch files to pass environment variables
     # to commands.  Setting a variable to empty in a batch file unsets, so if
     # there is no `TRY_COMMIT_MESSAGE`, pass a space instead, so that
     # mozharness doesn't try to find the commit message on its own.
     if 'try' in config.params['project']:
         env['TRY_COMMIT_MSG'] = config.params['message'] or 'no commit message'
 
     if not job['attributes']['build_platform'].startswith('win'):
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
-    'mozconfig_variant': 'opt-dmd',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
-    'mozconfig_variant': 'opt-dmd',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
-    'mozconfig_variant': 'opt-dmd',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
-    'mozconfig_variant': 'opt-dmd',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_dmd.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
-    'mozconfig_variant': 'opt-dmd',
-}
--- a/testing/mozharness/mozharness/base/config.py
+++ b/testing/mozharness/mozharness/base/config.py
@@ -459,16 +459,21 @@ class BaseConfig(object):
                     )
             except Exception:
                 if cf in options.opt_config_files:
                     print(
                         "WARNING: optional config file not found %s" % cf
                     )
                 else:
                     raise
+
+        if 'EXTRA_MOZHARNESS_CONFIG' in os.environ:
+            env_config = json.loads(os.environ['EXTRA_MOZHARNESS_CONFIG'])
+            all_cfg_files_and_dicts.append(("[EXTRA_MOZHARENSS_CONFIG]", env_config))
+
         return all_cfg_files_and_dicts
 
     def parse_args(self, args=None):
         """Parse command line arguments in a generic way.
         Return the parser object after adding the basic options, so
         child objects can manipulate it.
         """
         self.command_line = ' '.join(sys.argv)