Bug 1415618: Add support to taskcluster to specifying extra paths to mozharness. r=jlund draft
authorTom Prince <mozilla@hocat.ca>
Tue, 14 Nov 2017 15:16:22 -0700
changeset 698844 2a184e66c3413e9e206b999b1340a6ecb52bd234
parent 698843 e80f0f20213e76ee68e62757eab8c788b7d398c6
child 740446 34d5947d8342bf5116f2cee1fdcb9ef0aef50e7f
push id89369
push userbmo:mozilla@hocat.ca
push dateThu, 16 Nov 2017 04:26:10 +0000
reviewersjlund
bugs1415618
milestone59.0a1
Bug 1415618: Add support to taskcluster to specifying extra paths to mozharness. r=jlund MozReview-Commit-ID: 7CGQgSUUg0n
taskcluster/scripts/builder/build-linux.sh
taskcluster/taskgraph/transforms/job/mozharness.py
--- a/taskcluster/scripts/builder/build-linux.sh
+++ b/taskcluster/scripts/builder/build-linux.sh
@@ -7,16 +7,17 @@ echo "running as" $(id)
 ####
 # Taskcluster friendly wrapper for performing fx desktop builds via mozharness.
 ####
 
 # Inputs, with defaults
 
 : MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
 : MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
+: MOZHARNESS_CONFIG_PATHS       ${MOZHARNESS_CONFIG_PATHS}
 : MOZHARNESS_ACTIONS            ${MOZHARNESS_ACTIONS}
 : MOZHARNESS_OPTIONS            ${MOZHARNESS_OPTIONS}
 
 : TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache}
 
 : NEED_XVFB                     ${NEED_XVFB:=false}
 
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
@@ -83,16 +84,21 @@ if [ -n "${MH_CUSTOM_BUILD_VARIANT_CFG}"
     custom_build_variant_cfg_flag="--custom-build-variant-cfg=${MH_CUSTOM_BUILD_VARIANT_CFG}"
 fi
 
 # $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the
 # cache.  However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be
 # entirely effective.
 export TOOLTOOL_CACHE
 
+config_path_cmds=""
+for path in ${MOZHARNESS_CONFIG_PATHS}; do
+    config_path_cmds="${config_path_cmds} --extra-config-path ${WORKSPACE}/build/src/${path}"
+done
+
 # support multiple, space delimited, config files
 config_cmds=""
 for cfg in $MOZHARNESS_CONFIG; do
   config_cmds="${config_cmds} --config ${cfg}"
 done
 
 # if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions
 # in the mozharness configuration)
@@ -109,17 +115,19 @@ if [ -n "$MOZHARNESS_OPTIONS" ]; then
     options=""
     for option in $MOZHARNESS_OPTIONS; do
         options="$options --$option"
     done
 fi
 
 cd /builds/worker
 
-python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
+python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} \
+  ${config_path_cmds} \
+  ${config_cmds} \
   $debug_flag \
   $custom_build_variant_cfg_flag \
   --disable-mock \
   $actions \
   $options \
   --log-level=debug \
   --scm-level=$MOZ_SCM_LEVEL \
   --work-dir=$WORKSPACE/build \
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -29,18 +29,23 @@ from taskgraph.transforms.job.common imp
 
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
     # directory and using forward slashes even on Windows
     Required('script'): basestring,
 
+    # Additional paths to look for mozharness configs in. These should be
+    # relative to the base of the source checkout
+    Optional('config-paths'): [basestring],
+
     # the config files required for the task, relative to
-    # testing/mozharness/configs and using forward slashes even on Windows
+    # testing/mozharness/configs or one of the paths specified in
+    # `config-paths` and using forward slashes even on Windows
     Required('config'): [basestring],
 
     # any additional actions to pass to the mozharness command
     Optional('actions'): [basestring],
 
     # any additional options (without leading --) to be passed to mozharness
     Optional('options'): [basestring],
 
@@ -135,16 +140,19 @@ def mozharness_on_docker_worker_setup(co
     })
 
     if 'actions' in run:
         env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
 
     if 'options' in run:
         env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
 
+    if 'config-paths' in run:
+        env['MOZHARNESS_CONFIG_PATHS'] = ' '.join(run['config-paths'])
+
     if 'custom-build-variant-cfg' in run:
         env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
 
     if 'job-script' in run:
         env['JOB_SCRIPT'] = run['job-script']
 
     if 'try' in config.params['project']:
         env['TRY_COMMIT_MSG'] = config.params['message']
@@ -230,16 +238,22 @@ def mozharness_on_generic_worker(config,
 
     if not job['attributes']['build_platform'].startswith('win'):
         raise Exception(
             "Task generation for mozharness build jobs currently only supported on Windows"
         )
 
     mh_command = [r'c:\mozilla-build\python\python.exe']
     mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')]))
+
+    if 'config-paths' in run:
+        for path in run['config-paths']:
+            mh_command.append(r'--extra-config-path '
+                              r'.\build\src\{}'.format(path.replace('/', '\\')))
+
     for cfg in run['config']:
         mh_command.append('--config ' + cfg.replace('/', '\\'))
     if run['use-magic-mh-args']:
         mh_command.append('--branch ' + config.params['project'])
         mh_command.append(r'--skip-buildbot-actions')
     mh_command.append(r'--work-dir %cd:Z:=z:%\build')
     for action in run.get('actions', []):
         assert ' ' not in action