Bug 1231320: supply actions to mozharness from the task definition; r?jlund draft
authorDustin J. Mitchell <dustin@mozilla.com>
Wed, 16 Mar 2016 21:11:44 +0000
changeset 341380 73427a49930a1654288cf0e738c1b4032c2f12ea
parent 341379 69944817bb3ea581aa3311c5153ed20dca201c10
child 341802 9f68e61f7c29ed9cf0f81d041a9d24fbb44b20ff
push id13203
push userdmitchell@mozilla.com
push dateWed, 16 Mar 2016 22:40:30 +0000
reviewersjlund
bugs1231320
milestone48.0a1
Bug 1231320: supply actions to mozharness from the task definition; r?jlund MozReview-Commit-ID: GgKpDH6EKRa
testing/mozharness/configs/builds/releng_base_android_64_builds.py
testing/mozharness/configs/builds/releng_base_linux_32_builds.py
testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py
testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py
testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py
testing/taskcluster/scripts/builder/build-linux.sh
testing/taskcluster/tasks/builds/android_api_15.yml
testing/taskcluster/tasks/builds/android_api_15_b2gdroid.yml
testing/taskcluster/tasks/builds/android_api_15_frontend.yml
testing/taskcluster/tasks/builds/android_api_15_gradle_dependencies.yml
testing/taskcluster/tasks/builds/android_api_15_partner_sample1.yml
testing/taskcluster/tasks/builds/linux32_clobber.yml
testing/taskcluster/tasks/builds/linux64_clobber.yml
testing/taskcluster/tasks/builds/opt_linux64_artifact.yml
testing/taskcluster/tasks/builds/opt_linux64_st-an.yml
testing/taskcluster/tasks/builds/opt_linux64_st-an_clobber.yml
testing/taskcluster/tasks/builds/opt_macosx64_clobber.yml
--- a/testing/mozharness/configs/builds/releng_base_android_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_android_64_builds.py
@@ -1,14 +1,15 @@
 import os
 
 config = {
     #########################################################################
     ######## ANDROID GENERIC CONFIG KEYS/VAlUES
 
+    # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
     'default_actions': [
         'clobber',
         'clone-tools',
         'checkout-sources',
         'setup-mock',
         'build',
         'upload-files',
         'sendchange',
--- a/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
@@ -3,16 +3,17 @@ import os
 config = {
     #########################################################################
     ######## LINUX GENERIC CONFIG KEYS/VAlUES
     # if you are updating this with custom 32 bit keys/values please add them
     # below under the '32 bit specific' code block otherwise, update in this
     # code block and also make sure this is synced with
     # releng_base_linux_64_builds.py
 
+    # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
     'default_actions': [
         'clobber',
         'clone-tools',
         'checkout-sources',
         'setup-mock',
         'build',
         'upload-files',
         'sendchange',
--- a/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py
@@ -1,15 +1,16 @@
 import os
 import sys
 
 config = {
     #########################################################################
     ######## MACOSX CROSS GENERIC CONFIG KEYS/VAlUES
 
+    # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
     'default_actions': [
         'clobber',
         'clone-tools',
         'checkout-sources',
         'build',
         'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py
@@ -1,11 +1,12 @@
 import os
 
 config = {
+    # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
     'default_actions': [
         'clobber',
         'clone-tools',
         'checkout-sources',
         'setup-mock',
         'build',
         # 'generate-build-stats',
     ],
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py
@@ -1,11 +1,12 @@
 import os
 
 config = {
+    # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
     'default_actions': [
         'clobber',
         'clone-tools',
         'checkout-sources',
         'setup-mock',
         'build',
         # 'generate-build-stats',
     ],
--- a/testing/taskcluster/scripts/builder/build-linux.sh
+++ b/testing/taskcluster/scripts/builder/build-linux.sh
@@ -7,16 +7,17 @@ echo "running as" $(id)
 ####
 # Taskcluster friendly wrapper for performing fx desktop builds via mozharness.
 ####
 
 # Inputs, with defaults
 
 : MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
 : MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
+: MOZHARNESS_ACTIONS            ${MOZHARNESS_ACTIONS}
 
 : TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
 
 : NEED_XVFB                     ${NEED_XVFB:=false}
 
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
 : MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
 : MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
@@ -114,23 +115,27 @@ fi
 export TOOLTOOL_CACHE
 
 # support multiple, space delimited, config files
 config_cmds=""
 for cfg in $MOZHARNESS_CONFIG; do
   config_cmds="${config_cmds} --config ${cfg}"
 done
 
-# Mozharness would ordinarily do a whole mess of buildbot-specific steps, but those
-# are overridden by this list of steps.  The get-secrets step is unique to TC tasks
-# and not run in Buildbot
-steps="--get-secrets --build --check-test"
+# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions
+# in the mozharness configuration)
+if [ -n "$MOZHARNESS_ACTIONS" ]; then
+    actions=""
+    for action in $MOZHARNESS_ACTIONS; do
+        actions="$actions --$action"
+    done
+fi
 
 python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
   $debug_flag \
   $custom_build_variant_cfg_flag \
   --disable-mock \
-  $steps \
+  $actions \
   --log-level=debug \
   --scm-level=$MOZ_SCM_LEVEL \
   --work-dir=$WORKSPACE/build \
   --branch=${MH_BRANCH} \
   --build-pool=${MH_BUILD_POOL}
--- a/testing/taskcluster/tasks/builds/android_api_15.yml
+++ b/testing/taskcluster/tasks/builds/android_api_15.yml
@@ -31,16 +31,17 @@ task:
     env:
       # inputs to mozharness
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       # TODO: make these additional configuration files go away
       MOZHARNESS_CONFIG: >
           builds/releng_base_android_64_builds.py
           disable_signing.py
           platform_supports_post_upload_to_latest.py
+      MOZHARNESS_ACTIONS: "get-secrets build multi-l10n update"
       MH_CUSTOM_BUILD_VARIANT_CFG: api-15
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
 
     maxRunTime: 36000
 
     command: ["/bin/bash", "bin/build.sh"]
 
--- a/testing/taskcluster/tasks/builds/android_api_15_b2gdroid.yml
+++ b/testing/taskcluster/tasks/builds/android_api_15_b2gdroid.yml
@@ -31,16 +31,17 @@ task:
     env:
       # inputs to mozharness
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       # TODO: make these additional configuration files go away
       MOZHARNESS_CONFIG: >
           builds/releng_base_android_64_builds.py
           disable_signing.py
           platform_supports_post_upload_to_latest.py
+      MOZHARNESS_ACTIONS: 'get-secrets build multi-l10n update'
       MH_CUSTOM_BUILD_VARIANT_CFG: api-15-b2gdroid
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
 
       CHECKOUT_GAIA: true
 
     maxRunTime: 36000
 
--- a/testing/taskcluster/tasks/builds/android_api_15_frontend.yml
+++ b/testing/taskcluster/tasks/builds/android_api_15_frontend.yml
@@ -31,16 +31,17 @@ task:
     env:
       # inputs to mozharness
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       # TODO: make these additional configuration files go away
       MOZHARNESS_CONFIG: >
           builds/releng_base_android_64_builds.py
           disable_signing.py
           platform_supports_post_upload_to_latest.py
+      MOZHARNESS_ACTIONS: "get-secrets build multi-l10n update"
       MH_CUSTOM_BUILD_VARIANT_CFG: api-15-frontend
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       GRADLE_USER_HOME: '/home/worker/workspace/build/src/dotgradle'
 
     maxRunTime: 36000
 
     command: ["/bin/bash", "bin/build.sh"]
--- a/testing/taskcluster/tasks/builds/android_api_15_gradle_dependencies.yml
+++ b/testing/taskcluster/tasks/builds/android_api_15_gradle_dependencies.yml
@@ -34,16 +34,17 @@ task:
     env:
       # inputs to mozharness
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       # TODO: make these additional configuration files go away
       MOZHARNESS_CONFIG: >
           builds/releng_base_android_64_builds.py
           disable_signing.py
           platform_supports_post_upload_to_latest.py
+      MOZHARNESS_ACTIONS: "get-secrets build multi-l10n update"
       MH_CUSTOM_BUILD_VARIANT_CFG: api-15-gradle-dependencies
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       GRADLE_USER_HOME: '/home/worker/workspace/build/src/dotgradle-online'
 
     maxRunTime: 36000
 
     image:
--- a/testing/taskcluster/tasks/builds/android_api_15_partner_sample1.yml
+++ b/testing/taskcluster/tasks/builds/android_api_15_partner_sample1.yml
@@ -31,16 +31,17 @@ task:
     env:
       # inputs to mozharness
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       # TODO: make these additional configuration files go away
       MOZHARNESS_CONFIG: >
           builds/releng_base_android_64_builds.py
           disable_signing.py
           platform_supports_post_upload_to_latest.py
+      MOZHARNESS_ACTIONS: 'get-secrets build multi-l10n update'
       MH_CUSTOM_BUILD_VARIANT_CFG: api-15-partner-sample1
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
 
       # space separated list of repositories required for this build
       # for each ITEM in list you want checked out, you must also supply tc-vcs args:
       # e.g. ${ITEM}_BASE_REPOSITORY
       EXTRA_CHECKOUT_REPOSITORIES: >
--- a/testing/taskcluster/tasks/builds/linux32_clobber.yml
+++ b/testing/taskcluster/tasks/builds/linux32_clobber.yml
@@ -19,16 +19,17 @@ task:
       tooltool-cache: '/home/worker/tooltool-cache'
 
     features:
       relengAPIProxy: true
 
     env:
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       MOZHARNESS_CONFIG: 'builds/releng_base_linux_32_builds.py balrog/production.py'
+      MOZHARNESS_ACTIONS: 'get-secrets build check-test generate-build-stats update'
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
       NEED_XVFB: true
 
     maxRunTime: 36000
 
--- a/testing/taskcluster/tasks/builds/linux64_clobber.yml
+++ b/testing/taskcluster/tasks/builds/linux64_clobber.yml
@@ -15,16 +15,17 @@ task:
       tooltool-cache: '/home/worker/tooltool-cache'
 
     features:
       relengAPIProxy: true
 
     env:
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       MOZHARNESS_CONFIG: 'builds/releng_base_linux_64_builds.py balrog/production.py'
+      MOZHARNESS_ACTIONS: 'get-secrets build check-test generate-build-stats update'
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
       NEED_XVFB: true
 
     maxRunTime: 36000
 
--- a/testing/taskcluster/tasks/builds/opt_linux64_artifact.yml
+++ b/testing/taskcluster/tasks/builds/opt_linux64_artifact.yml
@@ -13,16 +13,17 @@ task:
     - 'index.buildbot.revisions.{{head_rev}}.{{project}}.linux64-artifact'
 
   workerType: opt-linux64
 
   payload:
     env:
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       MOZHARNESS_CONFIG: 'builds/releng_sub_linux_configs/64_artifact.py balrog/production.py'
+      MOZHARNESS_ACTIONS: 'get-secrets build'
       DIST_UPLOADS: ''
       DIST_TARGET_UPLOADS: ''
 
   extra:
     treeherder:
       groupSymbol: "?"
       symbol: AB
       tier: 2
--- a/testing/taskcluster/tasks/builds/opt_linux64_st-an.yml
+++ b/testing/taskcluster/tasks/builds/opt_linux64_st-an.yml
@@ -19,15 +19,16 @@ task:
   workerType: opt-linux64
 
   payload:
     cache:
       level-{{level}}-{{project}}-build-linux64-st-an-workspace: '/home/worker/workspace'
     env:
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       MOZHARNESS_CONFIG: 'builds/releng_sub_linux_configs/64_stat_and_opt.py balrog/production.py'
+      MOZHARNESS_ACTIONS: 'get-secrets build'
       DIST_UPLOADS: ''
       DIST_TARGET_UPLOADS: ''
 
   extra:
     treeherder:
       groupSymbol: "?"
       symbol: S
--- a/testing/taskcluster/tasks/builds/opt_linux64_st-an_clobber.yml
+++ b/testing/taskcluster/tasks/builds/opt_linux64_st-an_clobber.yml
@@ -13,15 +13,16 @@ task:
     - 'index.buildbot.revisions.{{head_rev}}.{{project}}.linux64-st-an'
 
   workerType: opt-linux64
 
   payload:
     env:
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       MOZHARNESS_CONFIG: 'builds/releng_sub_linux_configs/64_stat_and_opt.py balrog/production.py'
+      MOZHARNESS_ACTIONS: 'get-secrets build'
       DIST_UPLOADS: ''
       DIST_TARGET_UPLOADS: ''
 
   extra:
     treeherder:
       groupSymbol: "?"
       symbol: S
--- a/testing/taskcluster/tasks/builds/opt_macosx64_clobber.yml
+++ b/testing/taskcluster/tasks/builds/opt_macosx64_clobber.yml
@@ -20,16 +20,17 @@ task:
       tooltool-cache: '/home/worker/tooltool-cache'
 
     features:
       relengAPIProxy: true
 
     env:
       MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
       MOZHARNESS_CONFIG: 'builds/releng_base_mac_64_cross_builds.py balrog/production.py'
+      MOZHARNESS_ACTIONS: 'get-secrets build generate-build-stats update'
       MH_BRANCH: {{project}}
       MH_BUILD_POOL: taskcluster
       # image paths
       TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
 
     maxRunTime: 36000
 
     command: ["/bin/bash", "bin/build.sh"]