Bug 1231320: pull from secrets API in TaskCluster r=mshal r=pmoore r=garndt
This adds a mozharness action, only run in TaskCluster, to fetch secrets from
the TaskCluster secrets API via the TaskCluster Proxy.
It requires that the SCM level of the build be passed in with the --scm-level
argument, defaulting to 1 (try)
MozReview-Commit-ID: C3rvOPF6Bm1
--- a/testing/mozharness/configs/builds/releng_base_android_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_android_64_builds.py
@@ -34,16 +34,20 @@ config = {
('/home/cltbld/.boto', '/builds/.boto'),
('/builds/relengapi.tok', '/builds/relengapi.tok'),
('/tools/tooltool.py', '/builds/tooltool.py'),
('/builds/mozilla-api.key', '/builds/mozilla-api.key'),
('/builds/mozilla-fennec-geoloc-api.key', '/builds/mozilla-fennec-geoloc-api.key'),
('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
],
+ 'secret_files': [
+ {'filename': '/builds/mozilla-fennec-geoloc-api.key',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-fennec-geoloc-api.key'},
+ ],
'enable_ccache': True,
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': False,
'enable_talos_sendchange': True,
'enable_unittest_sendchange': True,
--- a/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
@@ -41,16 +41,26 @@ config = {
('/builds/relengapi.tok', '/builds/relengapi.tok'),
('/tools/tooltool.py', '/builds/tooltool.py'),
('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
],
+ 'secret_files': [
+ {'filename': '/builds/gapi.data',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+ {'filename': '/builds/mozilla-desktop-geoloc-api.key',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+ {'filename': '/builds/adjust-sdk.token',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+ {'filename': '/builds/adjust-sdk-beta.token',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+ ],
'enable_ccache': True,
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': True,
'enable_talos_sendchange': True,
'enable_unittest_sendchange': True,
--- a/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
@@ -41,16 +41,26 @@ config = {
('/builds/relengapi.tok', '/builds/relengapi.tok'),
('/tools/tooltool.py', '/builds/tooltool.py'),
('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
],
+ 'secret_files': [
+ {'filename': '/builds/gapi.data',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+ {'filename': '/builds/mozilla-desktop-geoloc-api.key',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+ {'filename': '/builds/adjust-sdk.token',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+ {'filename': '/builds/adjust-sdk-beta.token',
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+ ],
'enable_ccache': True,
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': True,
'enable_talos_sendchange': True,
'enable_unittest_sendchange': True,
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -38,16 +38,17 @@ from mozharness.mozilla.buildbot import
TBPL_FAILURE,
TBPL_RETRY,
TBPL_WARNING,
TBPL_SUCCESS,
TBPL_WORST_LEVEL_TUPLE,
)
from mozharness.mozilla.purge import PurgeMixin
from mozharness.mozilla.mock import MockMixin
+from mozharness.mozilla.secrets import SecretsMixin
from mozharness.mozilla.signing import SigningMixin
from mozharness.mozilla.mock import ERROR_MSGS as MOCK_ERROR_MSGS
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.testing.unittest import tbox_print_summary
from mozharness.mozilla.updates.balrog import BalrogMixin
from mozharness.mozilla.taskcluster_helper import Taskcluster
from mozharness.base.python import VirtualenvMixin
from mozharness.base.python import InfluxRecordingMixin
@@ -540,16 +541,24 @@ BUILD_BASE_CONFIG_OPTIONS = [
"type": "string",
"dest": "branch",
"help": "This sets the branch we will be building this for."
" If this branch is in branch_specifics.py, update our"
" config with specific keys/values from that. See"
" %s for possibilites" % (
BuildOptionParser.branch_cfg_file,
)}],
+ [['--scm-level'], {
+ "action": "store",
+ "type": "int",
+ "dest": "scm_level",
+ "default": 1,
+ "help": "This sets the SCM level for the branch being built."
+ " See https://www.mozilla.org/en-US/about/"
+ "governance/policies/commit/access-policy/"}],
[['--enable-pgo'], {
"action": "store_true",
"dest": "pgo_build",
"default": False,
"help": "Sets the build to run in PGO mode"}],
[['--enable-nightly'], {
"action": "store_true",
"dest": "nightly_build",
@@ -573,17 +582,17 @@ def generate_build_ID():
def generate_build_UID():
return uuid.uuid4().hex
class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
SigningMixin, VirtualenvMixin, MercurialScript,
- InfluxRecordingMixin):
+ InfluxRecordingMixin, SecretsMixin):
def __init__(self, **kwargs):
# objdir is referenced in _query_abs_dirs() so let's make sure we
# have that attribute before calling BaseScript.__init__
self.objdir = None
super(BuildScript, self).__init__(**kwargs)
# epoch is only here to represent the start of the buildbot build
# that this mozharn script came from. until I can grab bbot's
# status.build.gettime()[0] this will have to do as a rough estimate
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/mozharness/mozilla/secrets.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""Support for fetching secrets from the secrets API
+"""
+
+import os
+import mozharness
+import urllib2
+import json
+from mozharness.base.log import ERROR
+
+
+class SecretsMixin(object):
+
+ def get_secrets(self):
+ """
+ Get the secrets specified by the `secret_files` configuration. This is
+ a list of dictionaries, one for each secret. The `secret_name` key
+ names the key in the TaskCluster secrets API to fetch (see
+ http://docs.taskcluster.net/services/secrets/). It can contain
+ %-substitutions based on the `subst` dictionary below.
+
+ Since secrets must be JSON objects, the `content` property of the
+ secret is used as the value to be written to disk.
+
+ The `filename` key in the dictionary gives the filename to which the
+ secret should be written.
+ """
+ secret_files = self.config.get('secret_files', [])
+
+ subst = {
+ 'scm-level': self.config.get('scm-level', 1),
+ }
+
+ for sf in secret_files:
+ filename = sf['filename']
+ secret_name = sf['secret_name'] % subst
+ self.info("fetching {} from secret {}".format(filename, secret_name))
+
+ # fetch from http://taskcluster, which points to the taskcluster proxy
+ # within a taskcluster task. Outside of that environment, do not
+ # use this action.
+ url = "http://taskcluster/secrets/v1/secret/" + secret_name
+ res = urllib2.urlopen(url)
+ if res.getcode() != 200:
+ self.fatal("Error fetching from secrets API:" + res.read())
+
+ secret = json.load(res)['secret']['content']
+
+ open(filename, "w").write(filename)
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ b/testing/mozharness/scripts/fx_desktop_build.py
@@ -23,16 +23,17 @@ from mozharness.mozilla.building.buildba
BuildingConfig, BuildScript
class FxDesktopBuild(BuildScript, object):
def __init__(self):
buildscript_kwargs = {
'config_options': BUILD_BASE_CONFIG_OPTIONS,
'all_actions': [
+ 'get-secrets',
'clobber',
'clone-tools',
'checkout-sources',
'setup-mock',
'build',
'upload-files', # upload from BB to TC
'sendchange',
'check-test',
--- a/testing/taskcluster/scripts/builder/build-linux.sh
+++ b/testing/taskcluster/scripts/builder/build-linux.sh
@@ -15,16 +15,17 @@ echo "running as" $(id)
: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
: NEED_XVFB ${NEED_XVFB:=false}
: MH_CUSTOM_BUILD_VARIANT_CFG ${MH_CUSTOM_BUILD_VARIANT_CFG}
: MH_BRANCH ${MH_BRANCH:=mozilla-central}
: MH_BUILD_POOL ${MH_BUILD_POOL:=staging}
+: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1}
: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
# some linux variants, e.g. b2gdroid, require gaia
: CHECKOUT_GAIA ${CHECKOUT_GAIA:=false}
set -v
@@ -113,27 +114,23 @@ fi
export TOOLTOOL_CACHE
# support multiple, space delimited, config files
config_cmds=""
for cfg in $MOZHARNESS_CONFIG; do
config_cmds="${config_cmds} --config ${cfg}"
done
-# Mozharness would ordinarily do the checkouts itself, but they are disabled
-# here (--no-checkout-sources, --no-clone-tools) as the checkout is performed above.
+# Mozharness would ordinarily do a whole mess of buildbot-specific steps, but those
+# are overridden by this list of steps. The get-secrets step is unique to TC tasks
+# and not run in Buildbot
+steps="--get-secrets --build --check-test"
python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
$debug_flag \
$custom_build_variant_cfg_flag \
--disable-mock \
- --no-setup-mock \
- --no-checkout-sources \
- --no-clone-tools \
- --no-clobber \
- --no-update \
- --no-upload-files \
- --no-sendchange \
+ $steps \
--log-level=debug \
+ --scm-level=$MOZ_SCM_LEVEL \
--work-dir=$WORKSPACE/build \
- --no-action=generate-build-stats \
--branch=${MH_BRANCH} \
--build-pool=${MH_BUILD_POOL}
--- a/testing/taskcluster/tasks/build.yml
+++ b/testing/taskcluster/tasks/build.yml
@@ -44,16 +44,17 @@ task:
# Common environment variables for checking out gecko
GECKO_BASE_REPOSITORY: '{{base_repository}}'
GECKO_HEAD_REPOSITORY: '{{head_repository}}'
GECKO_HEAD_REV: '{{head_rev}}'
GECKO_HEAD_REF: '{{head_ref}}'
TOOLTOOL_REPO: 'https://git.mozilla.org/build/tooltool.git'
TOOLTOOL_REV: 'master'
MOZ_BUILD_DATE: '{{pushdate}}'
+ MOZ_SCM_LEVEL: '{{level}}'
extra:
build_product: '{{build_product}}'
build_name: '{{build_name}}'
build_type: '{{build_type}}'
index:
rank: {{pushlog_id}}
treeherder:
--- a/testing/taskcluster/tasks/builds/firefox_base.yml
+++ b/testing/taskcluster/tasks/builds/firefox_base.yml
@@ -1,14 +1,18 @@
$inherits:
from: 'tasks/build.yml'
variables:
build_product: 'firefox'
task:
+ scopes:
+ - "secrets:get:project/releng/gecko/build/level-{{level}}/*"
extra:
locations:
mozharness: 'public/build/mozharness.zip'
test_packages: 'public/build/target.test_packages.json'
payload:
image:
type: 'task-image'
path: 'public/image.tar'
taskId: '{{#task_id_for_image}}desktop-build{{/task_id_for_image}}'
+ features:
+ taskclusterProxy: true