Bug 1231320: support min_scm_level and default secret specs; r=mshal r=garndt r=pmoore
This improves support for fetching secrets to skip fetching, optionally
replacing with a default value, for builds based on trees with SCM levels below
a minimum. In practical terms, this allows try builds to operate without secrets.
MozReview-Commit-ID: 5DYSfzkYZQv
--- a/testing/mozharness/configs/builds/releng_base_android_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_android_64_builds.py
@@ -36,17 +36,18 @@ config = {
('/tools/tooltool.py', '/builds/tooltool.py'),
('/builds/mozilla-api.key', '/builds/mozilla-api.key'),
('/builds/mozilla-fennec-geoloc-api.key', '/builds/mozilla-fennec-geoloc-api.key'),
('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
],
'secret_files': [
{'filename': '/builds/mozilla-fennec-geoloc-api.key',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-fennec-geoloc-api.key'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-fennec-geoloc-api.key',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
],
'enable_ccache': True,
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': False,
'enable_talos_sendchange': True,
--- a/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
@@ -43,23 +43,27 @@ config = {
('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
],
'secret_files': [
{'filename': '/builds/gapi.data',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
{'filename': '/builds/mozilla-desktop-geoloc-api.key',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
{'filename': '/builds/adjust-sdk.token',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
{'filename': '/builds/adjust-sdk-beta.token',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
],
'enable_ccache': True,
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': True,
'enable_talos_sendchange': True,
--- a/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
@@ -43,23 +43,27 @@ config = {
('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
],
'secret_files': [
{'filename': '/builds/gapi.data',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
{'filename': '/builds/mozilla-desktop-geoloc-api.key',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
{'filename': '/builds/adjust-sdk.token',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
{'filename': '/builds/adjust-sdk-beta.token',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+ 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token',
+ 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
],
'enable_ccache': True,
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': True,
'enable_talos_sendchange': True,
--- a/testing/mozharness/mozharness/mozilla/secrets.py
+++ b/testing/mozharness/mozharness/mozilla/secrets.py
@@ -11,44 +11,60 @@ import os
import mozharness
import urllib2
import json
from mozharness.base.log import ERROR
class SecretsMixin(object):
+ def _fetch_secret(self, secret_name):
+ self.info("fetching secret {} from API".format(secret_name))
+ # fetch from http://taskcluster, which points to the taskcluster proxy
+ # within a taskcluster task. Outside of that environment, do not
+ # use this action.
+ url = "http://taskcluster/secrets/v1/secret/" + secret_name
+ res = urllib2.urlopen(url)
+ if res.getcode() != 200:
+ self.fatal("Error fetching from secrets API:" + res.read())
+
+ return json.load(res)['secret']['content']
+
def get_secrets(self):
"""
Get the secrets specified by the `secret_files` configuration. This is
a list of dictionaries, one for each secret. The `secret_name` key
names the key in the TaskCluster secrets API to fetch (see
http://docs.taskcluster.net/services/secrets/). It can contain
%-substitutions based on the `subst` dictionary below.
Since secrets must be JSON objects, the `content` property of the
secret is used as the value to be written to disk.
The `filename` key in the dictionary gives the filename to which the
secret should be written.
+
+ The optional `min_scm_level` key gives a minimum SCM level at which this
+ secret is required. For lower levels, the value of the 'default` key
+ is used, or no secret is written.
"""
secret_files = self.config.get('secret_files', [])
+ scm_level = self.config.get('scm-level', 1)
subst = {
- 'scm-level': self.config.get('scm-level', 1),
+ 'scm-level': scm_level,
}
for sf in secret_files:
filename = sf['filename']
secret_name = sf['secret_name'] % subst
- self.info("fetching {} from secret {}".format(filename, secret_name))
+ min_scm_level = sf.get('min_scm_level', 0)
+ if scm_level <= min_scm_level:
+ if 'default' in sf:
+ self.info("Using default value for " + filename)
+ secret = sf['default']
+ else:
+ self.info("No default for secret; not writing " + filename)
+ continue
+ else:
+ secret = self._fetch_secret(secret_name)
- # fetch from http://taskcluster, which points to the taskcluster proxy
- # within a taskcluster task. Outside of that environment, do not
- # use this action.
- url = "http://taskcluster/secrets/v1/secret/" + secret_name
- res = urllib2.urlopen(url)
- if res.getcode() != 200:
- self.fatal("Error fetching from secrets API:" + res.read())
-
- secret = json.load(res)['secret']['content']
-
- open(filename, "w").write(filename)
+ open(filename, "w").write(secret)