Bug 1231320: support min_scm_level and default secret specs; r=mshal r=garndt r=pmoore draft
authorDustin J. Mitchell <dustin@mozilla.com>
Tue, 15 Mar 2016 16:10:42 +0000
changeset 341379 69944817bb3ea581aa3311c5153ed20dca201c10
parent 341378 0ebfbdcb7fcd91da9aaea9d3b422a7117bb2fb2b
child 341380 73427a49930a1654288cf0e738c1b4032c2f12ea
push id13203
push userdmitchell@mozilla.com
push dateWed, 16 Mar 2016 22:40:30 +0000
reviewersmshal, garndt, pmoore
bugs1231320
milestone48.0a1
Bug 1231320: support min_scm_level and default secret specs; r=mshal r=garndt r=pmoore This improves support for fetching secrets to skip fetching, optionally replacing with a default value, for builds based on trees with SCM levels below a minimum. In practical terms, this allows try builds to operate without secrets. MozReview-Commit-ID: 5DYSfzkYZQv
testing/mozharness/configs/builds/releng_base_android_64_builds.py
testing/mozharness/configs/builds/releng_base_linux_32_builds.py
testing/mozharness/configs/builds/releng_base_linux_64_builds.py
testing/mozharness/mozharness/mozilla/secrets.py
--- a/testing/mozharness/configs/builds/releng_base_android_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_android_64_builds.py
@@ -36,17 +36,18 @@ config = {
         ('/tools/tooltool.py', '/builds/tooltool.py'),
         ('/builds/mozilla-api.key', '/builds/mozilla-api.key'),
         ('/builds/mozilla-fennec-geoloc-api.key', '/builds/mozilla-fennec-geoloc-api.key'),
         ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
     'secret_files': [
         {'filename': '/builds/mozilla-fennec-geoloc-api.key',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-fennec-geoloc-api.key'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-fennec-geoloc-api.key',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
     ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_bootstrap': "setup.sh",
     'enable_count_ctors': False,
     'enable_talos_sendchange': True,
--- a/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
@@ -43,23 +43,27 @@ config = {
         ('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
         ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
         ('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
         ('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
     'secret_files': [
         {'filename': '/builds/gapi.data',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
         {'filename': '/builds/mozilla-desktop-geoloc-api.key',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
         {'filename': '/builds/adjust-sdk.token',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
         {'filename': '/builds/adjust-sdk-beta.token',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
     ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_bootstrap': "setup.sh",
     'enable_count_ctors': True,
     'enable_talos_sendchange': True,
--- a/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
@@ -43,23 +43,27 @@ config = {
         ('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
         ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
         ('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
         ('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
     'secret_files': [
         {'filename': '/builds/gapi.data',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
         {'filename': '/builds/mozilla-desktop-geoloc-api.key',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
         {'filename': '/builds/adjust-sdk.token',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
         {'filename': '/builds/adjust-sdk-beta.token',
-         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
     ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_bootstrap': "setup.sh",
     'enable_count_ctors': True,
     'enable_talos_sendchange': True,
--- a/testing/mozharness/mozharness/mozilla/secrets.py
+++ b/testing/mozharness/mozharness/mozilla/secrets.py
@@ -11,44 +11,60 @@ import os
 import mozharness
 import urllib2
 import json
 from mozharness.base.log import ERROR
 
 
 class SecretsMixin(object):
 
+    def _fetch_secret(self, secret_name):
+        self.info("fetching secret {} from API".format(secret_name))
+        # fetch from http://taskcluster, which points to the taskcluster proxy
+        # within a taskcluster task.  Outside of that environment, do not
+        # use this action.
+        url = "http://taskcluster/secrets/v1/secret/" + secret_name
+        res = urllib2.urlopen(url)
+        if res.getcode() != 200:
+            self.fatal("Error fetching from secrets API:" + res.read())
+
+        return json.load(res)['secret']['content']
+
     def get_secrets(self):
         """
         Get the secrets specified by the `secret_files` configuration.  This is
         a list of dictionaries, one for each secret.  The `secret_name` key
         names the key in the TaskCluster secrets API to fetch (see
         http://docs.taskcluster.net/services/secrets/).  It can contain
         %-substitutions based on the `subst` dictionary below.
 
         Since secrets must be JSON objects, the `content` property of the
         secret is used as the value to be written to disk.
 
         The `filename` key in the dictionary gives the filename to which the
         secret should be written.
+
+        The optional `min_scm_level` key gives a minimum SCM level at which this
+        secret is required.  For lower levels, the value of the 'default` key
+        is used, or no secret is written.
         """
         secret_files = self.config.get('secret_files', [])
 
+        scm_level = self.config.get('scm-level', 1)
         subst = {
-            'scm-level': self.config.get('scm-level', 1),
+            'scm-level': scm_level,
         }
 
         for sf in secret_files:
             filename = sf['filename']
             secret_name = sf['secret_name'] % subst
-            self.info("fetching {} from secret {}".format(filename, secret_name))
+            min_scm_level = sf.get('min_scm_level', 0)
+            if scm_level <= min_scm_level:
+                if 'default' in sf:
+                    self.info("Using default value for " + filename)
+                    secret = sf['default']
+                else:
+                    self.info("No default for secret; not writing " + filename)
+                    continue
+            else:
+                secret = self._fetch_secret(secret_name)
 
-            # fetch from http://taskcluster, which points to the taskcluster proxy
-            # within a taskcluster task.  Outside of that environment, do not
-            # use this action.
-            url = "http://taskcluster/secrets/v1/secret/" + secret_name
-            res = urllib2.urlopen(url)
-            if res.getcode() != 200:
-                self.fatal("Error fetching from secrets API:" + res.read())
-
-            secret = json.load(res)['secret']['content']
-
-            open(filename, "w").write(filename)
+            open(filename, "w").write(secret)