Bug 1231320: pull from secrets API in TaskCluster r=mshal r=pmoore r=garndt draft
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 14 Mar 2016 22:17:18 +0000
changeset 341378 0ebfbdcb7fcd91da9aaea9d3b422a7117bb2fb2b
parent 341377 fd6c1f7f3e4eef20b5f08106b8ac73599bdc8c88
child 341379 69944817bb3ea581aa3311c5153ed20dca201c10
push id13203
push userdmitchell@mozilla.com
push dateWed, 16 Mar 2016 22:40:30 +0000
reviewersmshal, pmoore, garndt
bugs1231320
milestone48.0a1
Bug 1231320: pull from secrets API in TaskCluster r=mshal r=pmoore r=garndt This adds a mozharness action, only run in TaskCluster, to fetch secrets from the TaskCluster secrets API via the TaskCluster Proxy. It requires that the SCM level of the build be passed in with the --scm-level argument, defaulting to 1 (try) MozReview-Commit-ID: C3rvOPF6Bm1
testing/mozharness/configs/builds/releng_base_android_64_builds.py
testing/mozharness/configs/builds/releng_base_linux_32_builds.py
testing/mozharness/configs/builds/releng_base_linux_64_builds.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/mozharness/mozharness/mozilla/secrets.py
testing/mozharness/scripts/fx_desktop_build.py
testing/taskcluster/scripts/builder/build-linux.sh
testing/taskcluster/tasks/build.yml
testing/taskcluster/tasks/builds/firefox_base.yml
--- a/testing/mozharness/configs/builds/releng_base_android_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_android_64_builds.py
@@ -34,16 +34,20 @@ config = {
         ('/home/cltbld/.boto', '/builds/.boto'),
         ('/builds/relengapi.tok', '/builds/relengapi.tok'),
         ('/tools/tooltool.py', '/builds/tooltool.py'),
         ('/builds/mozilla-api.key', '/builds/mozilla-api.key'),
         ('/builds/mozilla-fennec-geoloc-api.key', '/builds/mozilla-fennec-geoloc-api.key'),
         ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
+    'secret_files': [
+        {'filename': '/builds/mozilla-fennec-geoloc-api.key',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-fennec-geoloc-api.key'},
+    ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_bootstrap': "setup.sh",
     'enable_count_ctors': False,
     'enable_talos_sendchange': True,
     'enable_unittest_sendchange': True,
--- a/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
@@ -41,16 +41,26 @@ config = {
         ('/builds/relengapi.tok', '/builds/relengapi.tok'),
         ('/tools/tooltool.py', '/builds/tooltool.py'),
         ('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
         ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
         ('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
         ('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
+    'secret_files': [
+        {'filename': '/builds/gapi.data',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+        {'filename': '/builds/mozilla-desktop-geoloc-api.key',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+        {'filename': '/builds/adjust-sdk.token',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+        {'filename': '/builds/adjust-sdk-beta.token',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+    ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_bootstrap': "setup.sh",
     'enable_count_ctors': True,
     'enable_talos_sendchange': True,
     'enable_unittest_sendchange': True,
--- a/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
@@ -41,16 +41,26 @@ config = {
         ('/builds/relengapi.tok', '/builds/relengapi.tok'),
         ('/tools/tooltool.py', '/builds/tooltool.py'),
         ('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
         ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
         ('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
         ('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
+    'secret_files': [
+        {'filename': '/builds/gapi.data',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+        {'filename': '/builds/mozilla-desktop-geoloc-api.key',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+        {'filename': '/builds/adjust-sdk.token',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+        {'filename': '/builds/adjust-sdk-beta.token',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+    ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_bootstrap': "setup.sh",
     'enable_count_ctors': True,
     'enable_talos_sendchange': True,
     'enable_unittest_sendchange': True,
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -38,16 +38,17 @@ from mozharness.mozilla.buildbot import 
     TBPL_FAILURE,
     TBPL_RETRY,
     TBPL_WARNING,
     TBPL_SUCCESS,
     TBPL_WORST_LEVEL_TUPLE,
 )
 from mozharness.mozilla.purge import PurgeMixin
 from mozharness.mozilla.mock import MockMixin
+from mozharness.mozilla.secrets import SecretsMixin
 from mozharness.mozilla.signing import SigningMixin
 from mozharness.mozilla.mock import ERROR_MSGS as MOCK_ERROR_MSGS
 from mozharness.mozilla.testing.errors import TinderBoxPrintRe
 from mozharness.mozilla.testing.unittest import tbox_print_summary
 from mozharness.mozilla.updates.balrog import BalrogMixin
 from mozharness.mozilla.taskcluster_helper import Taskcluster
 from mozharness.base.python import VirtualenvMixin
 from mozharness.base.python import InfluxRecordingMixin
@@ -540,16 +541,24 @@ BUILD_BASE_CONFIG_OPTIONS = [
         "type": "string",
         "dest": "branch",
         "help": "This sets the branch we will be building this for."
                 " If this branch is in branch_specifics.py, update our"
                 " config with specific keys/values from that. See"
                 " %s for possibilites" % (
                     BuildOptionParser.branch_cfg_file,
                 )}],
+    [['--scm-level'], {
+        "action": "store",
+        "type": "int",
+        "dest": "scm_level",
+        "default": 1,
+        "help": "This sets the SCM level for the branch being built."
+                " See https://www.mozilla.org/en-US/about/"
+                "governance/policies/commit/access-policy/"}],
     [['--enable-pgo'], {
         "action": "store_true",
         "dest": "pgo_build",
         "default": False,
         "help": "Sets the build to run in PGO mode"}],
     [['--enable-nightly'], {
         "action": "store_true",
         "dest": "nightly_build",
@@ -573,17 +582,17 @@ def generate_build_ID():
 
 
 def generate_build_UID():
     return uuid.uuid4().hex
 
 
 class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
                   SigningMixin, VirtualenvMixin, MercurialScript,
-                  InfluxRecordingMixin):
+                  InfluxRecordingMixin, SecretsMixin):
     def __init__(self, **kwargs):
         # objdir is referenced in _query_abs_dirs() so let's make sure we
         # have that attribute before calling BaseScript.__init__
         self.objdir = None
         super(BuildScript, self).__init__(**kwargs)
         # epoch is only here to represent the start of the buildbot build
         # that this mozharn script came from. until I can grab bbot's
         # status.build.gettime()[0] this will have to do as a rough estimate
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/mozharness/mozilla/secrets.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""Support for fetching secrets from the secrets API
+"""
+
+import os
+import mozharness
+import urllib2
+import json
+from mozharness.base.log import ERROR
+
+
+class SecretsMixin(object):
+
+    def get_secrets(self):
+        """
+        Get the secrets specified by the `secret_files` configuration.  This is
+        a list of dictionaries, one for each secret.  The `secret_name` key
+        names the key in the TaskCluster secrets API to fetch (see
+        http://docs.taskcluster.net/services/secrets/).  It can contain
+        %-substitutions based on the `subst` dictionary below.
+
+        Since secrets must be JSON objects, the `content` property of the
+        secret is used as the value to be written to disk.
+
+        The `filename` key in the dictionary gives the filename to which the
+        secret should be written.
+        """
+        secret_files = self.config.get('secret_files', [])
+
+        subst = {
+            'scm-level': self.config.get('scm-level', 1),
+        }
+
+        for sf in secret_files:
+            filename = sf['filename']
+            secret_name = sf['secret_name'] % subst
+            self.info("fetching {} from secret {}".format(filename, secret_name))
+
+            # fetch from http://taskcluster, which points to the taskcluster proxy
+            # within a taskcluster task.  Outside of that environment, do not
+            # use this action.
+            url = "http://taskcluster/secrets/v1/secret/" + secret_name
+            res = urllib2.urlopen(url)
+            if res.getcode() != 200:
+                self.fatal("Error fetching from secrets API:" + res.read())
+
+            secret = json.load(res)['secret']['content']
+
+            open(filename, "w").write(filename)
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ b/testing/mozharness/scripts/fx_desktop_build.py
@@ -23,16 +23,17 @@ from mozharness.mozilla.building.buildba
     BuildingConfig, BuildScript
 
 
 class FxDesktopBuild(BuildScript, object):
     def __init__(self):
         buildscript_kwargs = {
             'config_options': BUILD_BASE_CONFIG_OPTIONS,
             'all_actions': [
+                'get-secrets',
                 'clobber',
                 'clone-tools',
                 'checkout-sources',
                 'setup-mock',
                 'build',
                 'upload-files',  # upload from BB to TC
                 'sendchange',
                 'check-test',
--- a/testing/taskcluster/scripts/builder/build-linux.sh
+++ b/testing/taskcluster/scripts/builder/build-linux.sh
@@ -15,16 +15,17 @@ echo "running as" $(id)
 
 : TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
 
 : NEED_XVFB                     ${NEED_XVFB:=false}
 
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
 : MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
 : MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
+: MOZ_SCM_LEVEL                 ${MOZ_SCM_LEVEL:=1}
 
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
 # some linux variants, e.g. b2gdroid, require gaia
 : CHECKOUT_GAIA                      ${CHECKOUT_GAIA:=false}
 
 set -v
 
@@ -113,27 +114,23 @@ fi
 export TOOLTOOL_CACHE
 
 # support multiple, space delimited, config files
 config_cmds=""
 for cfg in $MOZHARNESS_CONFIG; do
   config_cmds="${config_cmds} --config ${cfg}"
 done
 
-# Mozharness would ordinarily do the checkouts itself, but they are disabled
-# here (--no-checkout-sources, --no-clone-tools) as the checkout is performed above.
+# Mozharness would ordinarily do a whole mess of buildbot-specific steps, but those
+# are overridden by this list of steps.  The get-secrets step is unique to TC tasks
+# and not run in Buildbot
+steps="--get-secrets --build --check-test"
 
 python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
   $debug_flag \
   $custom_build_variant_cfg_flag \
   --disable-mock \
-  --no-setup-mock \
-  --no-checkout-sources \
-  --no-clone-tools \
-  --no-clobber \
-  --no-update \
-  --no-upload-files \
-  --no-sendchange \
+  $steps \
   --log-level=debug \
+  --scm-level=$MOZ_SCM_LEVEL \
   --work-dir=$WORKSPACE/build \
-  --no-action=generate-build-stats \
   --branch=${MH_BRANCH} \
   --build-pool=${MH_BUILD_POOL}
--- a/testing/taskcluster/tasks/build.yml
+++ b/testing/taskcluster/tasks/build.yml
@@ -44,16 +44,17 @@ task:
       # Common environment variables for checking out gecko
       GECKO_BASE_REPOSITORY: '{{base_repository}}'
       GECKO_HEAD_REPOSITORY: '{{head_repository}}'
       GECKO_HEAD_REV: '{{head_rev}}'
       GECKO_HEAD_REF: '{{head_ref}}'
       TOOLTOOL_REPO: 'https://git.mozilla.org/build/tooltool.git'
       TOOLTOOL_REV: 'master'
       MOZ_BUILD_DATE: '{{pushdate}}'
+      MOZ_SCM_LEVEL: '{{level}}'
 
   extra:
     build_product: '{{build_product}}'
     build_name: '{{build_name}}'
     build_type: '{{build_type}}'
     index:
       rank: {{pushlog_id}}
     treeherder:
--- a/testing/taskcluster/tasks/builds/firefox_base.yml
+++ b/testing/taskcluster/tasks/builds/firefox_base.yml
@@ -1,14 +1,18 @@
 $inherits:
   from: 'tasks/build.yml'
   variables:
     build_product: 'firefox'
 task:
+  scopes:
+    - "secrets:get:project/releng/gecko/build/level-{{level}}/*"
   extra:
     locations:
       mozharness: 'public/build/mozharness.zip'
       test_packages: 'public/build/target.test_packages.json'
   payload:
     image:
       type: 'task-image'
       path: 'public/image.tar'
       taskId: '{{#task_id_for_image}}desktop-build{{/task_id_for_image}}'
+    features:
+      taskclusterProxy: true