bug 1259832 - add a post-build task to upload generated source files. r=dustin draft
authorTed Mielczarek <ted@mielczarek.org>
Wed, 09 Aug 2017 14:32:05 -0400
changeset 648517 c494bebe332e8c9fe1df7e25b6c23bbef20be289
parent 648516 6bc57d5e47a91265be853d8ba3dca1ecef2d065d
child 648518 768e25dea442661ec79727d45895bce9fed86a47
push id74775
push userbmo:ted@mielczarek.org
push dateThu, 17 Aug 2017 21:15:41 +0000
reviewersdustin
bugs1259832
milestone57.0a1
bug 1259832 - add a post-build task to upload generated source files. r=dustin This change adds an upload-generated-sources task kind that runs after nightly builds, fetches their `target.generated-files.tar.gz` artifact, and uploads all the contained files to an S3 bucket. For actual nightly and release builds on SCM level 3 trees, the S3 bucket is configured to be publicly accessible, so that tools like Socorro will be able to fetch generated source files that appear in crash reports, and debuggers will be able to fetch generated sources when they show up while debugging Nightly or Release builds. There are also level-2 and level-1 S3 buckets configured for builds happening on trees of other levels such as try. They are not configured as publicly accessible, but they exist so that these tasks can be tested in try. MozReview-Commit-ID: Js1HRftbtep
build/upload_generated_sources.py
python/mozbuild/mozbuild/generated_sources.py
taskcluster/ci/upload-generated-sources/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/upload_generated_sources.py
new file mode 100644
--- /dev/null
+++ b/build/upload_generated_sources.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+import gzip
+import io
+import logging
+from mozbuild.base import MozbuildObject
+from mozbuild.generated_sources import (
+    get_filename_with_digest,
+    get_s3_region_and_bucket,
+)
+import os
+from Queue import Queue
+import requests
+import sys
+import tarfile
+from threading import Event, Thread
+import time
+
+# Arbitrary, should probably measure this.
+NUM_WORKER_THREADS = 10
+log = logging.getLogger('upload-generated-sources')
+log.setLevel(logging.INFO)
+
+
+@contextmanager
+def timed():
+    '''
+    Yield a function that provides the elapsed time in seconds since this
+    function was called.
+    '''
+    start = time.time()
+    def elapsed():
+        return time.time() - start
+    yield elapsed
+
+
+def gzip_compress(data):
+    '''
+    Apply gzip compression to `data` and return the result as a `BytesIO`.
+    '''
+    b = io.BytesIO()
+    with gzip.GzipFile(fileobj=b, mode='w') as f:
+        f.write(data)
+    b.flush()
+    b.seek(0)
+    return b
+
+
+def upload_worker(queue, event, bucket, session_args):
+    '''
+    Get `(name, contents)` entries from `queue` and upload `contents`
+    to S3 with gzip compression using `name` as the key, prefixed with
+    the SHA-512 digest of `contents` as a hex string. If an exception occurs,
+    set `event`.
+    '''
+    try:
+        import boto3
+        session = boto3.session.Session(**session_args)
+        s3 = session.client('s3')
+        while True:
+            if event.is_set():
+                # Some other thread hit an exception.
+                return
+            (name, contents) = queue.get()
+            pathname = get_filename_with_digest(name, contents)
+            compressed = gzip_compress(contents)
+            extra_args = {
+                'ContentEncoding': 'gzip',
+                'ContentType': 'text/plain',
+            }
+            log.info('Uploading "{}" ({} bytes)'.format(pathname, len(compressed.getvalue())))
+            with timed() as elapsed:
+                s3.upload_fileobj(compressed, bucket, pathname, ExtraArgs=extra_args)
+                log.info('Finished uploading "{}" in {:0.3f}s'.format(pathname, elapsed()))
+            queue.task_done()
+    except Exception:
+        log.exception('Thread encountered exception:')
+        event.set()
+
+
+def do_work(artifact, region, bucket):
+    session_args = {'region_name': region}
+    session = requests.Session()
+    if 'TASK_ID' in os.environ:
+        level = os.environ.get('MOZ_SCM_LEVEL', '1')
+        secrets_url = 'http://taskcluster/secrets/v1/secret/project/releng/gecko/build/level-{}/gecko-generated-sources-upload'.format(level)
+        log.info('Using AWS credentials from the secrets service: "{}"'.format(secrets_url))
+        res = session.get(secrets_url)
+        res.raise_for_status()
+        secret = res.json()
+        session_args.update(
+            aws_access_key_id=secret['secret']['AWS_ACCESS_KEY_ID'],
+            aws_secret_access_key=secret['secret']['AWS_SECRET_ACCESS_KEY'],
+        )
+    else:
+        log.info('Trying to use your AWS credentials..')
+
+
+    # First, fetch the artifact containing the sources.
+    log.info('Fetching generated sources artifact: "{}"'.format(artifact))
+    with timed() as elapsed:
+        res = session.get(artifact)
+        log.info('Fetch HTTP status: {}, {} bytes downloaded in {:0.3f}s'.format(res.status_code, len(res.content), elapsed()))
+    res.raise_for_status()
+    # Create a queue and worker threads for uploading.
+    q = Queue()
+    event = Event()
+    log.info('Creating {} worker threads'.format(NUM_WORKER_THREADS))
+    for i in range(NUM_WORKER_THREADS):
+        t = Thread(target=upload_worker, args=(q, event, bucket, session_args))
+        t.daemon = True
+        t.start()
+    with tarfile.open(fileobj=io.BytesIO(res.content), mode='r|gz') as tar:
+        # Next, process each file.
+        for entry in tar:
+            if event.is_set():
+                break
+            log.info('Queueing "{}"'.format(entry.name))
+            q.put((entry.name, tar.extractfile(entry).read()))
+    # Wait until all uploads are finished.
+    # We don't use q.join() here because we want to also monitor event.
+    while q.unfinished_tasks:
+        if event.wait(0.1):
+            log.error('Worker thread encountered exception, exiting...')
+            break
+
+
+def main(argv):
+    logging.basicConfig(format='%(levelname)s - %(threadName)s - %(message)s')
+    parser = argparse.ArgumentParser(
+    description='Upload generated source files in ARTIFACT to BUCKET in S3.')
+    parser.add_argument('artifact',
+                        help='generated-sources artifact from build task')
+    args = parser.parse_args(argv)
+    region, bucket = get_s3_region_and_bucket()
+
+    config = MozbuildObject.from_environment()
+    config._activate_virtualenv()
+    config.virtualenv_manager.install_pip_package('boto3==1.4.4')
+
+    with timed() as elapsed:
+        do_work(region=region, bucket=bucket, artifact=args.artifact)
+        log.info('Finished in {:.03f}s'.format(elapsed()))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
--- a/python/mozbuild/mozbuild/generated_sources.py
+++ b/python/mozbuild/mozbuild/generated_sources.py
@@ -1,19 +1,37 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import hashlib
 import json
 import os
 
 from mozpack.files import FileFinder
 import mozpack.path as mozpath
 
 
+def sha512_digest(data):
+    '''
+    Generate the SHA-512 digest of `data` and return it as a hex string.
+    '''
+    return hashlib.sha512(data).hexdigest()
+
+
+def get_filename_with_digest(name, contents):
+    '''
+    Return the filename that will be used to store the generated file
+    in the S3 bucket, consisting of the SHA-512 digest of `contents`
+    joined with the relative path `name`.
+    '''
+    digest = sha512_digest(contents)
+    return mozpath.join(digest, name)
+
+
 def get_generated_sources():
     '''
     Yield tuples of `(objdir-rel-path, file)` for generated source files
     in this objdir, where `file` is either an absolute path to the file or
     a `mozpack.File` instance.
     '''
     import buildconfig
 
@@ -33,8 +51,23 @@ def get_generated_sources():
     rust_build_kind = 'debug' if buildconfig.substs.get('MOZ_DEBUG_RUST') else 'release'
     base = mozpath.join('toolkit/library',
                         buildconfig.substs['RUST_TARGET'],
                         rust_build_kind,
                         'build')
     finder = FileFinder(mozpath.join(buildconfig.topobjdir, base))
     for p, f in finder.find('**/*.rs'):
         yield mozpath.join(base, p), f
+
+
+def get_s3_region_and_bucket():
+    '''
+    Return a tuple of (region, bucket) giving the AWS region and S3
+    bucket to which generated sources should be uploaded.
+    '''
+    region = 'us-west-2'
+    level = os.environ.get('MOZ_SCM_LEVEL', '1')
+    bucket = {
+        '1': 'gecko-generated-sources-l1',
+        '2': 'gecko-generated-sources-l2',
+        '3': 'gecko-generated-sources',
+    }[level]
+    return (region, bucket)
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/upload-generated-sources/kind.yml
@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.upload_generated_sources:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build
+
+only-for-attributes:
+  - nightly
+
+job-template:
+  description: Upload generated source files from build
+  attributes:
+    nightly: true
+  worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+  treeherder:
+    symbol: Ugs
+    kind: build
+  worker:
+     docker-image: {in-tree: "lint"}
+     max-run-time: 600
+  run:
+    using: run-task
+    command: >
+            cd /home/worker/checkouts/gecko &&
+            ./mach python build/upload_generated_sources.py ${ARTIFACT_URL}
+  scopes:
+      - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -62,16 +62,21 @@ a source checkout, it is still possible 
 often they do not.
 
 upload-symbols
 --------------
 
 Upload-symbols tasks run after builds and upload the symbols files generated by
 build tasks to Socorro for later use in crash analysis.
 
+upload-generated-sources
+--------------
+
+Upload-generated-sources tasks run after builds and upload source files that were generated as part of the build process to an s3 bucket for later use in links from crash reports or when debugging shipped builds.
+
 valgrind
 --------
 
 Valgrind tasks produce builds instrumented by valgrind.
 
 static-analysis
 ---------------
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/upload_generated_sources.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the upload-generated-files task description template,
+  taskcluster/ci/upload-generated-sources/kind.yml
+into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.taskcluster import get_artifact_url
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_task_info(config, jobs):
+    for job in jobs:
+        dep_task = job['dependent-task']
+        del job['dependent-task']
+
+        # Add a dependency on the build task.
+        job['dependencies'] = {'build': dep_task.label}
+        # Label the job to match the build task it's uploading from.
+        job['label'] = dep_task.label.replace("build-", "upload-generated-sources-")
+        # Copy over some bits of metdata from the build task.
+        dep_th = dep_task.task['extra']['treeherder']
+        job.setdefault('attributes', {})
+        job['attributes']['build_platform'] = dep_task.attributes.get('build_platform')
+        plat = '{}/{}'.format(dep_th['machine']['platform'], dep_task.attributes.get('build_type'))
+        job['treeherder']['platform'] = plat
+        job['treeherder']['tier'] = dep_th['tier']
+        # Add an environment variable pointing at the artifact from the build.
+        artifact_url = get_artifact_url('<build>',
+                                        'public/build/target.generated-files.tar.gz')
+        job['worker'].setdefault('env', {})['ARTIFACT_URL'] = {
+            'task-reference': artifact_url
+        }
+
+        yield job