Bug 1286075: add upload-symbols kind; r=ted.mielczarek draft
authorDustin J. Mitchell <dustin@mozilla.com>
Wed, 07 Sep 2016 15:08:28 +0000
changeset 412749 ba9a1f6d653328a76a45bf14825848ee4d64bb50
parent 412748 03cd53775b2aec6385681a5a4f6165482e56873b
child 412750 8ad5e1a0866f64755e985d5ec61b379efa54b54d
push id29252
push userdmitchell@mozilla.com
push dateMon, 12 Sep 2016 19:16:39 +0000
reviewersted.mielczarek
bugs1286075
milestone51.0a1
Bug 1286075: add upload-symbols kind; r=ted.mielczarek MozReview-Commit-ID: C3Se6gdSPri
taskcluster/ci/upload-symbols/job-template.yml
taskcluster/ci/upload-symbols/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/task/post_build.py
taskcluster/taskgraph/transforms/upload_symbols.py
taskcluster/taskgraph/try_option_syntax.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/upload-symbols/job-template.yml
@@ -0,0 +1,19 @@
+label: # see transforms
+description: Upload Symbols
+dependencies: # see transforms
+expires-after: 7 days
+deadline-after: 24 hours
+run-on-projects:
+    - try
+worker-type: aws-provisioner-v1/symbol-upload
+worker:
+    implementation: docker-worker
+    max-run-time: 600
+    command: ["/bin/bash", "bin/upload.sh"]
+    docker-image: taskclusterprivate/upload_symbols:0.0.3
+    env:
+        GECKO_HEAD_REPOSITORY: # see transforms
+        GECKO_HEAD_REV: # see transforms
+        ARTIFACT_TASKID: {"task-reference": "<build>"}
+scopes:
+    - docker-worker:image:taskclusterprivate/upload_symbols:0.0.3
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/upload-symbols/kind.yml
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.post_build:PostBuildTask
+
+transforms:
+   - taskgraph.transforms.upload_symbols:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+    - build
+
+job-template: job-template.yml
+
+only-for-build-platforms:
+    - linux64/opt
+    - linux64/debug
+    - android-api-15/opt
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -33,16 +33,22 @@ TBD (Callek)
 source-check
 ------------
 
 Source-checks are tasks that look at the Gecko source directly to check
 correctness.  This can include linting, Python unit tests, source-code
 analysis, or measurement work -- basically anything that does not require a
 build.
 
+upload-symbols
+--------------
+
+Upload-symbols tasks run after builds and upload the symbols files generated by
+build tasks to Socorro for later use in crash analysis.
+
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/post_build.py
@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import logging
+
+from . import transform
+from ..util.yaml import load_yaml
+
+logger = logging.getLogger(__name__)
+
+
+class PostBuildTask(transform.TransformTask):
+    """
+    A task implementing a post-build job.  These depend on jobs and perform
+    various followup tasks after a build has completed.
+
+    The `only-for-build-platforms` kind configuration, if specified, will limit
+    the build platforms for which a post-build task will be created.
+
+    The `job-template' kind configuration points to a yaml file which will
+    be used to create the input to the transforms.  It will have added to it
+    keys `build-label`, the label for the build task, and `build-platform`, its
+    platform.
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        if config.get('kind-dependencies', []) != ["build"]:
+            raise Exception("PostBuildTask kinds must depend on builds")
+
+        only_platforms = config.get('only-for-build-platforms')
+        prototype = load_yaml(path, config.get('job-template'))
+
+        for task in loaded_tasks:
+            if task.kind != 'build':
+                continue
+
+            build_platform = task.attributes.get('build_platform')
+            build_type = task.attributes.get('build_type')
+            if not build_platform or not build_type:
+                continue
+            platform = "{}/{}".format(build_platform, build_type)
+            if only_platforms and platform not in only_platforms:
+                continue
+
+            post_task = copy.deepcopy(prototype)
+            post_task['build-label'] = task.label
+            post_task['build-platform'] = platform
+            yield post_task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/upload_symbols.py
@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the upload-symbols task description template,
+  taskcluster/ci/upload-symbols/job-template.yml
+into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def fill_template(config, tasks):
+    for task in tasks:
+        # Fill out the dynamic fields in the task description
+        task['label'] = task['build-label'] + '-upload-symbols'
+        task['dependencies'] = {'build': task['build-label']}
+        task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
+        task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev']
+
+        build_platform, build_type = task['build-platform'].split('/')
+        attributes = task.setdefault('attributes', {})
+        attributes['build_platform'] = build_platform
+        attributes['build_type'] = build_type
+
+        # clear out the stuff that's not part of a task description
+        del task['build-label']
+        del task['build-platform']
+
+        yield task
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -22,16 +22,17 @@ BUILD_TYPE_ALIASES = {
 }
 
 # consider anything in this whitelist of kinds to be governed by -b/-p
 BUILD_KINDS = set([
     'build',
     'artifact-build',
     'hazard',
     'l10n',
+    'upload-symbols',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
     'source-check',
 ])