Bug 1286075: add hazard kind; r=sfink draft
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:04:09 +0000
changeset 412746 7eeadb6ae5481164fe2c06034b312c18cfe1d7d7
parent 412745 8217167bebae2100b1b961abefe2645322efc7b0
child 412747 35b5ce42081d8d353323cdf8b0eb6e199a687e2c
push id29252
push userdmitchell@mozilla.com
push dateMon, 12 Sep 2016 19:16:39 +0000
reviewerssfink
bugs1286075
milestone51.0a1
Bug 1286075: add hazard kind; r=sfink MozReview-Commit-ID: qvFg2O6OwB
taskcluster/ci/hazard/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/job/hazard.py
taskcluster/taskgraph/try_option_syntax.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/hazard/kind.yml
@@ -0,0 +1,93 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+    treeherder:
+        kind: build
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+        docker-image: {in-tree: desktop-build}
+
+jobs:
+    linux64-shell-haz/debug:
+        description: "JS Shell Hazard Analysis Linux"
+        index:
+            product: firefox
+            job-name:
+                gecko-v1: shell-haz.debug
+                gecko-v2: shell-haz-debug
+        treeherder:
+            platform: linux64/debug
+            symbol: SM-tc(H)
+        run:
+            using: hazard
+            tooltool-manifest: "browser/config/tooltool-manifests/linux64/hazard.manifest"
+            secrets: true  # TODO(taskdiff): don't use secrets for these jobs
+            command: >
+                tc-vcs checkout workspace/gecko "$GECKO_BASE_REPOSITORY" "$GECKO_HEAD_REPOSITORY" "$GECKO_HEAD_REV" "$GECKO_HEAD_REF"
+                && cd ./workspace/gecko/taskcluster/scripts/builder
+                && ./build-haz-linux.sh --project shell $HOME/workspace
+        run-on-projects:
+            - integration
+            - release
+        when:
+            files-changed:
+                - js/public/**
+                - js/src/**
+
+    linux64-haz/debug:
+        description: "Browser Hazard Analysis Linux"
+        index:
+            product: firefox
+            job-name:
+                gecko-v1: browser-haz.debug
+                gecko-v2: browser-haz-debug
+        treeherder:
+            platform: linux64/debug
+            symbol: tc(H)
+        run:
+            using: hazard
+            tooltool-manifest: "browser/config/tooltool-manifests/linux64/hazard.manifest"
+            mozconfig: "browser/config/mozconfigs/linux64/hazards"
+            secrets: true  # TODO(taskdiff): don't use secrets for these jobs
+            command: >
+                tc-vcs checkout workspace/gecko "$GECKO_BASE_REPOSITORY" "$GECKO_HEAD_REPOSITORY" "$GECKO_HEAD_REV" "$GECKO_HEAD_REF"
+                && cd ./workspace/gecko/taskcluster/scripts/builder
+                && ./build-haz-linux.sh --project browser $HOME/workspace
+
+    linux64-mulet-haz/debug:
+        description: "Mulet Hazard Analysis Linux"
+        index:
+            product: b2g
+            job-name:
+                buildbot: linux64-haz-mulet
+                gecko-v1: mulet-haz.debug
+                gecko-v2: mulet-haz-debug
+        treeherder:
+            platform: mulet-linux64/opt
+            symbol: tc(H)
+            tier: 3
+        run-on-projects:
+            - try
+        worker-type: aws-provisioner-v1/mulet-debug  # TODO (taskdiff): change to default
+        worker:
+            docker-image: {in-tree: builder}
+        run:
+            using: hazard
+            tooltool-manifest: "gecko/b2g/dev/config/tooltool-manifests/linux64/hazard.manifest"
+            mozconfig: "b2g/dev/config/mozconfigs/linux64/mulet-hazards"
+            secrets: false  # TODO(taskdiff): don't use secrets for these jobs
+            command: >
+                checkout-gecko workspace && cd ./workspace/gecko/taskcluster/scripts/builder
+                && buildbot_step 'Build' ./build-mulet-haz-linux.sh $HOME/workspace
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -13,16 +13,23 @@ actions for static analysis or to produc
 
 artifact-build
 --------------
 
 This kind performs an artifact build: one based on precompiled binaries
 discovered via the TaskCluster index.  This task verifies that such builds
 continue to work correctly.
 
+hazard
+------
+
+Hazard builds are similar to "regular' builds, but use a compiler extension to
+extract a bunch of data from the build and then analyze that data looking for
+hazardous behaviors.
+
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/hazard.py
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running hazard jobs via dedicated scripts
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required, Optional, Any
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_workspace_cache,
+    docker_worker_setup_secrets,
+    docker_worker_add_tc_vcs_cache,
+    docker_worker_add_gecko_vcs_env_vars,
+    docker_worker_add_public_artifacts
+)
+
+haz_run_schema = Schema({
+    Required('using'): 'hazard',
+
+    # The command to run within the task image (passed through to the worker)
+    Required('command'): basestring,
+
+    # The tooltool manifest to use; default in the script is used if omitted
+    Optional('tooltool-manifest'): basestring,
+
+    # The mozconfig to use; default in the script is used if omitted
+    Optional('mozconfig'): basestring,
+
+    # The set of secret names to which the task has access; these are prefixed
+    # with `project/releng/gecko/{treeherder.kind}/level-{level}/`.   Setting
+    # this will enable any worker features required and set the task's scopes
+    # appropriately.  `true` here means ['*'], all secrets.  Not supported on
+    # Windows
+    Required('secrets', default=False): Any(bool, [basestring]),
+})
+
+
+@run_job_using("docker-worker", "hazard", schema=haz_run_schema)
+def docker_worker_hazard(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['artifacts'] = []
+    worker['caches'] = []
+
+    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_workspace_cache(config, job, taskdesc)
+    docker_worker_setup_secrets(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+        'MOZ_SCM_LEVEL': config.params['level'],
+    })
+
+    # script parameters
+    if run.get('tooltool-manifest'):
+        env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
+    if run.get('mozconfig'):
+        env['MOZCONFIG'] = run['mozconfig']
+
+    # tooltool downloads
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        'mount-point': '/home/worker/tooltool-cache',
+    })
+    worker['relengapi-proxy'] = True
+    taskdesc['scopes'].extend([
+        'docker-worker:relengapi-proxy:tooltool.download.public',
+    ])
+    env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+    env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+    env['TOOLTOOL_REV'] = 'master'
+
+    worker['command'] = ["/bin/bash", "-c", run['command']]
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -20,16 +20,17 @@ BUILD_TYPE_ALIASES = {
     'o': 'opt',
     'd': 'debug'
 }
 
 # consider anything in this whitelist of kinds to be governed by -b/-p
 BUILD_KINDS = set([
     'build',
     'artifact-build',
+    'hazard',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
 ])
 
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying