Bug 1286075: add a spidermonkey kind; r=sfink draft
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:04:24 +0000
changeset 412753 ec4070eab92f63694fadf070856aafdc5689efb5
parent 412752 caaf989297c35888ed3cf8f1156b53649c417d02
child 412754 81f48fbd380d79773319781c5db84a0056c9341f
push id29252
push userdmitchell@mozilla.com
push dateMon, 12 Sep 2016 19:16:39 +0000
reviewerssfink
bugs1286075
milestone51.0a1
Bug 1286075: add a spidermonkey kind; r=sfink MozReview-Commit-ID: 4uI8LxHrSOS
taskcluster/ci/spidermonkey/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/job/spidermonkey.py
taskcluster/taskgraph/try_option_syntax.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -0,0 +1,221 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+    treeherder:
+        platform: linux64/opt
+        kind: build
+        tier: 1
+    index:
+        product: firefox
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+        docker-image: {in-tree: desktop-build}
+    run:
+        using: spidermonkey
+    when:
+        files-changed:
+            # any when.files-changed specified below in a job will be
+            # appended to this list
+            - js/public/**
+            - js/src/**
+
+jobs:
+    sm-package/opt:
+        description: "Spidermonkey source package and test"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-package.opt
+                gecko-v2: sm-package-opt
+        treeherder:
+            symbol: SM-tc(pkg)
+        run:
+            using: spidermonkey-package
+            spidermonkey-variant: plain
+        run-on-projects:
+            - integration
+            - release
+        when:
+            files-changed:
+                - build/**
+                - configure.py
+                - dom/bindings/**
+                - intl/icu/**
+                - js/moz.configure
+                - layout/tools/reftest/reftest/**
+                - media/webrtc/trunk/tools/gyp/**
+                - memory/**
+                - mfbt/**
+                - modules/fdlibm/**
+                - modules/zlib/src/**
+                - mozglue/**
+                - moz.configure
+                - nsprpub/**
+                - python/**
+                - taskcluster/moz.build
+                - testing/mozbase/**
+                - test.mozbuild
+                - toolkit/mozapps/installer/package-name.mk
+                - toolkit/mozapps/installer/upload-files.mk
+
+    sm-plain/debug:
+        description: "Spidermonkey Plain"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-plaindebug.debug
+                gecko-v2: sm-plaindebug-debug
+        treeherder:
+            platform: linux64/debug
+            symbol: SM-tc(p)
+        run:
+            spidermonkey-variant: plaindebug
+        run-on-projects:
+            - integration
+            - release
+
+    sm-plain/opt:
+        description: "Spidermonkey Plain"
+        index:
+            job-name: sm-plain-opt
+        treeherder:
+            symbol: SM-tc(p)
+        run:
+            spidermonkey-variant: plain
+        run-on-projects:
+            - integration
+            - release
+
+    sm-arm-sim/debug:
+        description: "Spidermonkey ARM sim"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-arm-sim.debug
+                gecko-v2: sm-arm-sim-debug
+        treeherder:
+            symbol: SM-tc(arm)
+        run:
+            spidermonkey-variant: arm-sim
+        run-on-projects:
+            - integration
+            - release
+
+    sm-arm64-sim/debug:
+        description: "Spidermonkey ARM64 sim"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-arm64-sim.debug
+                gecko-v2: sm-arm64-sim-debug
+        treeherder:
+            symbol: SM-tc(arm64)
+        run:
+            spidermonkey-variant: arm64-sim
+        run-on-projects:
+            - integration
+            - release
+
+    sm-asan/opt:
+        description: "Spidermonkey Address Sanitizer"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-asan.opt
+                gecko-v2: sm-asan-opt
+        treeherder:
+            symbol: SM-tc(asan)
+        run:
+            spidermonkey-variant: asan
+            tooltool-manifest: browser/config/tooltool-manifests/linux64/asan.manifest
+        run-on-projects:
+            - integration
+            - release
+
+    sm-compacting/debug:
+        description: "Spidermonkey Compacting"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-compacting.debug
+                gecko-v2: sm-compacting-debug
+        treeherder:
+            symbol: SM-tc(cgc)
+        run:
+            spidermonkey-variant: compacting
+        run-on-projects:
+            - integration
+            - release
+
+    sm-msan/opt:
+        description: "Spidermonkey Memory Sanitizer"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-msan.opt
+                gecko-v2: sm-msan-opt
+        treeherder:
+            symbol: SM-tc(msan)
+        run:
+            spidermonkey-variant: msan
+            tooltool-manifest: browser/config/tooltool-manifests/linux64/msan.manifest
+        run-on-projects:
+            - integration
+            - release
+
+    sm-tsan/opt:
+        description: "Spidermonkey Thread Sanitizer"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-tsan.opt
+                gecko-v2: sm-tsan-opt
+        treeherder:
+            symbol: SM-tc(tsan)
+            tier: 3
+        run-on-projects: []
+        run:
+            spidermonkey-variant: tsan
+            tooltool-manifest: browser/config/tooltool-manifests/linux64/tsan.manifest
+
+    sm-rootanalysis/debug:
+        description: "Spidermonkey Root Analysis"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-rootanalysis.debug
+                gecko-v2: sm-rootanalysis-debug
+        treeherder:
+            symbol: SM-tc(r)
+        run:
+            spidermonkey-variant: rootanalysis
+        run-on-projects:
+            - integration
+            - release
+
+    sm-nonunified/debug:
+        description: "Spidermonkey Non-Unified Debug"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-nonunified.debug
+                gecko-v2: sm-nonunified-debug
+        treeherder:
+            platform: linux64/debug
+            symbol: SM-tc(nu)
+        run:
+            spidermonkey-variant: nonunified
+        run-on-projects:
+            - integration
+            - release
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -59,16 +59,22 @@ artifact.
 
 toolchain
 ---------
 
 Toolchain builds create the compiler toolchains used to build Firefox.  These
 will eventually be dependencies of the builds themselves, but for the moment
 are run manually via try pushes and the results uploaded to tooltool.
 
+spidermonkey
+------------
+
+Spidermonkey tasks check out the full gecko source tree, then compile only the
+spidermonkey portion.  Each task runs specific tests after the build.
+
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running spidermonkey jobs via dedicated scripts
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required, Optional, Any
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_gecko_vcs_env_vars,
+    docker_worker_add_tc_vcs_cache,
+    docker_worker_add_public_artifacts
+)
+
+sm_run_schema = Schema({
+    Required('using'): Any('spidermonkey', 'spidermonkey-package'),
+
+    # The SPIDERMONKEY_VARIANT
+    Required('spidermonkey-variant'): basestring,
+
+    # The tooltool manifest to use; default from sm-tooltool-config.sh  is used
+    # if omitted
+    Optional('tooltool-manifest'): basestring,
+})
+
+
+@run_job_using("docker-worker", "spidermonkey")
+@run_job_using("docker-worker", "spidermonkey-package")
+def docker_worker_spidermonkey(config, job, taskdesc, schema=sm_run_schema):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['artifacts'] = []
+    worker['caches'] = []
+
+    if int(config.params['level']) > 1:
+        worker['caches'].append({
+            'type': 'persistent',
+            'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
+                config.params['level'], config.params['project']),
+            'mount-point': "/home/worker/workspace",
+        })
+
+    # include secrets access
+    # TODO (taskdiff): this is unused? remove it
+    worker['taskcluster-proxy'] = True
+    taskdesc['scopes'].append(
+        'secrets:get:project/releng/gecko/build/level-{}/*'.format(config.params['level']))
+
+    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZHARNESS_DISABLE': 'true',
+        'TOOLS_DISABLE': 'true',
+        'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
+        'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+        'MOZ_SCM_LEVEL': config.params['level'],
+    })
+
+    # tooltool downloads; note that this script downloads using the API
+    # endpoiint directly, rather than via relengapi-proxy
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        'mount-point': '/home/worker/tooltool-cache',
+    })
+    env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+    env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+    env['TOOLTOOL_REV'] = 'master'
+    if run.get('tooltool-manifest'):
+        env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
+
+    script = "build-sm.sh"
+    if run['using'] == 'spidermonkey-package':
+        script = "build-sm-package.sh"
+
+    worker['command'] = [
+        "/bin/bash",
+        "-c",
+        "cd /home/worker/ "
+        "&& ./bin/checkout-sources.sh "
+        "&& ./workspace/build/src/taskcluster/scripts/builder/" + script
+    ]
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -25,16 +25,17 @@ BUILD_TYPE_ALIASES = {
 BUILD_KINDS = set([
     'build',
     'artifact-build',
     'hazard',
     'l10n',
     'upload-symbols',
     'valgrind',
     'static-analysis',
+    'spidermonkey',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
     'source-check',
     'toolchain',
 ])