Bug 1286075: add a toolchain kind; r?gps draft
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:04:31 +0000
changeset 412752 caaf989297c35888ed3cf8f1156b53649c417d02
parent 412751 d1e1086e24f8e772689089626a3bb47e04164fe4
child 412753 ec4070eab92f63694fadf070856aafdc5689efb5
push id29252
push userdmitchell@mozilla.com
push dateMon, 12 Sep 2016 19:16:39 +0000
reviewersgps
bugs1286075
milestone51.0a1
Bug 1286075: add a toolchain kind; r?gps MozReview-Commit-ID: EEeQdA9aPPq
taskcluster/ci/toolchain/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/job/toolchain.py
taskcluster/taskgraph/try_option_syntax.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+    description: toolchain build
+    treeherder:
+        kind: build
+        tier: 1
+        platform: linux64/opt
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 36000
+
+jobs:
+    linux64-clang/opt:
+        index:
+            product: firefox
+            job-name:
+                gecko-v2: linux64-clang-opt
+        treeherder:
+            symbol: Cc(Clang)
+        run:
+            using: toolchain-script
+            script: build-clang-linux.sh
+        when:
+            files-changed:
+                - 'build/build-clang/**'
+                - 'taskcluster/scripts/misc/build-clang-linux.sh'
+
+    linux64-gcc/opt:
+        index:
+            product: firefox
+            job-name:
+                gecko-v2: linux64-gcc-opt
+        treeherder:
+            symbol: Cc(GCC)
+        run:
+            using: toolchain-script
+            script: build-gcc-linux.sh
+        when:
+            files-changed:
+                - 'build/unix/build-gcc/**'
+                - 'taskcluster/scripts/misc/build-gcc-linux.sh'
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -52,16 +52,23 @@ Valgrind tasks produce builds instrument
 static-analysis
 ---------------
 
 Static analysis builds use the compiler to perform some detailed analysis of
 the source code while building.  The useful output from these tasks are their
 build logs, and while they produce a binary, they do not upload it as an
 artifact.
 
+toolchain
+---------
+
+Toolchain builds create the compiler toolchains used to build Firefox.  These
+will eventually be dependencies of the builds themselves, but for the moment
+are run manually via try pushes and the results uploaded to tooltool.
+
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running toolchain-building jobs via dedicated scripts
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_tc_vcs_cache,
+    docker_worker_add_gecko_vcs_env_vars
+)
+
+toolchain_run_schema = Schema({
+    Required('using'): 'toolchain-script',
+
+    # the script (in taskcluster/scripts/misc) to run
+    Required('script'): basestring,
+})
+
+
+@run_job_using("docker-worker", "toolchain-script", schema=toolchain_run_schema)
+def docker_worker_toolchain(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['artifacts'] = []
+    worker['caches'] = []
+
+    worker['artifacts'].append({
+        'name': 'public',
+        'path': '/home/worker/workspace/artifacts/',
+        'type': 'directory',
+    })
+
+    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+        'MOZ_SCM_LEVEL': config.params['level'],
+        'TOOLS_DISABLE': 'true',
+    })
+
+    # tooltool downloads; note that this downloads using the API endpoint directly,
+    # rather than via relengapi-proxy
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        'mount-point': '/home/worker/tooltool-cache',
+    })
+    env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+    env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+    env['TOOLTOOL_REV'] = 'master'
+
+    command = ' && '.join([
+        "cd /home/worker/",
+        "./bin/checkout-sources.sh",
+        "./workspace/build/src/taskcluster/scripts/misc/" + run['script'],
+    ])
+    worker['command'] = ["/bin/bash", "-c", command]
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -30,16 +30,17 @@ BUILD_KINDS = set([
     'upload-symbols',
     'valgrind',
     'static-analysis',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
     'source-check',
+    'toolchain',
 ])
 
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying
 # matching test names
 def alias_prefix(prefix):
     return lambda name: name.startswith(prefix)