Bug 1286075: add source-check kind; r?ahal draft
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:39:05 +0000
changeset 412748 03cd53775b2aec6385681a5a4f6165482e56873b
parent 412747 35b5ce42081d8d353323cdf8b0eb6e199a687e2c
child 412749 ba9a1f6d653328a76a45bf14825848ee4d64bb50
push id29252
push userdmitchell@mozilla.com
push dateMon, 12 Sep 2016 19:16:39 +0000
reviewersahal
bugs1286075
milestone51.0a1
Bug 1286075: add source-check kind; r?ahal MozReview-Commit-ID: EDdPraYxNlH
taskcluster/ci/source-check/doc.yml
taskcluster/ci/source-check/kind.yml
taskcluster/ci/source-check/mozlint.yml
taskcluster/ci/source-check/python-tests.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/job/mach.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/try_option_syntax.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/source-check/doc.yml
@@ -0,0 +1,36 @@
+sphinx/opt:
+    description: Generate the Sphinx documentation
+    index:
+        product: lint
+        job-name:
+            gecko-v2: sphinx-opt
+    treeherder:
+        symbol: tc(Doc)
+        kind: test
+        tier: 1
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+        artifacts:
+            - type: file
+              name: public/docs.tar.gz
+              path: /home/worker/checkouts/gecko/docs.tar.gz
+    run:
+        using: run-task
+        command: >
+            cd /home/worker/checkouts/gecko &&
+            ./mach doc --outdir docs-out --no-open &&
+            rm -rf docs-out/html/Mozilla_Source_Tree_Docs/_venv &&
+            mv docs-out/html/Mozilla_Source_Tree_Docs docs &&
+            tar -czf docs.tar.gz docs
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - '**/*.py'
+            - '**/*.rst'
+            - 'tools/docs/**'
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/source-check/kind.yml
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+jobs-from:
+    - python-tests.yml
+    - mozlint.yml
+    - doc.yml
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/source-check/mozlint.yml
@@ -0,0 +1,107 @@
+mozlint-eslint/opt:
+    description: JS lint check
+    index:
+        product: lint
+        job-name:
+            gecko-v2: eslint-gecko-opt
+    treeherder:
+        symbol: ES
+        kind: test
+        tier: 1
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: run-task
+        command: >
+            cd /home/worker/checkouts/gecko/tools/lint/eslint &&
+            /build/tooltool.py fetch -m manifest.tt &&
+            tar xvfz eslint.tar.gz &&
+            rm eslint.tar.gz &&
+            ln -s ../eslint-plugin-mozilla node_modules &&
+            cd ../../.. &&
+            ./mach lint -l eslint -f treeherder --quiet
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            # Files that are likely audited.
+            - '**/*.js'
+            - '**/*.jsm'
+            - '**/*.jsx'
+            - '**/*.html'
+            - '**/*.xhtml'
+            - '**/*.xml'
+            # Run when eslint policies change.
+            - '**/.eslintignore'
+            - '**/*eslintrc*'
+            # The plugin implementing custom checks.
+            - 'tools/lint/eslint/eslint-plugin-mozilla/**'
+            # Other misc lint related files.
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
+            - 'testing/docker/lint/**'
+
+mozlint-flake8/opt:
+    description: flake8 run over the gecko codebase
+    index:
+        product: lint
+        job-name:
+            gecko-v2: mozlint-flake8-opt
+    treeherder:
+        symbol: f8
+        kind: test
+        tier: 2
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: lint -l flake8 -f treeherder
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - '**/*.py'
+            - '**/.flake8'
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
+            - 'testing/docker/lint/**'
+
+wptlint-gecko/opt:
+    description: web-platform-tests linter
+    index:
+        product: lint
+        job-name:
+            gecko-v2: wpt-lint-gecko-opt
+    treeherder:
+        symbol: W
+        kind: test
+        tier: 2
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: lint -l wpt -f treeherder
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'testing/web-platform/tests/**'
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
+            - 'testing/docker/lint/**'
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/source-check/python-tests.yml
@@ -0,0 +1,57 @@
+taskgraph-tests/opt:
+    description: taskcluster/taskgraph unit tests
+    index:
+        product: lint
+        job-name:
+            gecko-v2: taskgraph-tests-opt
+    treeherder:
+        symbol: tg
+        kind: test
+        tier: 2
+        platform: linux64/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: taskgraph python-tests
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'taskcluster/**/*.py'
+            - 'config/mozunit.py'
+            - 'python/mach/**/*.py'
+
+mozharness/opt:
+    description: mozharness integration tests
+    index:
+        product: lint
+        job-name:
+            gecko-v2: mozharness-tox-opt
+    treeherder:
+        symbol: MH
+        kind: test
+        tier: 2
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: run-task
+        cache-dotcache: true
+        command: >
+            cd /home/worker/checkouts/gecko/testing/mozharness &&
+            /usr/bin/pip2 install tox &&
+            /home/worker/.local/bin/tox -e py27-hg3.7
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'testing/mozharness/**'
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -25,16 +25,24 @@ Hazard builds are similar to "regular' b
 extract a bunch of data from the build and then analyze that data looking for
 hazardous behaviors.
 
 l10n
 ----
 
 TBD (Callek)
 
+source-check
+------------
+
+Source-checks are tasks that look at the Gecko source directly to check
+correctness.  This can include linting, Python unit tests, source-code
+analysis, or measurement work -- basically anything that does not require a
+build.
+
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running mach tasks (via run-task)
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.run_task import docker_worker_run_task
+from voluptuous import Schema, Required
+
+mach_schema = Schema({
+    Required('using'): 'mach',
+
+    # The mach command (omitting `./mach`) to run
+    Required('mach'): basestring,
+})
+
+
+@run_job_using("docker-worker", "mach", schema=mach_schema)
+def docker_worker_mach(config, job, taskdesc):
+    run = job['run']
+
+    # defer to the run_task implementation
+    run['command'] = 'cd /home/worker/checkouts/gecko && ./mach ' + run['mach']
+    run['checkout'] = True
+    del run['mach']
+    docker_worker_run_task(config, job, taskdesc)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -78,17 +78,18 @@ task_description_schema = Schema({
         # treeherder environments (defaults to both staging and production)
         Required('environments', default=['production', 'staging']): ['production', 'staging'],
     },
 
     # information for indexing this build so its artifacts can be discovered;
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
-        'product': Any('firefox', 'mobile', 'b2g'),
+        # TODO: remove lint when not used
+        'product': Any('firefox', 'mobile', 'b2g', 'lint'),
 
         # the names to use for this job in the TaskCluster index
         'job-name': Any(
             # Assuming the job is named "normally", this is the v2 job name,
             # and the v1 and buildbot routes will be determined appropriately.
             basestring,
 
             # otherwise, give separate names for each of the legacy index
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -26,16 +26,17 @@ BUILD_KINDS = set([
     'build',
     'artifact-build',
     'hazard',
     'l10n',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
+    'source-check',
 ])
 
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying
 # matching test names
 def alias_prefix(prefix):
     return lambda name: name.startswith(prefix)