Bug 1388016 - Add support for specifying the Python version to use in TaskCluster; r?ahal draft
authorDave Hunt <dhunt@mozilla.com>
Thu, 21 Jun 2018 06:07:01 +0100
changeset 810734 2723c85f9318a0f83c5d82c14a894f77eef10ccb
parent 810712 ca98b6f47b4e33673291a4dc2a4ebca28ce6db00
child 810735 a11876c9501065ed9a788fdc45a538c02e80b1b4
push id114078
push userbmo:dave.hunt@gmail.com
push dateTue, 26 Jun 2018 10:56:45 +0000
reviewersahal
bugs1388016
milestone63.0a1
Bug 1388016 - Add support for specifying the Python version to use in TaskCluster; r?ahal MozReview-Commit-ID: Mo6KWrzPU3
taskcluster/ci/config.yml
taskcluster/ci/source-test/python.yml
taskcluster/docs/transforms.rst
taskcluster/taskgraph/transforms/job/python_test.py
taskcluster/taskgraph/transforms/source_test.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -1,16 +1,17 @@
 trust-domain: gecko
 project-repo-param-prefix: ''
 treeherder:
     group-names:
         'cram': 'Cram tests'
         'js-bench': 'JavaScript shell benchmarks'
         'mocha': 'Mocha unit tests'
-        'py': 'Python unit tests'
+        'py2': 'Python 2 unit tests'
+        'py3': 'Python 3 unit tests'
         'A': 'Android Gradle tests'
         'Fxfn-l': 'Firefox functional tests (local)'
         'Fxfn-l-e10s': 'Firefox functional tests (local) with e10s'
         'Fxfn-r': 'Firefox functional tests (remote)'
         'Fxfn-r-e10s': 'Firefox functional tests (remote) with e10s'
         'M': 'Mochitests'
         'M-e10s': 'Mochitests with e10s'
         'M-V': 'Mochitests on Valgrind'
--- a/taskcluster/ci/source-test/python.yml
+++ b/taskcluster/ci/source-test/python.yml
@@ -16,48 +16,52 @@ job-defaults:
         using: mach
     when:
         files-changed:
             - 'config/mozunit/**'
             - 'python/mach_commands.py'
 
 taskgraph-tests:
     description: taskcluster/taskgraph unit tests
+    python-version: [2]
     treeherder:
-        symbol: py(tg)
+        symbol: tg
     run:
-        mach: python-test --subsuite taskgraph
+        using: python-test
+        subsuite: taskgraph
     when:
         files-changed:
             - 'taskcluster/**/*.py'
             - 'python/mach/**/*.py'
 
 marionette-harness:
     description: testing/marionette/harness unit tests
+    python-version: [2]
     treeherder:
-        symbol: py(mnh)
+        symbol: mnh
     run:
-        mach: python-test --subsuite marionette-harness
+        using: python-test
+        subsuite: marionette-harness
     when:
         files-changed:
             - 'testing/marionette/harness/**'
             - 'testing/mozbase/mozlog/mozlog/**'
             - 'testing/mozbase/mozlog/setup.py'
             - 'testing/mozbase/packages.txt'
 
 mochitest-harness:
     description: testing/mochitest unittests
     platform:
         - linux64/opt
         - linux64/debug
         - linux64-asan/opt
     require-build: true
     always-target: false
     treeherder:
-        symbol: py(mch)
+        symbol: py2(mch)
     worker:
         by-platform:
             linux64.*:
                 docker-image: {in-tree: "desktop1604-test"}
                 max-run-time: 3600
     run:
         using: run-task
         use-artifacts:
@@ -70,101 +74,111 @@ mochitest-harness:
             start_xvfb '1600x1200x24' 0 &&
             cd $USE_ARTIFACT_PATH/build &&
             tar -xf target.tar.bz2 &&
             unzip -q -d tests target.common.tests.zip &&
             unzip -q -d tests target.mochitest.tests.zip &&
             export GECKO_BINARY_PATH=$USE_ARTIFACT_PATH/build/firefox/firefox &&
             export TEST_HARNESS_ROOT=$USE_ARTIFACT_PATH/build/tests &&
             cd /builds/worker/checkouts/gecko &&
-            ./mach python-test --subsuite mochitest
+            ./mach python-test --python 2 --subsuite mochitest
     when:
         files-changed:
             - 'testing/mochitest/**'
             - 'testing/mozbase/moztest/moztest/selftest/**'
             - 'testing/mozharness/mozharness/base/log.py'
             - 'testing/mozharness/mozharness/mozilla/structuredlog.py'
             - 'testing/mozharness/mozharness/mozilla/testing/errors.py'
             - 'testing/profiles/**'
 
 mozbase:
     description: testing/mozbase unit tests
+    python-version: [2]
     treeherder:
-        symbol: py(mb)
+        symbol: mb
     run:
-        mach: python-test --subsuite mozbase
+        using: python-test
+        subsuite: mozbase
     when:
         files-changed:
             - 'testing/mozbase/**'
 
 mozharness:
     description: mozharness integration tests
     treeherder:
-        symbol: py(mh)
+        symbol: py2(mh)
     run:
         using: run-task
         cache-dotcache: true
         command: >
             cd /builds/worker/checkouts/gecko/testing/mozharness &&
             /usr/local/bin/tox -e py27-hg4.3
     when:
         files-changed:
             - 'testing/mozharness/**'
 
 mozlint:
     description: python/mozlint unit tests
+    python-version: [2]
     treeherder:
-        symbol: py(ml)
+        symbol: ml
     run:
-        mach: python-test --subsuite mozlint
+        using: python-test
+        subsuite: mozlint
     when:
         files-changed:
             - 'python/mozlint/**'
             - 'tools/lint/**'
 
 mozterm:
     description: python/mozterm unit tests
+    python-version: [2]
     treeherder:
-        symbol: py(term)
+        symbol: term
     run:
-        mach: python-test --subsuite mozterm
+        using: python-test
+        subsuite: mozterm
     when:
         files-changed:
             - 'python/mozterm/**'
 
 mozversioncontrol:
     description: python/mozversioncontrol unit tests
+    python-version: [2]
     treeherder:
-        symbol: py(vcs)
+        symbol: vcs
     run:
-        mach: python-test --subsuite mozversioncontrol
+        using: python-test
+        subsuite: mozversioncontrol
     when:
         files-changed:
             - 'python/mozversioncontrol/**'
 
 raptor:
     description: testing/raptor unit tests
+    python-version: [2]
     treeherder:
-        symbol: py(rap)
+        symbol: rap
     run:
-        mach: python-test --subsuite raptor
+        using: python-test
+        subsuite: raptor
     when:
         files-changed:
             - 'testing/raptor/**'
 
 reftest-harness:
     description: layout/tools/reftest unittests
     platform:
         - linux64/opt
         - linux64/debug
         - linux64-asan/opt
     require-build: true
     always-target: false
     treeherder:
-        symbol: py(ref)
+        symbol: py2(ref)
     worker:
         by-platform:
             linux64.*:
                 docker-image: {in-tree: "desktop1604-test"}
                 max-run-time: 3600
     run:
         using: run-task
         use-artifacts:
@@ -177,26 +191,28 @@ reftest-harness:
             start_xvfb '1600x1200x24' 0 &&
             cd $USE_ARTIFACT_PATH/build &&
             tar -xf target.tar.bz2 &&
             unzip -q -d tests target.common.tests.zip &&
             unzip -q -d tests target.reftest.tests.zip &&
             export GECKO_BINARY_PATH=$USE_ARTIFACT_PATH/build/firefox/firefox &&
             export TEST_HARNESS_ROOT=$USE_ARTIFACT_PATH/build/tests &&
             cd /builds/worker/checkouts/gecko &&
-            ./mach python-test --subsuite reftest
+            ./mach python-test --python 2 --subsuite reftest
     when:
         files-changed:
             - 'layout/tools/reftest/**'
             - 'testing/mozbase/moztest/moztest/selftest/**'
             - 'testing/mozharness/mozharness/base/log.py'
             - 'testing/mozharness/mozharness/mozilla/structuredlog.py'
             - 'testing/mozharness/mozharness/mozilla/testing/errors.py'
 
 tryselect:
     description: tools/tryselect unit tests
+    python-version: [2]
     treeherder:
-        symbol: py(try)
+        symbol: try
     run:
-        mach: python-test --subsuite try
+        using: python-test
+        subsuite: try
     when:
         files-changed:
             - 'tools/tryselect/**'
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -142,16 +142,17 @@ following ``run-using`` are available
   * ``mozharness``
   * ``mozharness-test``
   * ``run-task``
   * ``spidermonkey`` or ``spidermonkey-package`` or ``spidermonkey-mozjs-crate`` or ``spidermonkey-rust-bindings``
   * ``debian-package``
   * ``toolchain-script``
   * ``always-optimized``
   * ``fetch-url``
+  * ``python-test``
 
 
 Task Descriptions
 -----------------
 
 Every kind needs to create tasks, and all of those tasks have some things in
 common.  They all run on one of a small set of worker implementations, each
 with their own idiosyncracies.  And they all report to TreeHerder in a similar
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/python_test.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running mach python-test tasks (via run-task)
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
+from taskgraph.util.schema import Schema
+from voluptuous import Required
+
+python_test_schema = Schema({
+    Required('using'): 'python-test',
+
+    # Python version to use
+    Required('python-version'): int,
+
+    # The subsuite to run
+    Required('subsuite'): basestring,
+
+    # Base work directory used to set up the task.
+    Required('workdir'): basestring,
+})
+
+
+@run_job_using(
+    'docker-worker',
+    'python-test',
+    schema=python_test_schema,
+    defaults={'python-version': 2, 'subsuite': 'default'})
+def docker_worker_python_test(config, job, taskdesc):
+    run = job['run']
+
+    # defer to the run_task implementation
+    run['command'] = 'cd {workdir}/checkouts/gecko && ' \
+        './mach python-test --python {python-version} --subsuite {subsuite}'.format(**run)
+    run['using'] = 'run-task'
+    del run['python-version']
+    del run['subsuite']
+    configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -14,19 +14,22 @@ import os
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.job import job_description_schema
 from taskgraph.util.attributes import keymatch
 from taskgraph.util.schema import (
     validate_schema,
     resolve_keyed_by,
 )
+from taskgraph.util.treeherder import join_symbol, split_symbol
+
 from voluptuous import (
     Any,
     Extra,
+    Optional,
     Required,
     Schema,
 )
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 
 job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
 
@@ -51,16 +54,17 @@ source_test_description_schema = Schema(
     Required('worker-type'): Any(
         job_description_schema['worker-type'],
         {'by-platform': {basestring: job_description_schema['worker-type']}},
     ),
     Required('worker'): Any(
         job_description_schema['worker'],
         {'by-platform': {basestring: job_description_schema['worker']}},
     ),
+    Optional('python-version'): [int],
 })
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def set_defaults(config, jobs):
     for job in jobs:
@@ -98,16 +102,37 @@ def expand_platforms(config, jobs):
 
             if 'name' in pjob:
                 pjob['name'] = '{}-{}'.format(pjob['name'], platform)
             else:
                 pjob['label'] = '{}-{}'.format(pjob['label'], platform)
             yield pjob
 
 
+@transforms.add
+def split_python(config, jobs):
+    for job in jobs:
+        key = 'python-version'
+        versions = job.pop(key, [])
+        if not versions:
+            yield job
+            continue
+        for version in versions:
+            group = 'py{0}'.format(version)
+            pyjob = copy.deepcopy(job)
+            if 'name' in pyjob:
+                pyjob['name'] += '-{0}'.format(group)
+            else:
+                pyjob['label'] += '-{0}'.format(group)
+            symbol = split_symbol(pyjob['treeherder']['symbol'])[1]
+            pyjob['treeherder']['symbol'] = join_symbol(group, symbol)
+            pyjob['run'][key] = version
+            yield pyjob
+
+
 def add_build_dependency(config, job):
     """
     Add build dependency to the job and installer_url to env.
     """
     key = job['platform']
     build_labels = config.config.get('dependent-build-platforms', {})
     matches = keymatch(build_labels, key)
     if not matches: