Bug 1401189 - Part 1: Add new npm TC kind.
This patch adds a new npm kind. A single npm declaration corresponds
to a single npm command against a single package.json in the source
tree. The npm declaration expands to two task declarations:
1) a toolchain task that runs |npm install| and collects the resulting
`node_modules` directory;
2) another task that installs the `node_modules` directory produced
by the toolchain task and then runs whatever npm command was
specified.
This two stage process -- dependency collection before task execution
-- has worked well for the Android-Gradle dependency hierarchy.
MozReview-Commit-ID: GAuTdOVVsfH
--- a/build/sparse-profiles/taskgraph
+++ b/build/sparse-profiles/taskgraph
@@ -37,10 +37,13 @@ glob:**/*.configure
# Tooltool manifests also need to be opened. Assume they
# are all somewhere in "tooltool-manifests" directories.
glob:**/tooltool-manifests/**
# For scheduling android-gradle-dependencies.
path:mobile/android/config/
glob:**/*.gradle
+# For scheduling npm tasks.
+glob:**/package.json
+
# for action-task building
path:.taskcluster.yml
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -1,12 +1,13 @@
trust-domain: gecko
treeherder:
group-names:
'cram': 'Cram tests'
+ 'npm': 'npm scripts'
'mocha': 'Mocha unit tests'
'py': 'Python unit tests'
'tc': 'Executed by TaskCluster'
'tc-A': 'Android Gradle tests executed by TaskCluster'
'tc-e10s': 'Executed by TaskCluster with e10s'
'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster'
'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s'
'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster'
copy from taskcluster/ci/diffoscope/kind.yml
copy to taskcluster/ci/npm/kind.yml
--- a/taskcluster/ci/diffoscope/kind.yml
+++ b/taskcluster/ci/npm/kind.yml
@@ -1,59 +1,23 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
kind-dependencies:
- - build
- toolchain
transforms:
- - taskgraph.transforms.diffoscope:transforms
+ - taskgraph.transforms.npm:transforms
- taskgraph.transforms.use_toolchains:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
-# Note: --exclude-command .--line-numbers is because of
-# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=879003
-# That skips `objdump --disassemble --line-numbers` and falls back to
-# `objdump --disassemble`
-# Note: the .chk excludes are for files that are known to differ between
-# builds because they are signed with an ephemeral private key that is
-# generated for each build.
job-defaults:
- args: >-
- --no-default-limits
- --max-page-size 100000000
- --max-page-diff-block-lines 10000
- --exclude-directory-metadata
- --exclude-command .--line-numbers
- --exclude **/*freeblpriv3.chk
- --exclude **/*nssdbm3.chk
- --exclude **/*softokn3.chk
-# Make a task for each diff we might want. The following are just examples,
-# Both original and new can point to builds from the full set of tasks or
-# from other sets through an index-search. Other kinds than `build` can be
-# compared (for example, static-analysis), provided you adjust the
-# kind-dependencies above.
-
-# jobs:
-# android-build-vs-previous-try:
-# symbol: A
-# new: build-android-api-16/opt
-# original: {index-search: gecko.v2.try.revision.aabd5deb0156f9b55ab60ad6a01ebfc4580bf2e1.mobile.android-api-16-opt}
-# linux64-build-vs-previous-try:
-# symbol: L
-# new: build-linux64/opt
-# original: {index-search: gecko.v2.try.revision.aabd5deb0156f9b55ab60ad6a01ebfc4580bf2e1.firefox.linux64-opt}
-# extra-args: >-
-# --exclude-command .--hex-dump=.gnu_debuglink
-# macosx-build-vs-previous-try:
-# symbol: M
-# new: build-macosx64/opt
-# original: {index-search: gecko.v2.try.revision.aabd5deb0156f9b55ab60ad6a01ebfc4580bf2e1.firefox.macosx64-opt}
-# win32-build-vs-previous-try:
-# symbol: W
-# new: build-win32/opt
-# original: {index-search: gecko.v2.try.revision.aabd5deb0156f9b55ab60ad6a01ebfc4580bf2e1.firefox.win32-opt}
+jobs:
+ eslint-plugin-mozilla:
+ description: eslint-plugin-mozilla integration tests
+ module: tools/lint/eslint/eslint-plugin-mozilla
+ npm: run test
+ symbol: epm
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -358,8 +358,12 @@ packages
--------
Tasks used to build packages for use in docker images.
diffoscope
----------
Tasks used to compare pairs of Firefox builds using https://diffoscope.org/.
As of writing, this is mainly meant to be used in try builds, by editing
taskcluster/ci/diffoscope/kind.yml for your needs.
+
+npm
+----------
+Tasks used to run npm script commands.
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -137,16 +137,17 @@ part of the documentation.
following ``run-using`` are available
* ``buildbot``
* ``hazard``
* ``mach``
* ``mozharness``
* ``mozharness-test``
+ * ``npm``
* ``run-task``
* ``spidermonkey`` or ``spidermonkey-package`` or ``spidermonkey-mozjs-crate`` or ``spidermonkey-rust-bindings``
* ``debian-package``
* ``toolchain-script``
* ``always-optimized``
Task Descriptions
copy from taskcluster/scripts/misc/android-gradle-dependencies.sh
copy to taskcluster/scripts/misc/repack-node-modules.sh
--- a/taskcluster/scripts/misc/android-gradle-dependencies.sh
+++ b/taskcluster/scripts/misc/repack-node-modules.sh
@@ -5,18 +5,33 @@ set -x -e
echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
set -v
cd $WORKSPACE/build/src
+MODULE_NAME=$1
+MODULE_PATH=$2
+echo "packaging node_modules for module $MODULE_NAME at path $MODULE_PATH"
+
+if [ ! -e "$MODULE_PATH/package.json" ]; then
+ echo "cannot find $MODULE_PATH/package.json"
+ exit 1
+fi
+
# Download toolchain artifacts.
. taskcluster/scripts/misc/tooltool-download.sh
-. taskcluster/scripts/misc/android-gradle-dependencies/before.sh
+cd $MODULE_PATH
+
+rm -rf node_modules
+rm -f package-lock.json
+
+npm install
-export MOZCONFIG=mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
-./mach build
-./mach android gradle-dependencies
+# Package everything up.
+mkdir -p node_modules-$MODULE_NAME /builds/worker/artifacts
-. taskcluster/scripts/misc/android-gradle-dependencies/after.sh
+cp -R $WORKSPACE/build/src/$MODULE_PATH/node_modules node_modules-$MODULE_NAME
+
+tar cf - node_modules-$MODULE_NAME | xz > /builds/worker/artifacts/node_modules-$MODULE_NAME.tar.xz
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/npm.py
@@ -0,0 +1,125 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+
+This transform constructs toolchain tasks to collect npm package dependencies
+via |npm install| and |npm ...| tasks that consume them.
+
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import (
+ Schema,
+)
+from voluptuous import (
+ Required,
+)
+
+transforms = TransformSequence()
+
+npm = Schema({
+ # Name of the npm task.
+ Required('name'): basestring,
+
+ # Treeherder symbol.
+ Required('symbol'): basestring,
+
+ # npm command, without the 'npm', like 'test'.
+ Required('npm'): basestring,
+
+ # Path to npm module, like 'path/to/module' where
+ # `$topsrcdir/path/to/module/package.json` exists.
+ Required('module'): basestring,
+
+ # TODO: support yarn and npm?
+ # TODO: path to package.json rather than directory, which is awkward for $topsrcdir/package.json.
+ # TODO: multiple npm commands/verbs in one declaration.
+})
+
+
+@transforms.add
+def fill_template(config, tasks):
+ for task in tasks:
+ name = task['name']
+
+ # TODO: verify $topsrcdir/{module}/package.json exists.
+ module_path = task['module'].rstrip('/')
+ module_name = os.path.basename(module_path)
+
+ tooldesc = {
+ # 'toolchains': ['node_modules-{}'.format(module_name)],
+ 'name': 'linux64-node_modules-{}'.format(module_name),
+ # 'label': 'linux64-node_modules-{}'.format(module_name),
+ 'description': '{} node_modules toolchain task'.format(module_name),
+ 'treeherder': {
+ 'symbol': 'TL({})'.format(task['symbol']),
+ 'platform': 'toolchains/opt',
+ 'kind': 'other',
+ 'tier': 2,
+ },
+ 'when': {
+ 'files-changed': ['{}/package.json'.format(module_path)],
+ },
+ 'worker-type': 'aws-provisioner-v1/gecko-{}-b-linux'.format(
+ config.params['level']),
+ 'worker': {
+ 'docker-image': {'in-tree': 'lint'}, # TODO: figure out a better image for Node.js/npm tests.
+ 'max-run-time': 1800,
+ },
+ 'run': {
+ 'using': 'toolchain-script',
+ 'script': 'repack-node-modules.sh',
+ 'arguments': [
+ module_name,
+ module_path,
+ ],
+ 'sparse-profile': None,
+ 'resources': [
+ 'taskcluster/scripts/misc/tooltool-download.sh',
+ '{}/package.json'.format(module_path),
+ ],
+ 'toolchain-artifact': 'public/build/node_modules-{}.tar.xz'.format(module_name),
+ },
+ }
+
+ taskdesc = {
+ # 'label': 'npm-' + name,
+ 'name': name,
+ 'description': name,
+ 'treeherder': {
+ 'symbol': 'npm({})'.format(task['symbol']),
+ 'platform': 'npm/opt',
+ 'kind': 'other',
+ 'tier': 2,
+ },
+ 'when': {
+ 'files-changed': ['{}/**'.format(module_path)],
+ },
+ 'worker-type': 'aws-provisioner-v1/gecko-{}-b-linux'.format(
+ config.params['level']),
+ 'worker': {
+ 'docker-image': {'in-tree': 'lint'}, # TODO: figure out a better image for Node.js/npm tests.
+ 'max-run-time': 1800,
+ },
+ 'run': {
+ 'using': 'run-task',
+ 'command': (
+ 'cd /builds/worker/checkouts/gecko && '
+ '. taskcluster/scripts/misc/tooltool-download.sh && '
+ 'cd /builds/worker/checkouts/gecko/{module_path} && '
+ 'ln -s /builds/worker/checkouts/gecko/node_modules-{module_name}/node_modules node_modules && '
+ 'npm {npm}').format(
+ module_name=module_name,
+ module_path=module_path,
+ npm=task['npm']),
+ },
+ 'toolchains': ['linux64-node_modules-{}'.format(module_name)],
+ }
+
+ yield tooldesc
+ yield taskdesc
--- a/taskcluster/taskgraph/transforms/use_toolchains.py
+++ b/taskcluster/taskgraph/transforms/use_toolchains.py
@@ -12,40 +12,45 @@ transforms = TransformSequence()
@transforms.add
def use_toolchains(config, jobs):
"""Add dependencies corresponding to toolchains to use, and pass a list
of corresponding artifacts to jobs using toolchains.
"""
artifacts = {}
aliases_by_job = {}
+ kinds = {}
def get_attribute(dict, key, attributes, attribute_name):
'''Get `attribute_name` from the given `attributes` dict, and if there
- is a corresponding value, set `key` in `dict` to that value.'''
+ is a corresponding value, set `key` in `dict` to that value.
+ Returns the value.'''
value = attributes.get(attribute_name)
if value:
dict[key] = value
+ return value
- # Toolchain jobs can depend on other toolchain jobs, but we don't have full
- # tasks for them, since they're being transformed. So scan the jobs list in
- # that case, otherwise, use the list of tasks for the kind dependencies.
- if config.kind == 'toolchain':
- jobs = list(jobs)
- for job in jobs:
- run = job.get('run', {})
- get_attribute(artifacts, job['name'], run, 'toolchain-artifact')
+ # Any job can be a toolchain job -- it just needs to produce a
+ # toolchain-artifact. That is, there are toolchain jobs that aren't of the
+ # toolchain kind. Toolchain jobs can depend on other toolchain jobs, and
+ # for jobs of the same kind we don't have full tasks for them, since
+ # they're being transformed. So we first scan the jobs list in that case,
+ # and then we use the list of tasks for the kind dependencies.
+ jobs = list(jobs)
+ for job in jobs:
+ run = job.get('run', {})
+ if get_attribute(artifacts, job['name'], run, 'toolchain-artifact'):
get_attribute(aliases_by_job, job['name'], run, 'toolchain-alias')
- else:
- for task in config.kind_dependencies_tasks:
- if task.kind != 'toolchain':
- continue
- name = task.label.replace('%s-' % task.kind, '')
- get_attribute(artifacts, name, task.attributes, 'toolchain-artifact')
+ kinds[job['name']] = config.kind
+
+ for task in config.kind_dependencies_tasks:
+ name = task.label.replace('%s-' % task.kind, '')
+ if get_attribute(artifacts, name, task.attributes, 'toolchain-artifact'):
get_attribute(aliases_by_job, name, task.attributes, 'toolchain-alias')
+ kinds[name] = task.kind
aliases = {}
for job, alias in aliases_by_job.items():
if alias in aliases:
raise Exception(
"Cannot use the alias %s for %s, it's already used for %s"
% (alias, job, aliases[alias]))
if alias in artifacts:
@@ -103,22 +108,22 @@ def use_toolchains(config, jobs):
if scope not in scopes:
scopes.append(scope)
if t.endswith('-sccache'):
job['needs-sccache'] = True
if toolchains:
job.setdefault('dependencies', {}).update(
- ('toolchain-%s' % t, 'toolchain-%s' % t)
+ ('%s-%s' % (kinds[t], t), '%s-%s' % (kinds[t], t))
for t in toolchains
)
# Pass a list of artifact-path@task-id to the job for all the
# toolchain artifacts it's going to need, where task-id is
# corresponding to the (possibly optimized) toolchain job, and
# artifact-path to the toolchain-artifact defined for that
# toolchain job.
env['MOZ_TOOLCHAINS'] = {'task-reference': ' '.join(
- '%s@<toolchain-%s>' % (artifacts[t], t)
+ '%s@<%s-%s>' % (artifacts[t], kinds[t], t)
for t in toolchains
)}
yield job