Bug 1247168 - Use a cache for repo checkout in lint tasks; r?dustin
Previously, every lint task would have to create its own checkout. This
was time consuming.
The robustcheckout extension purges the working copy of *all*
untracked and ignored files. It also restores modified files to
their original state. In other words, as long as you trust Mercurial
to go from revision X to revision Y, robustcheckout is as good as
a fresh checkout.
This commit adds a cache for the working directory checkout so lint
tasks only have to effectively perform incremental `hg update` between
task executions. This should make tasks spend a lot less time
doing version control foo.
On Try, time for flake8 tasks is currently hovering around 4 minutes.
After this change, I've seen tasks finish as quickly as 11s! But that
was with a hacked up legacy.py that added the workspace cache to the
whitelist for Try. While I would like to see workspace reuse on Try
eventually, this is not the right commit to roll that out in.
MozReview-Commit-ID: 66P2rt896qE
--- a/taskcluster/ci/legacy/tasks/lint.yml
+++ b/taskcluster/ci/legacy/tasks/lint.yml
@@ -19,23 +19,25 @@ task:
provisionerId: aws-provisioner-v1
schedulerId: task-graph-scheduler
routes:
- 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}'
- 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}'
scopes:
- 'docker-worker:cache:level-{{level}}-hg-shared'
+ - 'docker-worker:cache:level-{{level}}-workspace'
payload:
# Thirty minutes should be enough for lint checks
maxRunTime: 1800
cache:
level-{{level}}-hg-shared: '/home/worker/hg-shared'
+ level-{{level}}-workspace: '/home/worker/workspace'
env:
GECKO_BASE_REPOSITORY: '{{base_repository}}'
GECKO_HEAD_REPOSITORY: '{{head_repository}}'
GECKO_HEAD_REV: '{{head_rev}}'
extra:
build_product: '{{build_product}}'
--- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
@@ -16,21 +16,21 @@ task:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
- /home/worker/bin/checkout-gecko-and-run
- - gecko
+ - /home/worker/workspace/gecko
- bash
- -cx
- >
- cd gecko/tools/lint/eslint &&
+ cd /home/worker/workspace/gecko/tools/lint/eslint &&
/build/tooltool.py fetch -m manifest.tt &&
tar xvfz eslint.tar.gz &&
rm eslint.tar.gz &&
cd ../../.. &&
tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
extra:
locations:
--- a/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
@@ -14,21 +14,21 @@ task:
payload:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
- /home/worker/bin/checkout-gecko-and-run
- - gecko
+ - /home/worker/workspace/gecko
- bash
- -cx
- >
- cd gecko &&
+ cd /home/worker/workspace/gecko &&
./mach lint -l flake8 -f treeherder
extra:
locations:
build: null
tests: null
treeherder:
machine:
platform: lint
--- a/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
+++ b/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
@@ -14,21 +14,21 @@ task:
payload:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
- /home/worker/bin/checkout-gecko-and-run
- - gecko
+ - /home/worker/workspace/gecko
- bash
- -cx
- >
- cd gecko &&
+ cd /home/worker/workspace/gecko &&
./mach taskgraph python-tests
extra:
locations:
build: null
tests: null
treeherder:
machine:
# this should probably get a fresh new platform, but linux64 is what, for example, marionette uses