Bug 1247168 - Convert lint image and tasks to use robustcheckout; r?dustin
The robustcheckout Mercurial extension does a clone+checkout optimally.
Read the bug for more on it.
robustcheckout is already used by mozharness automation. It has resulted
in a significant reduction in I/O usage and utilization in automation.
This commit replaces tc-vcs with the robustcheckout equivalent.
We replace the existing tc-vcs scope and cache with a new one.
Because Dustin and I are paranoid, we maintain separate caches per
SCM level - even though we could arguably share the same cache. Defense
in depth.
Robustcheckout (when used with --sharebase) pools storage for related
repos automatically. i.e. changesets from inbound and central will
be in the same store. This means you likely only have one copy of
each changeset per cache. This can result in significant space savings.
And, since there are fewer copies floating around, hg.mozilla.org
and various network appliances are working less too!
Since tc-vcs is no longer used, we stop it from being installed.
While we're here, we also change the images to execute as the
"worker" user. This happens automatically as a result of using
the "checkout-and-run" script.
MozReview-Commit-ID: EDeebuP7TkT
--- a/taskcluster/ci/legacy/tasks/lint.yml
+++ b/taskcluster/ci/legacy/tasks/lint.yml
@@ -18,26 +18,29 @@ task:
workerType: b2gtest
provisionerId: aws-provisioner-v1
schedulerId: task-graph-scheduler
routes:
- 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}'
- 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}'
scopes:
- # Nearly all of our build tasks use tc-vcs so just include the scope across
- # the board.
- - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
+ - 'docker-worker:cache:level-{{level}}-hg-shared'
payload:
# Thirty minutes should be enough for lint checks
maxRunTime: 1800
cache:
- level-{{level}}-{{project}}-tc-vcs: '/home/worker/.tc-vcs'
+ level-{{level}}-hg-shared: '/home/worker/hg-shared'
+
+ env:
+ GECKO_BASE_REPOSITORY: '{{base_repository}}'
+ GECKO_HEAD_REPOSITORY: '{{head_repository}}'
+ GECKO_HEAD_REV: '{{head_rev}}'
extra:
build_product: '{{build_product}}'
build_name: {{build_name}}
build_type: {{build_type}}
index:
rank: {{rank}}
treeherder:
--- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
@@ -15,20 +15,21 @@ task:
payload:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
+ - /home/worker/bin/checkout-gecko-and-run
+ - gecko
- bash
- -cx
- >
- tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
cd gecko/tools/lint/eslint &&
/build/tooltool.py fetch -m manifest.tt &&
tar xvfz eslint.tar.gz &&
rm eslint.tar.gz &&
cd ../../.. &&
tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
extra:
--- a/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
@@ -13,20 +13,21 @@ task:
description: 'Python flake8 linter'
payload:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
+ - /home/worker/bin/checkout-gecko-and-run
+ - gecko
- bash
- -cx
- >
- tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
cd gecko &&
./mach lint -l flake8 -f treeherder
extra:
locations:
build: null
tests: null
treeherder:
machine:
--- a/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
+++ b/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
@@ -13,20 +13,21 @@ task:
description: 'Run test suite for taskcluster/taskgraph'
payload:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
+ - /home/worker/bin/checkout-gecko-and-run
+ - gecko
- bash
- -cx
- >
- tc-vcs checkout ./gecko {{base_repository}} {{head_repository}} {{head_rev}} {{head_ref}} &&
cd gecko &&
./mach taskgraph python-tests
extra:
locations:
build: null
tests: null
treeherder:
machine:
--- a/testing/docker/lint/system-setup.sh
+++ b/testing/docker/lint/system-setup.sh
@@ -36,56 +36,44 @@ cd /build
###
# ESLint Setup
###
# install node
# For future reference things like this don't need to be uploaded to tooltool, as long
# as we verify the hash, we can download it from the external net.
+cd /setup
tooltool_fetch <<'EOF'
[
{
"size": 8310316,
"digest": "95f4fa3d9b215348393dfac4a1c5eff72e9ef85dca38eb69cc8e6c1fe5aada0136c3b182dc04ed5c19fb69f0ac7df85d9c4045b9eb382fcb545b0ccacfece25b",
"algorithm": "sha512",
"filename": "node-v4.4.5-linux-x64.tar.xz"
}
]
EOF
tar -C /usr/local --strip-components 1 -xJ < node-*.tar.xz
node -v # verify
npm -v
-# install taskcluster-vcs@2.3.12
-tooltool_fetch <<'EOF'
-[
-{
-"size": 6282161,
-"visibility": "public",
-"digest": "a781a96e596f6403eca6ec2300adb9c1a396659393e16993c66f98a658050e557bc681d521f70b50c1162aa4b435274e0098ffcbd37cbe969c0e4f69be19a1e0",
-"algorithm": "sha512",
-"filename": "taskcluster-vcs-v2.3.12.tar.gz"
-}
-]
-EOF
-npm install -g taskcluster-vcs-v2.3.12.tar.gz
-
-
###
# Flake8 Setup
###
+cd /setup
cat >requirements.txt <<'EOF'
flake8==2.5.4 \
--hash=sha256:fb5a67af4024622287a76abf6b7fe4fb3cfacf765a790976ce64f52c44c88e4a
mccabe==0.4.0 \
--hash=sha256:cbc2938f6c01061bc6d21d0c838c2489664755cb18676f0734d7617f4577d09e
pep8==1.7.0 \
--hash=sha256:4fc2e478addcf17016657dff30b2d8d611e8341fac19ccf2768802f6635d7b8a
pyflakes==1.2.3 \
--hash=sha256:e87bac26c62ea5b45067cc89e4a12f56e1483f1f2cda17e7c9b375b9fd2f40da
EOF
pip install --require-hashes -r requirements.txt
cd /
rm -rf /setup
+
--- a/testing/docker/recipes/install-mercurial.sh
+++ b/testing/docker/recipes/install-mercurial.sh
@@ -22,21 +22,38 @@ tooltool_fetch <<EOF
"algorithm": "sha512",
"filename": "mercurial-common_3.8.4_all.deb"
}
]
EOF
dpkg -i mercurial-common_3.8.4_all.deb mercurial_3.8.4_amd64.deb
+mkdir -p /usr/local/mercurial
+cd /usr/local/mercurial
+tooltool_fetch <<'EOF'
+[
+{
+ "size": 11849,
+ "digest": "c88d9b8afd6649bd28bbacfa654ebefec8087a01d1662004aae088d485edeb03a92df1193d1310c0369d7721f475b974fcd4a911428ec65936f7e40cf1609c49",
+ "algorithm": "sha512",
+ "filename": "robustcheckout.py"
+}
+]
+EOF
+chmod 644 /usr/local/mercurial/robustcheckout.py
+
mkdir -p /etc/mercurial
cat >/etc/mercurial/hgrc <<EOF
# By default the progress bar starts after 3s and updates every 0.1s. We
# change this so it shows and updates every 1.0s.
[progress]
delay = 1.0
refresh = 1.0
[web]
cacerts = /etc/ssl/certs/ca-certificates.crt
+
+[extensions]
+robustcheckout = /usr/local/mercurial/robustcheckout.py
EOF
chmod 644 /etc/mercurial/hgrc