Bug 1289643 - Change path for checkouts from "workspace" to "checkouts"; r?dustin
Currently, TaskCluster tasks tend to use the "workspace" directory as
a cache that manages the source checkout *and* additional state.
Historically at Mozilla, we've lumped "source checkout" and "workspace"
(sometimes known as an "objdir") into the same directory. This is
not ideal. Ideally, there is an immutable, read-only source checkout
and all files produced from that source live in a separate directory.
In this commit, the "workspace" directory for the "lint" image has been
renamed to "checkouts" and all tasks using the image have been updated
accordingly. By having "checkout" in the name, we clearly identify this
cache as being relevant to source checkouts, which IMO can serve a
different role from "workspaces." This distinction is important, as the
next commit will prevent the "checkouts" cache from getting optimized
out in certain tasks.
To hammer this point home, documentation on common caches has been
introduced.
MozReview-Commit-ID: BSEc4dM5YCt
--- a/taskcluster/ci/legacy/tasks/lint.yml
+++ b/taskcluster/ci/legacy/tasks/lint.yml
@@ -19,25 +19,25 @@ task:
provisionerId: aws-provisioner-v1
schedulerId: task-graph-scheduler
routes:
- 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}'
- 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}'
scopes:
- 'docker-worker:cache:level-{{level}}-hg-shared'
- - 'docker-worker:cache:level-{{level}}-workspace'
+ - 'docker-worker:cache:level-{{level}}-checkouts'
payload:
# Thirty minutes should be enough for lint checks
maxRunTime: 1800
cache:
level-{{level}}-hg-shared: '/home/worker/hg-shared'
- level-{{level}}-workspace: '/home/worker/workspace'
+ level-{{level}}-checkouts: '/home/worker/checkouts'
env:
GECKO_BASE_REPOSITORY: '{{base_repository}}'
GECKO_HEAD_REPOSITORY: '{{head_repository}}'
GECKO_HEAD_REV: '{{head_rev}}'
extra:
build_product: '{{build_product}}'
--- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
@@ -16,21 +16,21 @@ task:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
- /home/worker/bin/checkout-gecko-and-run
- - /home/worker/workspace/gecko
+ - /home/worker/checkouts/gecko
- bash
- -cx
- >
- cd /home/worker/workspace/gecko/tools/lint/eslint &&
+ cd /home/worker/checkouts/gecko/tools/lint/eslint &&
/build/tooltool.py fetch -m manifest.tt &&
tar xvfz eslint.tar.gz &&
rm eslint.tar.gz &&
ln -s ../eslint-plugin-mozilla node_modules &&
cd ../../.. &&
tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
extra:
--- a/taskcluster/ci/legacy/tasks/tests/mozharness-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozharness-gecko.yml
@@ -22,21 +22,21 @@ task:
taskId:
task-reference: "<docker-image>"
cache:
level-{{level}}-{{project}}-dotcache: '/home/worker/.cache'
command:
- /home/worker/bin/checkout-gecko-and-run
- - /home/worker/workspace/gecko
+ - /home/worker/checkouts/gecko
- bash
- -cx
- >
- cd /home/worker/workspace/gecko/testing/mozharness &&
+ cd /home/worker/checkouts/gecko/testing/mozharness &&
/usr/bin/pip2 install tox &&
/home/worker/.local/bin/tox -e py27-hg3.7
extra:
extra:
build_product: '{{build_product}}'
locations:
build: null
tests: null
--- a/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
+++ b/taskcluster/ci/legacy/tasks/tests/mozlint-flake8.yml
@@ -14,21 +14,21 @@ task:
payload:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
- /home/worker/bin/checkout-gecko-and-run
- - /home/worker/workspace/gecko
+ - /home/worker/checkouts/gecko
- bash
- -cx
- >
- cd /home/worker/workspace/gecko &&
+ cd /home/worker/checkouts/gecko &&
./mach lint -l flake8 -f treeherder
extra:
locations:
build: null
tests: null
treeherder:
machine:
platform: lint
--- a/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
+++ b/taskcluster/ci/legacy/tasks/tests/taskgraph-tests.yml
@@ -14,21 +14,21 @@ task:
payload:
image:
type: 'task-image'
path: 'public/image.tar'
taskId:
task-reference: "<docker-image>"
command:
- /home/worker/bin/checkout-gecko-and-run
- - /home/worker/workspace/gecko
+ - /home/worker/checkouts/gecko
- bash
- -cx
- >
- cd /home/worker/workspace/gecko &&
+ cd /home/worker/checkouts/gecko &&
./mach taskgraph python-tests
extra:
locations:
build: null
tests: null
treeherder:
machine:
# this should probably get a fresh new platform, but linux64 is what, for example, marionette uses
new file mode 100644
--- /dev/null
+++ b/taskcluster/docs/caches.rst
@@ -0,0 +1,43 @@
+.. taskcluster_caches:
+
+=============
+Common Caches
+=============
+
+There are various caches used by the in-tree tasks. This page attempts to
+document them and their appropriate use.
+
+Version Control Caches
+======================
+
+level-{{level}}-hg-shared
+ This cache holds Mercurial *stores*: this is the historical data of a
+ Mercurial repository (as opposed to a checkout or working directory).
+
+ This cache should ideally be written to by the ``hg robustcheckout``
+ command, which is the preferred mechanism to clone a Mercurial repository
+ in automation.
+
+ When using ``hg robustcheckout``, data for the same logical repository
+ will automatically be stored in the same directory in this cache. e.g.
+ data for the Firefox repository will be in the directory
+ ``8ba995b74e18334ab3707f27e9eb8f4e37ba3d29``, which is the SHA-1 of the
+ first revision in the Firefox repository. This helps ensure that any
+ given changeset is only stored once in the cache. This is why there is
+ no ``{{project}}`` component in the cache name.
+
+level-{{level}}-checkouts
+ This cache holds version control checkouts or working directories. This
+ is distinct from the cache that holds the version control *store* data.
+
+ Ideally, checkouts should be read-only. If a task needs to create new
+ files from content of a checkout, this content should be written in
+ a separate directory/cache (like a workspace).
+
+Workspace Caches
+================
+
+level-{{level}}-*-workspace
+ These caches (of various names typically ending with ``workspace``)
+ contain state to be shared between task invocations. Use cases are
+ dependent on the task.
--- a/taskcluster/docs/index.rst
+++ b/taskcluster/docs/index.rst
@@ -24,8 +24,9 @@ check out the :doc:`how-to section <how-
taskgraph
parameters
attributes
kinds
transforms
yaml-templates
how-tos
docker-images
+ caches
--- a/testing/docker/recipes/checkout-gecko-and-run
+++ b/testing/docker/recipes/checkout-gecko-and-run
@@ -1,27 +1,29 @@
#!/bin/bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
set -ex
+DEST=$1
+CHECKOUTS=`dirname ${DEST}`
+
# The script starts executing as root. We need to change ownership
# of the caches because they are initially owned by root:root. There
# may not be a cache mount/directory on some instances. So create the
# directory if missing.
if [ $(id -u) = 0 ]; then
- mkdir -p /home/worker/workspace
- chown worker:worker /home/worker/hg-shared /home/worker/workspace
+ mkdir -p ${CHECKOUTS}
+ chown worker:worker /home/worker/hg-shared ${CHECKOUTS}
exec sudo -E -u worker ${0} "${@}"
fi
-DEST=$1
shift
# We set the base repository to mozilla-central so tc-vcs doesn't get
# confused. Switch to mozilla-unified because robustcheckout works best
# with it.
if [ "${GECKO_BASE_REPOSITORY}" = "https://hg.mozilla.org/mozilla-central" ]; then
GECKO_BASE_REPOSITORY=https://hg.mozilla.org/mozilla-unified
fi