Bug 1430037 - Use sparse checkouts for docker image builds. r?dustin
In many cases, building docker images starts on machines that don't have
a cached checkout, and it often takes forever to get a full clone. It
used to be worsened when 3 jobs could run at the same time because the
worker would start up clean, and 3 jobs would be doing a mercurial clone
at the same time, thrashing I/O, but that part is fortunately fixed.
It is still, however, appreciable not to waste time in the mercurial
clone part of image creation.
new file mode 100644
--- /dev/null
+++ b/build/sparse-profiles/docker-image
@@ -0,0 +1,16 @@
+%include build/sparse-profiles/mach
+
+[include]
+path:taskcluster/
+
+# Result from `grep -hr %include taskcluster/docker | grep -v " taskcluster/" | sort -u`
+path:build/unix/build-gtk3/build-gtk3.sh
+path:build/valgrind/valgrind-epochs.patch
+path:python/mozbuild/mozbuild/action/tooltool.py
+path:testing/config/tooltool-manifests/linux64/releng.manifest
+path:testing/mozharness/external_tools/robustcheckout.py
+path:tools/lint/eslint/eslint-plugin-mozilla/manifest.tt
+path:tools/lint/eslint/manifest.tt
+path:tools/lint/python/flake8_requirements.txt
+path:tools/lint/tox/tox_requirements.txt
+
--- a/taskcluster/docker/image_builder/build-image.sh
+++ b/taskcluster/docker/image_builder/build-image.sh
@@ -19,16 +19,17 @@ test -n "$HASH" || raise_error "Co
test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided."
# Construct a CONTEXT_FILE
CONTEXT_FILE=/builds/worker/workspace/context.tar
# Run ./mach taskcluster-build-image with --context-only to build context
run-task \
--vcs-checkout "/builds/worker/checkouts/gecko" \
+ --sparse-profile build/sparse-profiles/docker-image \
-- \
/builds/worker/checkouts/gecko/mach taskcluster-build-image \
--context-only "$CONTEXT_FILE" \
"$IMAGE_NAME"
test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created"
# Create artifact folder (note that this must occur after run-task)
mkdir -p /builds/worker/workspace/artifacts
--- a/taskcluster/taskgraph/transforms/docker_image.py
+++ b/taskcluster/taskgraph/transforms/docker_image.py
@@ -163,17 +163,17 @@ def fill_template(config, tasks):
# docker image whose digest is referenced above.
worker['volumes'] = [
'/builds/worker/checkouts',
'/builds/worker/workspace',
]
cache_name = 'imagebuilder-v1'
else:
worker['docker-image'] = {'in-tree': 'image_builder'}
- cache_name = 'imagebuilder-{}'.format(_run_task_suffix())
+ cache_name = 'imagebuilder-sparse-{}'.format(_run_task_suffix())
# Force images built against the in-tree image builder to
# have a different digest by adding a fixed string to the
# hashed data.
digest_data.append('image_builder')
worker['caches'] = [{
'type': 'persistent',
'name': 'level-{}-{}'.format(config.params['level'], cache_name),