Bug 1409876 - Print warning when vcs operations aren't running on a cache or volume; r?dustin
A lot of effort has been spent optimizing VCS operations for peak
performance. But not utilizing caches or volumes for the VCS store
or checkouts can undermine that work.
Let's print a warning when VCS is configured sub-optimally.
I'm pretty sure we still have some rogue tasks not using caches
or volumes. We can convert this to a fatal error once those are
fixed.
MozReview-Commit-ID: C6CT1zViy75
--- a/taskcluster/docker/recipes/run-task
+++ b/taskcluster/docker/recipes/run-task
@@ -462,28 +462,51 @@ def main(args):
# The volume is almost certainly owned by root:root. Chown it so it
# is writable.
if running_as_root:
print_line(b'volume', b'changing ownership of volume %s '
b'to %d:%d\n' % (volume, uid, gid))
set_dir_permissions(volume, uid, gid)
+ all_caches_and_volumes = set(map(os.path.normpath, caches))
+ all_caches_and_volumes |= set(map(os.path.normpath, volumes))
+
+ def path_in_cache_or_volume(path):
+ path = os.path.normpath(path)
+
+ while path:
+ if path in all_caches_and_volumes:
+ return True
+
+ path, child = os.path.split(path)
+ if not child:
+ break
+
+ return False
+
def prepare_checkout_dir(checkout):
if not checkout:
return
# The checkout path becomes the working directory. Since there are
# special cache files in the cache's root directory and working
# directory purging could blow them away, disallow this scenario.
if os.path.exists(os.path.join(checkout, '.cacherequires')):
print('error: cannot perform vcs checkout into cache root: %s' %
checkout)
sys.exit(1)
+ # TODO given the performance implications, consider making this a fatal
+ # error.
+ if not path_in_cache_or_volume(checkout):
+ print_line(b'vcs', b'WARNING: vcs checkout path (%s) not in cache '
+ b'or volume; performance will likely suffer\n' %
+ checkout)
+
# Ensure the directory for the source checkout exists.
try:
os.makedirs(os.path.dirname(checkout))
except OSError as e:
if e.errno != errno.EEXIST:
raise
# And that it is owned by the appropriate user/group.
@@ -492,16 +515,22 @@ def main(args):
def prepare_hg_store_path():
# And ensure the shared store path exists and has proper permissions.
if 'HG_STORE_PATH' not in os.environ:
print('error: HG_STORE_PATH environment variable not set')
sys.exit(1)
store_path = os.environ['HG_STORE_PATH']
+
+ if not path_in_cache_or_volume(store_path):
+ print_line(b'vcs', b'WARNING: HG_STORE_PATH (%s) not in cache or '
+ b'volume; performance will likely suffer\n' %
+ store_path)
+
try:
os.makedirs(store_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if running_as_root:
os.chown(store_path, uid, gid)