Bug 1409876 - Print warning when vcs operations aren't running on a cache or volume; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Wed, 25 Oct 2017 17:38:35 -0700
changeset 686955 8e5abffbc6b7ee751ad0c74404f622972ad1b154
parent 686954 4a8a9d1528101d3a6734839c3c7722bb58be7202
child 737527 3e4fc025224b212a01f2b6f426987e523e5b16df
push id86355
push usergszorc@mozilla.com
push dateThu, 26 Oct 2017 16:53:42 +0000
reviewersdustin
bugs1409876
milestone58.0a1
Bug 1409876 - Print warning when vcs operations aren't running on a cache or volume; r?dustin A lot of effort has been spent optimizing VCS operations for peak performance. But not utilizing caches or volumes for the VCS store or checkouts can undermine that work. Let's print a warning when VCS is configured sub-optimally. I'm pretty sure we still have some rogue tasks not using caches or volumes. We can convert this to a fatal error once those are fixed. MozReview-Commit-ID: C6CT1zViy75
taskcluster/docker/recipes/run-task
--- a/taskcluster/docker/recipes/run-task
+++ b/taskcluster/docker/recipes/run-task
@@ -462,28 +462,51 @@ def main(args):
         # The volume is almost certainly owned by root:root. Chown it so it
         # is writable.
 
         if running_as_root:
             print_line(b'volume', b'changing ownership of volume %s '
                                   b'to %d:%d\n' % (volume, uid, gid))
             set_dir_permissions(volume, uid, gid)
 
+    all_caches_and_volumes = set(map(os.path.normpath, caches))
+    all_caches_and_volumes |= set(map(os.path.normpath, volumes))
+
+    def path_in_cache_or_volume(path):
+        path = os.path.normpath(path)
+
+        while path:
+            if path in all_caches_and_volumes:
+                return True
+
+            path, child = os.path.split(path)
+            if not child:
+                break
+
+        return False
+
     def prepare_checkout_dir(checkout):
         if not checkout:
             return
 
         # The checkout path becomes the working directory. Since there are
         # special cache files in the cache's root directory and working
         # directory purging could blow them away, disallow this scenario.
         if os.path.exists(os.path.join(checkout, '.cacherequires')):
             print('error: cannot perform vcs checkout into cache root: %s' %
                   checkout)
             sys.exit(1)
 
+        # TODO given the performance implications, consider making this a fatal
+        # error.
+        if not path_in_cache_or_volume(checkout):
+            print_line(b'vcs', b'WARNING: vcs checkout path (%s) not in cache '
+                               b'or volume; performance will likely suffer\n' %
+                               checkout)
+
         # Ensure the directory for the source checkout exists.
         try:
             os.makedirs(os.path.dirname(checkout))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         # And that it is owned by the appropriate user/group.
@@ -492,16 +515,22 @@ def main(args):
 
     def prepare_hg_store_path():
         # And ensure the shared store path exists and has proper permissions.
         if 'HG_STORE_PATH' not in os.environ:
             print('error: HG_STORE_PATH environment variable not set')
             sys.exit(1)
 
         store_path = os.environ['HG_STORE_PATH']
+
+        if not path_in_cache_or_volume(store_path):
+            print_line(b'vcs', b'WARNING: HG_STORE_PATH (%s) not in cache or '
+                               b'volume; performance will likely suffer\n' %
+                               store_path)
+
         try:
             os.makedirs(store_path)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         if running_as_root:
             os.chown(store_path, uid, gid)