Bug 1255585 - Prevent Python executable mis-match from constantly clobbering the virtualenv on OS X. r=gps draft
authorChris Manchester <cmanchester@mozilla.com>
Fri, 11 Mar 2016 12:24:10 -0800
changeset 339690 d2b87325e10da6dfcd74f8b0d2ef7c0efb71595b
parent 339688 b8efc6dc729ea6b1b5de4e6aca866ee52d7dccf9
child 516052 4aa12d0c3d850e2dcd7daea401182c401e9c49c0
push id12801
push usercmanchester@mozilla.com
push dateSat, 12 Mar 2016 21:35:32 +0000
reviewersgps
bugs1255585
milestone48.0a1
Bug 1255585 - Prevent Python executable mis-match from constantly clobbering the virtualenv on OS X. r=gps Virtualenv will sometimes find a different executable from its sys.executable on OS X, causing a check in the build system comparing filesizes between sys.executable and virtualenv python to fail, resulting in clobbering and re-building the virtualenv every time the virtualenv is activated, causing the build backend and more to be re-built. Instead of checking file sizes directly, this commit causes us to record the size and version of the Python executable that created the virtualenv. If the Python executable checked is not the virtualenv Python, or we have a different version than was used to create the virtualenv, then the virtualenv is considered to be out of date. MozReview-Commit-ID: KmrVfQCtbS3
python/mozbuild/mozbuild/virtualenv.py
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -45,16 +45,23 @@ class VirtualenvManager(object):
 
         Each manager is associated with a source directory, a path where you
         want the virtualenv to be created, and a handle to write output to.
         """
         assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
         self.topsrcdir = topsrcdir
         self.topobjdir = topobjdir
         self.virtualenv_root = virtualenv_path
+
+        # Record the Python executable that was used to create the Virtualenv
+        # so we can check this against sys.executable when verifying the
+        # integrity of the virtualenv.
+        self.exe_info_path = os.path.join(self.virtualenv_root,
+                                          'python_exe.txt')
+
         self.log_handle = log_handle
         self.manifest_path = manifest_path
 
     @property
     def virtualenv_script_path(self):
         """Path to virtualenv's own populator script."""
         return os.path.join(self.topsrcdir, 'python', 'virtualenv',
             'virtualenv.py')
@@ -77,16 +84,35 @@ class VirtualenvManager(object):
             binary += '.exe'
 
         return os.path.join(self.bin_path, binary)
 
     @property
     def activate_path(self):
         return os.path.join(self.bin_path, 'activate_this.py')
 
+    def get_exe_info(self):
+        """Returns the version and file size of the python executable that was in
+        use when this virutalenv was created.
+        """
+        with open(self.exe_info_path, 'r') as fh:
+            version, size = fh.read().splitlines()
+        return int(version), int(size)
+
+    def write_exe_info(self, python):
+        """Records the the version of the python executable that was in use when
+        this virutalenv was created. We record this explicitly because
+        on OS X our python path may end up being a different or modified
+        executable.
+        """
+        ver = subprocess.check_output([python, '-c', 'import sys; print(sys.hexversion)']).rstrip()
+        with open(self.exe_info_path, 'w') as fh:
+            fh.write("%s\n" % ver)
+            fh.write("%s\n" % os.path.getsize(python))
+
     def up_to_date(self, python=sys.executable):
         """Returns whether the virtualenv is present and up to date."""
 
         deps = [self.manifest_path, __file__]
 
         # check if virtualenv exists
         if not os.path.exists(self.virtualenv_root) or \
             not os.path.exists(self.activate_path):
@@ -94,20 +120,23 @@ class VirtualenvManager(object):
             return False
 
         # check modification times
         activate_mtime = os.path.getmtime(self.activate_path)
         dep_mtime = max(os.path.getmtime(p) for p in deps)
         if dep_mtime > activate_mtime:
             return False
 
-        # check interpreter. Assume that a different size is enough to
-        # know whether we've been given a different interpreter, and thus
-        # should refresh the virtualenv.
-        if os.path.getsize(python) != os.path.getsize(self.python_path):
+        # Verify that the Python we're checking here is either the virutalenv
+        # python, or we have the Python version that was used to create the
+        # virtualenv. If this fails, it is likely system Python has been
+        # upgraded, and our virtualenv would not be usable.
+        python_size = os.path.getsize(python)
+        if ((python, python_size) != (self.python_path, os.path.getsize(self.python_path)) and
+            (sys.hexversion, python_size) != self.get_exe_info()):
             return False
 
         # recursively check sub packages.txt files
         submanifests = [i[1] for i in self.packages()
                         if i[0] == 'packages.txt']
         for submanifest in submanifests:
             submanifest = os.path.join(self.topsrcdir, submanifest)
             submanager = VirtualenvManager(self.topsrcdir,
@@ -148,16 +177,18 @@ class VirtualenvManager(object):
 
         result = subprocess.call(args, stdout=self.log_handle,
             stderr=subprocess.STDOUT, env=env)
 
         if result:
             raise Exception(
                 'Failed to create virtualenv: %s' % self.virtualenv_root)
 
+        self.write_exe_info(python)
+
         return self.virtualenv_root
 
     def packages(self):
         with file(self.manifest_path, 'rU') as fh:
             packages = [line.rstrip().split(':')
                         for line in fh]
         return packages