Bug 1255585 - Prevent Python executable mis-match from constantly clobbering the virtualenv on OS X. r=gps
Virtualenv will sometimes find a different executable from its sys.executable on OS X,
causing a check in the build system comparing filesizes between sys.executable and virtualenv
python to fail, resulting in clobbering and re-building the virtualenv every time the virtualenv
is activated, causing the build backend and more to be re-built.
Instead of checking file sizes directly, this commit causes us to record the size and version of the
Python executable that created the virtualenv. If the Python executable checked is not the virtualenv
Python, or we have a different version than was used to create the virtualenv, then the virtualenv is
considered to be out of date.
MozReview-Commit-ID: KmrVfQCtbS3
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -45,16 +45,23 @@ class VirtualenvManager(object):
Each manager is associated with a source directory, a path where you
want the virtualenv to be created, and a handle to write output to.
"""
assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
self.topsrcdir = topsrcdir
self.topobjdir = topobjdir
self.virtualenv_root = virtualenv_path
+
+ # Record the Python executable that was used to create the Virtualenv
+ # so we can check this against sys.executable when verifying the
+ # integrity of the virtualenv.
+ self.exe_info_path = os.path.join(self.virtualenv_root,
+ 'python_exe.txt')
+
self.log_handle = log_handle
self.manifest_path = manifest_path
@property
def virtualenv_script_path(self):
"""Path to virtualenv's own populator script."""
return os.path.join(self.topsrcdir, 'python', 'virtualenv',
'virtualenv.py')
@@ -77,16 +84,35 @@ class VirtualenvManager(object):
binary += '.exe'
return os.path.join(self.bin_path, binary)
@property
def activate_path(self):
return os.path.join(self.bin_path, 'activate_this.py')
+ def get_exe_info(self):
+ """Returns the version and file size of the python executable that was in
+ use when this virutalenv was created.
+ """
+ with open(self.exe_info_path, 'r') as fh:
+ version, size = fh.read().splitlines()
+ return int(version), int(size)
+
+ def write_exe_info(self, python):
+ """Records the the version of the python executable that was in use when
+ this virutalenv was created. We record this explicitly because
+ on OS X our python path may end up being a different or modified
+ executable.
+ """
+ ver = subprocess.check_output([python, '-c', 'import sys; print(sys.hexversion)']).rstrip()
+ with open(self.exe_info_path, 'w') as fh:
+ fh.write("%s\n" % ver)
+ fh.write("%s\n" % os.path.getsize(python))
+
def up_to_date(self, python=sys.executable):
"""Returns whether the virtualenv is present and up to date."""
deps = [self.manifest_path, __file__]
# check if virtualenv exists
if not os.path.exists(self.virtualenv_root) or \
not os.path.exists(self.activate_path):
@@ -94,20 +120,23 @@ class VirtualenvManager(object):
return False
# check modification times
activate_mtime = os.path.getmtime(self.activate_path)
dep_mtime = max(os.path.getmtime(p) for p in deps)
if dep_mtime > activate_mtime:
return False
- # check interpreter. Assume that a different size is enough to
- # know whether we've been given a different interpreter, and thus
- # should refresh the virtualenv.
- if os.path.getsize(python) != os.path.getsize(self.python_path):
+ # Verify that the Python we're checking here is either the virutalenv
+ # python, or we have the Python version that was used to create the
+ # virtualenv. If this fails, it is likely system Python has been
+ # upgraded, and our virtualenv would not be usable.
+ python_size = os.path.getsize(python)
+ if ((python, python_size) != (self.python_path, os.path.getsize(self.python_path)) and
+ (sys.hexversion, python_size) != self.get_exe_info()):
return False
# recursively check sub packages.txt files
submanifests = [i[1] for i in self.packages()
if i[0] == 'packages.txt']
for submanifest in submanifests:
submanifest = os.path.join(self.topsrcdir, submanifest)
submanager = VirtualenvManager(self.topsrcdir,
@@ -148,16 +177,18 @@ class VirtualenvManager(object):
result = subprocess.call(args, stdout=self.log_handle,
stderr=subprocess.STDOUT, env=env)
if result:
raise Exception(
'Failed to create virtualenv: %s' % self.virtualenv_root)
+ self.write_exe_info(python)
+
return self.virtualenv_root
def packages(self):
with file(self.manifest_path, 'rU') as fh:
packages = [line.rstrip().split(':')
for line in fh]
return packages