Bug 1356101 - Derive the mach_bootstrap search path from build/virtualenv_packages.txt. r?gps draft
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 14 Apr 2017 08:19:08 +0900
changeset 563015 b6a215807af313cdafbf0595db2ac4f8663908b9
parent 562868 1a1069b27f40edbfbcf1aa81a5e7dfb39845a5fe
child 624376 529b943683be022b457dd0d026961714c153d6cd
push id54189
push userbmo:mh+mozilla@glandium.org
push dateFri, 14 Apr 2017 21:39:40 +0000
reviewersgps
bugs1356101, 893976, 1244736
milestone55.0a1
Bug 1356101 - Derive the mach_bootstrap search path from build/virtualenv_packages.txt. r?gps Most entries in virtualenv_packages.txt that are .pth or packages.txt are currently in SEARCH_PATHS in mach_bootstrap. The ones that are missing would make sense in SEARCH_PATHS. None of non-.pth or packages.txt entries, however, are in SEARCH_PATHS and don't make sense there. On the other hand, virtualenv_packages.txt misses a lot of things that are in SEARCH_PATHS, all of which should be there. One exception: xpcom/idl-parser, which causes problems due to the xpidl package containing an xpidl module, which causes problems with the in-tree scripts using it. Plus, it needs a cache directory, which is messy, so it's preferable to keep it away from the virtualenv. It turns out it was added to mach_bootstrap.py in bug 893976 for a command that was since then removed (bug 1244736), so we can get away with removing it. So instead of keeping those two separate lists out of sync, we replace the SEARCH_PATHS list from mach_bootstrap with one that is derived at runtime from the contents of virtualenv_packages.txt. And since a .pth can't fail to install in the virtualenv, it makes no sense to have psutil.pth defined as optional, which allows it to end up in the mach_bootstrap search path automatically. Finally, because we do have overlapping module names in the tree (e.g. runtests), and mach_bootstrap's SEARCH_PATHS had a guaranteed order, we change the order of the virtualenv_packages.txt file to match what used to be in mach_bootstrap, and make all the pth entries use the same file name so that the order is more guaranteed in the virtualenv too.
build/mach_bootstrap.py
build/virtualenv_packages.txt
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -27,84 +27,16 @@ If you would like to use a different dir
 MOZBUILD_STATE_PATH environment variable to the directory you would like to
 use and re-run mach. For this change to take effect forever, you'll likely
 want to export this environment variable from your shell's init scripts.
 
 Press ENTER/RETURN to continue or CTRL+c to abort.
 '''.lstrip()
 
 
-# TODO Bug 794506 Integrate with the in-tree virtualenv configuration.
-SEARCH_PATHS = [
-    'python/mach',
-    'python/mozboot',
-    'python/mozbuild',
-    'python/mozlint',
-    'python/mozversioncontrol',
-    'python/blessings',
-    'python/compare-locales',
-    'python/configobj',
-    'python/dlmanager',
-    'python/futures',
-    'python/jsmin',
-    'python/psutil',
-    'python/pylru',
-    'python/which',
-    'python/pystache',
-    'python/pyyaml/lib',
-    'python/requests',
-    'python/slugid',
-    'python/py',
-    'python/pytest',
-    'python/pytoml',
-    'python/redo',
-    'python/voluptuous',
-    'build',
-    'build/pymake',
-    'config',
-    'dom/bindings',
-    'dom/bindings/parser',
-    'dom/media/test/external',
-    'layout/tools/reftest',
-    'other-licenses/ply',
-    'taskcluster',
-    'testing',
-    'testing/firefox-ui/harness',
-    'testing/marionette/client',
-    'testing/marionette/harness',
-    'testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py',
-    'testing/marionette/puppeteer/firefox',
-    'testing/mozbase/mozcrash',
-    'testing/mozbase/mozdebug',
-    'testing/mozbase/mozdevice',
-    'testing/mozbase/mozfile',
-    'testing/mozbase/mozhttpd',
-    'testing/mozbase/mozinfo',
-    'testing/mozbase/mozinstall',
-    'testing/mozbase/mozleak',
-    'testing/mozbase/mozlog',
-    'testing/mozbase/moznetwork',
-    'testing/mozbase/mozprocess',
-    'testing/mozbase/mozprofile',
-    'testing/mozbase/mozrunner',
-    'testing/mozbase/mozsystemmonitor',
-    'testing/mozbase/mozscreenshot',
-    'testing/mozbase/moztest',
-    'testing/mozbase/mozversion',
-    'testing/mozbase/manifestparser',
-    'testing/taskcluster',
-    'testing/tools/autotry',
-    'testing/web-platform',
-    'testing/web-platform/harness',
-    'testing/web-platform/tests/tools/wptserve',
-    'testing/web-platform/tests/tools/six',
-    'testing/xpcshell',
-    'xpcom/idl-parser',
-]
-
 # Individual files providing mach commands.
 MACH_MODULES = [
     'addon-sdk/mach_commands.py',
     'build/valgrind/mach_commands.py',
     'devtools/shared/css/generated/mach_commands.py',
     'dom/bindings/mach_commands.py',
     'dom/media/test/external/mach_commands.py',
     'layout/tools/reftest/mach_commands.py',
@@ -179,16 +111,31 @@ CATEGORIES = {
     }
 }
 
 
 # We submit data to telemetry approximately every this many mach invocations
 TELEMETRY_SUBMISSION_FREQUENCY = 10
 
 
+def search_path(mozilla_dir, packages_txt):
+    with open(os.path.join(mozilla_dir, packages_txt)) as f:
+        packages = [line.rstrip().split(':') for line in f]
+
+    for package in packages:
+        if package[0] == 'packages.txt':
+            assert len(package) == 2
+            for p in search_path(mozilla_dir, package[1]):
+                yield os.path.join(mozilla_dir, p)
+
+        if package[0].endswith('.pth'):
+            assert len(package) == 2
+            yield os.path.join(mozilla_dir, package[1])
+
+
 def bootstrap(topsrcdir, mozilla_dir=None):
     if mozilla_dir is None:
         mozilla_dir = topsrcdir
 
     # Ensure we are running Python 2.7+. We put this check here so we generate a
     # user-friendly error message rather than a cryptic stack trace on module
     # import.
     if sys.version_info[0] != 2 or sys.version_info[1] < 7:
@@ -199,17 +146,19 @@ def bootstrap(topsrcdir, mozilla_dir=Non
     # Global build system and mach state is stored in a central directory. By
     # default, this is ~/.mozbuild. However, it can be defined via an
     # environment variable. We detect first run (by lack of this directory
     # existing) and notify the user that it will be created. The logic for
     # creation is much simpler for the "advanced" environment variable use
     # case. For default behavior, we educate users and give them an opportunity
     # to react. We always exit after creating the directory because users don't
     # like surprises.
-    sys.path[0:0] = [os.path.join(mozilla_dir, path) for path in SEARCH_PATHS]
+    sys.path[0:0] = [os.path.join(mozilla_dir, path)
+                     for path in search_path(mozilla_dir,
+                                             'build/virtualenv_packages.txt')]
     import mach.main
     from mozboot.util import get_state_dir
 
     from mozbuild.util import patch_main
     patch_main()
 
     def telemetry_handler(context, data):
         # We have not opted-in to telemetry
--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -1,43 +1,56 @@
-marionette_driver.pth:testing/marionette/client
-marionette_harness.pth:testing/marionette/harness
-browsermobproxy.pth:testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
-six.pth:testing/web-platform/tests/tools/six
-wptserve.pth:testing/web-platform/tests/tools/wptserve
-blessings.pth:python/blessings
-configobj.pth:python/configobj
-jsmin.pth:python/jsmin
-mach.pth:python/mach
-mozbuild.pth:python/mozbuild
-mozversioncontrol.pth:python/mozversioncontrol
-mozlint.pth:python/mozlint
-pymake.pth:build/pymake
+mozilla.pth:python/mach
+mozilla.pth:python/mozboot
+mozilla.pth:python/mozbuild
+mozilla.pth:python/mozlint
+mozilla.pth:python/mozversioncontrol
+mozilla.pth:python/blessings
+mozilla.pth:python/compare-locales
+mozilla.pth:python/configobj
+mozilla.pth:python/dlmanager
+mozilla.pth:python/futures
+mozilla.pth:python/jsmin
 optional:setup.py:python/psutil:build_ext:--inplace
-optional:psutil.pth:python/psutil
-which.pth:python/which
-ply.pth:other-licenses/ply/
-mock.pth:python/mock-1.0.0
-py.pth:python/py
-pytest.pth:python/pytest
+mozilla.pth:python/psutil
+mozilla.pth:python/pylru
+mozilla.pth:python/which
+mozilla.pth:python/pystache
+mozilla.pth:python/pyyaml/lib
+mozilla.pth:python/requests
+mozilla.pth:python/slugid
+mozilla.pth:python/py
+mozilla.pth:python/pytest
+mozilla.pth:python/pytoml
+mozilla.pth:python/redo
+mozilla.pth:python/voluptuous
 mozilla.pth:build
+objdir:build
+mozilla.pth:build/pymake
 mozilla.pth:config
-mozilla.pth:xpcom/typelib/xpt/tools
 mozilla.pth:dom/bindings
 mozilla.pth:dom/bindings/parser
+mozilla.pth:dom/media/test/external
 mozilla.pth:layout/tools/reftest
-moztreedocs.pth:tools/docs
+mozilla.pth:other-licenses/ply/
+mozilla.pth:taskcluster
+mozilla.pth:testing
+mozilla.pth:testing/firefox-ui/harness
+mozilla.pth:testing/marionette/client
+mozilla.pth:testing/marionette/harness
+mozilla.pth:testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
+mozilla.pth:testing/marionette/puppeteer/firefox
 packages.txt:testing/mozbase/packages.txt
-objdir:build
-gyp.pth:media/webrtc/trunk/tools/gyp/pylib
-pyasn1.pth:python/pyasn1
-pyasn1_modules.pth:python/pyasn1-modules
-redo.pth:python/redo
-requests.pth:python/requests
-rsa.pth:python/rsa
-futures.pth:python/futures
-ecc.pth:python/PyECC
-xpcshell.pth:testing/xpcshell
-pyyaml.pth:python/pyyaml/lib
-pytoml.pth:python/pytoml
-pylru.pth:python/pylru
-taskcluster.pth:taskcluster
-dlmanager.pth:python/dlmanager
+mozilla.pth:testing/taskcluster
+mozilla.pth:testing/tools/autotry
+mozilla.pth:testing/web-platform
+mozilla.pth:testing/web-platform/harness
+mozilla.pth:testing/web-platform/tests/tools/wptserve
+mozilla.pth:testing/web-platform/tests/tools/six
+mozilla.pth:testing/xpcshell
+mozilla.pth:python/mock-1.0.0
+mozilla.pth:xpcom/typelib/xpt/tools
+mozilla.pth:tools/docs
+mozilla.pth:media/webrtc/trunk/tools/gyp/pylib
+mozilla.pth:python/pyasn1
+mozilla.pth:python/pyasn1-modules
+mozilla.pth:python/rsa
+mozilla.pth:python/PyECC