Bug 1245684 - Download bootstrap files via a zip archive; r?mshal draft
authorGregory Szorc <gps@mozilla.com>
Mon, 08 Feb 2016 17:07:10 -0800
changeset 329737 b098d506fbfaab2b4a7946af48fd49700b1845dc
parent 329566 815d689a6e1e7187b10238f2f840d49201d67c2b
child 514015 b2511c87a9592edcf73de4656bbf25b1dcda6f0d
push id10586
push usergszorc@mozilla.com
push dateTue, 09 Feb 2016 01:07:29 +0000
reviewersmshal
bugs1245684
milestone47.0a1
Bug 1245684 - Download bootstrap files via a zip archive; r?mshal This will perform a single HTTP request that completes in <1s. Contrast with before when we performed multiple HTTP requests and the process took several seconds. MozReview-Commit-ID: DjX3LBdSOIk
python/mozboot/bin/bootstrap.py
--- a/python/mozboot/bin/bootstrap.py
+++ b/python/mozboot/bin/bootstrap.py
@@ -11,44 +11,31 @@
 # content from the upstream source repository.
 
 # If we add unicode_literals, optparse breaks on Python 2.6.1 (which is needed
 # to support OS X 10.6).
 from __future__ import print_function
 
 import os
 import shutil
+from StringIO import StringIO
 import sys
 import tempfile
 try:
     from urllib2 import urlopen
 except ImportError:
     from urllib.request import urlopen
+import zipfile
 
 from optparse import OptionParser
 
 # The next two variables define where in the repository the Python files
 # reside. This is used to remotely download file content when it isn't
 # available locally.
-REPOSITORY_PATH_PREFIX = 'python/mozboot'
-
-REPOSITORY_PATHS = [
-    'mozboot/__init__.py',
-    'mozboot/android.py',
-    'mozboot/archlinux.py',
-    'mozboot/base.py',
-    'mozboot/bootstrap.py',
-    'mozboot/centos.py',
-    'mozboot/debian.py',
-    'mozboot/fedora.py',
-    'mozboot/freebsd.py',
-    'mozboot/gentoo.py',
-    'mozboot/openbsd.py',
-    'mozboot/osx.py',
-]
+REPOSITORY_PATH_PREFIX = 'python/mozboot/'
 
 TEMPDIR = None
 
 
 def setup_proxy():
     # Some Linux environments define ALL_PROXY, which is a SOCKS proxy
     # intended for all protocols. Python doesn't currently automatically
     # detect this like it does for http_proxy and https_proxy.
@@ -60,21 +47,32 @@ def setup_proxy():
 
 def fetch_files(repo_url, repo_type):
     setup_proxy()
     repo_url = repo_url.rstrip('/')
 
     files = {}
 
     if repo_type == 'hgweb':
-        for path in REPOSITORY_PATHS:
-            url = repo_url + '/raw-file/default/python/mozboot/' + path
+        url = repo_url + '/archive/default.zip/python/mozboot'
+        req = urlopen(url=url, timeout=30)
+        data = StringIO(req.read())
+        data.seek(0)
+        zip = zipfile.ZipFile(data, 'r')
+        for f in zip.infolist():
+            # The paths are prefixed with the repo and revision name before the
+            # directory name.
+            offset = f.filename.find(REPOSITORY_PATH_PREFIX) + len(REPOSITORY_PATH_PREFIX)
+            name = f.filename[offset:]
 
-            req = urlopen(url=url, timeout=30)
-            files[path] = req.read()
+            # We only care about the Python modules.
+            if not name.startswith('mozboot/'):
+                continue
+
+            files[name] = zip.read(f)
     else:
         raise NotImplementedError('Not sure how to handle repo type.', repo_type)
 
     return files
 
 
 def ensure_environment(repo_url=None, repo_type=None):
     """Ensure we can load the Python modules necessary to perform bootstrap."""