Bug 1245684 - Download bootstrap files via a zip archive; r?mshal
This will perform a single HTTP request that completes in <1s. Contrast
with before when we performed multiple HTTP requests and the process
took several seconds.
MozReview-Commit-ID: DjX3LBdSOIk
--- a/python/mozboot/bin/bootstrap.py
+++ b/python/mozboot/bin/bootstrap.py
@@ -11,44 +11,31 @@
# content from the upstream source repository.
# If we add unicode_literals, optparse breaks on Python 2.6.1 (which is needed
# to support OS X 10.6).
from __future__ import print_function
import os
import shutil
+from StringIO import StringIO
import sys
import tempfile
try:
from urllib2 import urlopen
except ImportError:
from urllib.request import urlopen
+import zipfile
from optparse import OptionParser
# The next two variables define where in the repository the Python files
# reside. This is used to remotely download file content when it isn't
# available locally.
-REPOSITORY_PATH_PREFIX = 'python/mozboot'
-
-REPOSITORY_PATHS = [
- 'mozboot/__init__.py',
- 'mozboot/android.py',
- 'mozboot/archlinux.py',
- 'mozboot/base.py',
- 'mozboot/bootstrap.py',
- 'mozboot/centos.py',
- 'mozboot/debian.py',
- 'mozboot/fedora.py',
- 'mozboot/freebsd.py',
- 'mozboot/gentoo.py',
- 'mozboot/openbsd.py',
- 'mozboot/osx.py',
-]
+REPOSITORY_PATH_PREFIX = 'python/mozboot/'
TEMPDIR = None
def setup_proxy():
# Some Linux environments define ALL_PROXY, which is a SOCKS proxy
# intended for all protocols. Python doesn't currently automatically
# detect this like it does for http_proxy and https_proxy.
@@ -60,21 +47,32 @@ def setup_proxy():
def fetch_files(repo_url, repo_type):
setup_proxy()
repo_url = repo_url.rstrip('/')
files = {}
if repo_type == 'hgweb':
- for path in REPOSITORY_PATHS:
- url = repo_url + '/raw-file/default/python/mozboot/' + path
+ url = repo_url + '/archive/default.zip/python/mozboot'
+ req = urlopen(url=url, timeout=30)
+ data = StringIO(req.read())
+ data.seek(0)
+ zip = zipfile.ZipFile(data, 'r')
+ for f in zip.infolist():
+ # The paths are prefixed with the repo and revision name before the
+ # directory name.
+ offset = f.filename.find(REPOSITORY_PATH_PREFIX) + len(REPOSITORY_PATH_PREFIX)
+ name = f.filename[offset:]
- req = urlopen(url=url, timeout=30)
- files[path] = req.read()
+ # We only care about the Python modules.
+ if not name.startswith('mozboot/'):
+ continue
+
+ files[name] = zip.read(f)
else:
raise NotImplementedError('Not sure how to handle repo type.', repo_type)
return files
def ensure_environment(repo_url=None, repo_type=None):
"""Ensure we can load the Python modules necessary to perform bootstrap."""