testing: support BMO Docker container in hgdev environment (
bug 1363509); r?glob
This commit adds the --docker-bmo flag to `create-environment hgdev`.
The flag will enable Docker support in the environment by installing
Python packages required by Docker. It will also trigger generation
of the bmoweb Docker image.
The pip requirements file also has a number of extra packages. These
are required by various tests that get unskipped as part of adding
Docker support to the virtualenv.
$ ./run-tests -j8
no docker: 35.9s wall; Ran 112 tests, 66 skipped, 0 warned, 0 failed
w/ docker: 48.0s wall; Ran 118 tests, 60 skipped, 0 warned, 0 failed
The newly-runable tests are everything with "#require bmodocker", which
is a handful of tests for the bzexport, bzpost, and qimportbz extensions.
MozReview-Commit-ID: B7tON1kwZNM
--- a/create-environment
+++ b/create-environment
@@ -19,25 +19,28 @@ from vcttesting.environment import (
def main():
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(title='environment', dest='environment')
sp = subparsers.add_parser('hgdev',
help='Mercurial hooks and extensions')
+ sp.add_argument('--docker-bmo', action='store_true',
+ help='Enable Docker support for bugzilla.mozilla.org')
+
sp = subparsers.add_parser('vcssync',
help='Version control synchronization')
args = parser.parse_args()
env = args.environment
if env == 'hgdev':
- info = create_hgdev()
+ info = create_hgdev(docker_bmo=args.docker_bmo)
elif env == 'vcssync':
info = create_vcssync()
else:
raise Exception('unhandled environment: %s' % env)
print('%s environment created successfully.' % env)
print('')
print('To activate this environment, source a shell script:')
--- a/docs/hgmods/index.rst
+++ b/docs/hgmods/index.rst
@@ -18,16 +18,21 @@ Creating and Maintaining a Development E
In order to run tests for the extensions and hooks, you'll need to create
and activate an isolated environment.
From the root directory of a version-control-tools checkout::
$ ./create-environment hgdev
+Or to create an environment with support for running Bugzilla tests
+requiring Docker::
+
+ $ ./create-environment hgdev --docker-bmo
+
This will create a Python virtualenv in ``venv/hgdev``. Assuming all
goes well, it will print instructions on how to *activate* that
environment in your local shell and how to run tests.
If the command fails, a likely culprit is missing system package
dependencies.
On Debian/Ubuntu based distros, install required system packages via::
new file mode 100644
--- /dev/null
+++ b/testing/requirements-hgdev-docker.txt
@@ -0,0 +1,42 @@
+-r requirements-hgdev.txt
+
+backports.lzma==0.0.8 \
+ --hash=sha256:200584ad5079d8ca6b1bfe14890c7be58666ab0128d8ca26cfb2669b476085f3
+
+backports.ssl-match-hostname==3.5.0.1 \
+ --hash=sha256:502ad98707319f4a51fa2ca1c677bd659008d27ded9f6380c79e8932e38dcdf2
+
+blessings==1.6 \
+ --hash=sha256:edc5713061f10966048bf6b40d9a514b381e0ba849c64e034c4ef6c1847d3007
+
+docker-py==1.10.6 \
+ --hash=sha256:35b506e95861914fa5ad57a6707e3217b4082843b883be246190f57013948aba
+
+docker-pycreds==0.2.1 \
+ --hash=sha256:58d2688f92de5d6f1a6ac4fe25da461232f0e0a4c1212b93b256b046b2d714a9
+
+futures==3.1.1 \
+ --hash=sha256:c4884a65654a7c45435063e14ae85280eb1f111d94e542396717ba9828c4337f
+
+mach==0.5.1 \
+ --hash=sha256:61bda1a1033c110f781427c479dd151a45fc45e46ce28e37fabf0d3cc2edf91e
+
+mechanize==0.2.5 \
+ --hash=sha256:2e67b20d107b30c00ad814891a095048c35d9d8cb9541801cebe85684cc84766
+
+mozinfo==0.7 \
+ --hash=sha256:d27f75b668614f28df2e61b8aa4beb6f68f3c9936fc9139fae755728f2b4844a
+
+mozprocess==0.22 \
+ --hash=sha256:2d56e015a3594f06130e8c015f7ca87f3f8d83d9c4bf54588f077ce9bad9a08b
+
+PyYAML==3.11 \
+ --hash=sha256:c36c938a872e5ff494938b33b14aaa156cb439ec67548fcab3535bb78b0846e8 \
+ --hash=sha256:19bb3ac350ef878dda84a62d37c7d5c17a137386dde9c2ce7249c7a21d7f6ac9
+
+requests==2.10.0 \
+ --hash=sha256:09bc1b5f3a56cd8c48d433213a8cba51a67d12936568f73b5f1793fcb0c0979e \
+ --hash=sha256:63f1815788157130cee16a933b2ee184038e975f0017306d723ac326b5525b54
+
+websocket-client==0.37.0 \
+ --hash=sha256:678b246d816b94018af5297e72915160e2feb042e0cde1a9397f502ac3a52f41
--- a/testing/vcttesting/environment.py
+++ b/testing/vcttesting/environment.py
@@ -50,16 +50,17 @@ def create_virtualenv(name):
activate = os.path.join(bin_dir, 'activate')
res = {
'path': path,
'bin_dir': bin_dir,
'pip': pip,
'python': python,
'activate': activate,
+ 'activate_this': os.path.join(bin_dir, 'activate_this.py'),
}
env = dict(os.environ)
env['ROOT'] = ROOT
env['VENV'] = path
if not os.path.exists(path):
subprocess.check_call([CREATE_VIRTUALENV, path], env=env)
@@ -67,16 +68,21 @@ def create_virtualenv(name):
# Install a sitecustomize.py that starts code coverage if an environment
# variable is set.
with open(os.path.join(bin_dir, 'sitecustomize.py'), 'wb') as fh:
fh.write(SITECUSTOMIZE)
return res
+def activate_virtualenv(venv):
+ """Activate a virtualenv in the current Python process."""
+ execfile(venv['activate_this'], dict(__file__=venv['activate_this']))
+
+
def process_pip_requirements(venv, requirements):
args = [
venv['pip'], 'install', '--upgrade', '--require-hashes',
'-r', os.path.join(ROOT, requirements),
]
subprocess.check_call(args)
@@ -158,27 +164,63 @@ def install_mercurials(venv, hg):
'purge', '--all'],
cwd=hg_dir, env=hg_env,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print('error installing: %s' % e.output)
raise Exception('could not install Mercurial')
-def create_hgdev():
+def docker_client():
+ """Attempt to obtain a Docker client.
+
+ Returns a client on success. None on failure.
+ """
+ from .docker import (
+ Docker,
+ params_from_env,
+ )
+
+ state_file = os.path.join(ROOT, '.dockerstate')
+ docker_url, tls = params_from_env(os.environ)
+
+ d = Docker(state_file, docker_url, tls=tls)
+
+ return d if d.is_alive() else None
+
+
+def create_hgdev(docker_bmo=False):
"""Create an environment used for hacking on Mercurial extensions."""
venv = create_virtualenv('hgdev')
- process_pip_requirements(venv, 'testing/requirements-hgdev.txt')
+
+ if docker_bmo:
+ reqs = 'testing/requirements-hgdev-docker.txt'
+ else:
+ reqs = 'testing/requirements-hgdev.txt'
+
+ process_pip_requirements(venv, reqs)
install_editable(venv, 'hghooks')
+ install_editable(venv, 'pylib/Bugsy')
install_editable(venv, 'pylib/mozhginfo')
install_editable(venv, 'pylib/mozautomation')
install_editable(venv, 'testing')
install_mercurials(venv, hg=os.path.join(venv['bin_dir'], 'hg'))
+ if docker_bmo:
+ activate_virtualenv(venv)
+ docker = docker_client()
+
+ if docker:
+ docker.build_bmo(verbose=True)
+ else:
+ print('Docker not available; cannot build BMO Docker image')
+
+ print('attempting to builder bugzilla.mozilla.org Docker image')
+
return venv
def create_vcssync():
"""Create an environment used for testing VCSSync."""
venv = create_virtualenv('vcssync')
process_pip_requirements(venv, 'vcssync/test-requirements.txt')
install_editable(venv, 'pylib/mozautomation')