testing: support BMO Docker container in hgdev environment (bug 1363509); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Tue, 09 May 2017 17:08:31 -0700
changeset 11325 17f7a68f7dbb44a00613ee4e763fb0a1e8d073db
parent 11324 ae2612777cd96ac7e7fd6ecc61207fa1d825dc68
push id1719
push userbmo:gps@mozilla.com
push dateSat, 24 Jun 2017 00:30:23 +0000
reviewersglob
bugs1363509
testing: support BMO Docker container in hgdev environment (bug 1363509); r?glob This commit adds the --docker-bmo flag to `create-environment hgdev`. The flag will enable Docker support in the environment by installing Python packages required by Docker. It will also trigger generation of the bmoweb Docker image. The pip requirements file also has a number of extra packages. These are required by various tests that get unskipped as part of adding Docker support to the virtualenv. $ ./run-tests -j8 no docker: 35.9s wall; Ran 112 tests, 66 skipped, 0 warned, 0 failed w/ docker: 48.0s wall; Ran 118 tests, 60 skipped, 0 warned, 0 failed The newly-runable tests are everything with "#require bmodocker", which is a handful of tests for the bzexport, bzpost, and qimportbz extensions. MozReview-Commit-ID: B7tON1kwZNM
create-environment
docs/hgmods/index.rst
testing/requirements-hgdev-docker.txt
testing/vcttesting/environment.py
--- a/create-environment
+++ b/create-environment
@@ -19,25 +19,28 @@ from vcttesting.environment import (
 
 def main():
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers(title='environment', dest='environment')
 
     sp = subparsers.add_parser('hgdev',
                                help='Mercurial hooks and extensions')
 
+    sp.add_argument('--docker-bmo', action='store_true',
+                    help='Enable Docker support for bugzilla.mozilla.org')
+
     sp = subparsers.add_parser('vcssync',
                                help='Version control synchronization')
 
     args = parser.parse_args()
 
     env = args.environment
 
     if env == 'hgdev':
-        info = create_hgdev()
+        info = create_hgdev(docker_bmo=args.docker_bmo)
     elif env == 'vcssync':
         info = create_vcssync()
     else:
         raise Exception('unhandled environment: %s' % env)
 
     print('%s environment created successfully.' % env)
     print('')
     print('To activate this environment, source a shell script:')
--- a/docs/hgmods/index.rst
+++ b/docs/hgmods/index.rst
@@ -18,16 +18,21 @@ Creating and Maintaining a Development E
 
 In order to run tests for the extensions and hooks, you'll need to create
 and activate an isolated environment.
 
 From the root directory of a version-control-tools checkout::
 
    $ ./create-environment hgdev
 
+Or to create an environment with support for running Bugzilla tests
+requiring Docker::
+
+   $ ./create-environment hgdev --docker-bmo
+
 This will create a Python virtualenv in ``venv/hgdev``. Assuming all
 goes well, it will print instructions on how to *activate* that
 environment in your local shell and how to run tests.
 
 If the command fails, a likely culprit is missing system package
 dependencies.
 
 On Debian/Ubuntu based distros, install required system packages via::
new file mode 100644
--- /dev/null
+++ b/testing/requirements-hgdev-docker.txt
@@ -0,0 +1,42 @@
+-r requirements-hgdev.txt
+
+backports.lzma==0.0.8 \
+    --hash=sha256:200584ad5079d8ca6b1bfe14890c7be58666ab0128d8ca26cfb2669b476085f3
+
+backports.ssl-match-hostname==3.5.0.1 \
+    --hash=sha256:502ad98707319f4a51fa2ca1c677bd659008d27ded9f6380c79e8932e38dcdf2
+
+blessings==1.6 \
+    --hash=sha256:edc5713061f10966048bf6b40d9a514b381e0ba849c64e034c4ef6c1847d3007
+
+docker-py==1.10.6 \
+    --hash=sha256:35b506e95861914fa5ad57a6707e3217b4082843b883be246190f57013948aba
+
+docker-pycreds==0.2.1 \
+    --hash=sha256:58d2688f92de5d6f1a6ac4fe25da461232f0e0a4c1212b93b256b046b2d714a9
+
+futures==3.1.1 \
+    --hash=sha256:c4884a65654a7c45435063e14ae85280eb1f111d94e542396717ba9828c4337f
+
+mach==0.5.1 \
+    --hash=sha256:61bda1a1033c110f781427c479dd151a45fc45e46ce28e37fabf0d3cc2edf91e
+
+mechanize==0.2.5 \
+    --hash=sha256:2e67b20d107b30c00ad814891a095048c35d9d8cb9541801cebe85684cc84766
+
+mozinfo==0.7 \
+    --hash=sha256:d27f75b668614f28df2e61b8aa4beb6f68f3c9936fc9139fae755728f2b4844a
+
+mozprocess==0.22 \
+    --hash=sha256:2d56e015a3594f06130e8c015f7ca87f3f8d83d9c4bf54588f077ce9bad9a08b
+
+PyYAML==3.11 \
+    --hash=sha256:c36c938a872e5ff494938b33b14aaa156cb439ec67548fcab3535bb78b0846e8 \
+    --hash=sha256:19bb3ac350ef878dda84a62d37c7d5c17a137386dde9c2ce7249c7a21d7f6ac9
+
+requests==2.10.0 \
+    --hash=sha256:09bc1b5f3a56cd8c48d433213a8cba51a67d12936568f73b5f1793fcb0c0979e \
+    --hash=sha256:63f1815788157130cee16a933b2ee184038e975f0017306d723ac326b5525b54
+
+websocket-client==0.37.0 \
+    --hash=sha256:678b246d816b94018af5297e72915160e2feb042e0cde1a9397f502ac3a52f41
--- a/testing/vcttesting/environment.py
+++ b/testing/vcttesting/environment.py
@@ -50,16 +50,17 @@ def create_virtualenv(name):
         activate = os.path.join(bin_dir, 'activate')
 
     res = {
         'path': path,
         'bin_dir': bin_dir,
         'pip': pip,
         'python': python,
         'activate': activate,
+        'activate_this': os.path.join(bin_dir, 'activate_this.py'),
     }
 
     env = dict(os.environ)
     env['ROOT'] = ROOT
     env['VENV'] = path
 
     if not os.path.exists(path):
         subprocess.check_call([CREATE_VIRTUALENV, path], env=env)
@@ -67,16 +68,21 @@ def create_virtualenv(name):
     # Install a sitecustomize.py that starts code coverage if an environment
     # variable is set.
     with open(os.path.join(bin_dir, 'sitecustomize.py'), 'wb') as fh:
         fh.write(SITECUSTOMIZE)
 
     return res
 
 
+def activate_virtualenv(venv):
+    """Activate a virtualenv in the current Python process."""
+    execfile(venv['activate_this'], dict(__file__=venv['activate_this']))
+
+
 def process_pip_requirements(venv, requirements):
     args = [
         venv['pip'], 'install', '--upgrade', '--require-hashes',
         '-r', os.path.join(ROOT, requirements),
     ]
     subprocess.check_call(args)
 
 
@@ -158,27 +164,63 @@ def install_mercurials(venv, hg):
                                      'purge', '--all'],
                                     cwd=hg_dir, env=hg_env,
                                     stderr=subprocess.STDOUT)
         except subprocess.CalledProcessError as e:
             print('error installing: %s' % e.output)
             raise Exception('could not install Mercurial')
 
 
-def create_hgdev():
+def docker_client():
+    """Attempt to obtain a Docker client.
+
+    Returns a client on success. None on failure.
+    """
+    from .docker import (
+        Docker,
+        params_from_env,
+    )
+
+    state_file = os.path.join(ROOT, '.dockerstate')
+    docker_url, tls = params_from_env(os.environ)
+
+    d = Docker(state_file, docker_url, tls=tls)
+
+    return d if d.is_alive() else None
+
+
+def create_hgdev(docker_bmo=False):
     """Create an environment used for hacking on Mercurial extensions."""
     venv = create_virtualenv('hgdev')
-    process_pip_requirements(venv, 'testing/requirements-hgdev.txt')
+
+    if docker_bmo:
+        reqs = 'testing/requirements-hgdev-docker.txt'
+    else:
+        reqs = 'testing/requirements-hgdev.txt'
+
+    process_pip_requirements(venv, reqs)
     install_editable(venv, 'hghooks')
+    install_editable(venv, 'pylib/Bugsy')
     install_editable(venv, 'pylib/mozhginfo')
     install_editable(venv, 'pylib/mozautomation')
     install_editable(venv, 'testing')
 
     install_mercurials(venv, hg=os.path.join(venv['bin_dir'], 'hg'))
 
+    if docker_bmo:
+        activate_virtualenv(venv)
+        docker = docker_client()
+
+        if docker:
+            docker.build_bmo(verbose=True)
+        else:
+            print('Docker not available; cannot build BMO Docker image')
+
+        print('attempting to builder bugzilla.mozilla.org Docker image')
+
     return venv
 
 
 def create_vcssync():
     """Create an environment used for testing VCSSync."""
     venv = create_virtualenv('vcssync')
     process_pip_requirements(venv, 'vcssync/test-requirements.txt')
     install_editable(venv, 'pylib/mozautomation')