testing: delay import modules to facilitate wider use (bug 1363509); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Tue, 09 May 2017 16:13:09 -0700
changeset 11320 32903c78a1249f00c16fb4ab36d778bccbe49fab
parent 11319 cd82c4b973e81f5685f654b3b14621fd294c756a
child 11321 9e6b71ec1df0864a4ad875605f3ce9d0422a487c
push id1719
push userbmo:gps@mozilla.com
push dateSat, 24 Jun 2017 00:30:23 +0000
reviewersglob
bugs1363509
testing: delay import modules to facilitate wider use (bug 1363509); r?glob We now have smaller, leaner development environments. Some environments/virtualenvs don't have all the bells and whistles like support for Docker, Kafka, SSH, etc. In an upcoming commit, I'll be teaching the new hgdev environment to have *optional* support for Docker. For this, I'll need to import vcttesting.docker. However, this module imports util.py which imports tons of random modules used for various service tests. These modules are related to functionality unused by the hgdev environment. Since we are striving for minimalism in our new environments, we don't want to take on the Python package dependencies. This commit moves importing of uncommon modules to the functions that use them so as to not require extra dependencies to import util.py. MozReview-Commit-ID: DMDpVCXkK0B
testing/vcttesting/bugzilla/__init__.py
testing/vcttesting/bugzilla/mach_commands.py
testing/vcttesting/util.py
--- a/testing/vcttesting/bugzilla/__init__.py
+++ b/testing/vcttesting/bugzilla/__init__.py
@@ -1,16 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import base64
 
 import bugsy
-import mechanize
 import xmlrpclib
 import yaml
 
 from mozautomation.bugzilla_transports import bugzilla_transport
 
 
 def create_xmlrpc_proxy(url, username, password):
     """Obtain an XMLRPC proxy to the Bugzilla service."""
@@ -124,16 +123,19 @@ class Bugzilla(object):
 
     def update_user_login_denied_text(self, email, text):
         return self.proxy.User.update({
             'names': [email],
             'login_denied_text': text,
         })
 
     def create_login_cookie(self):
+        # Delay import to facilitate module use in limited virtualenvs.
+        import mechanize
+
         # We simulate a browser's HTML interaction with Bugzilla to obtain a
         # login cookie. Is there a better way?
         br = mechanize.Browser()
         url = self.base_url + '/'
         br.set_handle_robots(False)
         br.open(url)
         br.select_form(nr=0)
         br.form['Bugzilla_login'] = self.username
--- a/testing/vcttesting/bugzilla/mach_commands.py
+++ b/testing/vcttesting/bugzilla/mach_commands.py
@@ -6,29 +6,32 @@ import os
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 from vcttesting.bugzilla import Bugzilla
-from vcttesting.mozreview import MozReview
+
 
 @CommandProvider
 class BugzillaCommands(object):
     def __init__(self, context):
         if 'BUGZILLA_URL' in os.environ:
             self.base_url = os.environ['BUGZILLA_URL']
             username = os.environ['BUGZILLA_USERNAME']
             password = os.environ['BUGZILLA_PASSWORD']
 
             self.b = Bugzilla(self.base_url, username=username, password=password)
 
         elif 'MOZREVIEW_HOME' in os.environ:
+            # Delay import to facilitate module use in limited virtualenvs.
+            from vcttesting.mozreview import MozReview
+
             mr = MozReview(os.environ['MOZREVIEW_HOME'])
             username = os.environ.get('BUGZILLA_USERNAME')
             password = os.environ.get('BUGZILLA_PASSWORD')
 
             self.b = mr.get_bugzilla(username=username, password=password)
 
         else:
             raise Exception('Do not know which Bugzilla instance to talk to. '
--- a/testing/vcttesting/util.py
+++ b/testing/vcttesting/util.py
@@ -4,19 +4,16 @@
 
 from __future__ import absolute_import, unicode_literals
 
 import os
 import socket
 import time
 
 import concurrent.futures as futures
-from kafka.client import KafkaClient
-import kombu
-import paramiko
 import requests
 
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 ROOT = os.path.normpath(os.path.join(HERE, '..', '..'))
 
 
 def get_available_port():
@@ -53,16 +50,18 @@ def wait_for_http(host, port, path='', t
         if time.time() - start > timeout:
             raise Exception('Timeout reached waiting for HTTP')
 
         time.sleep(0.1)
 
 
 def wait_for_amqp(hostname, port, userid, password, ssl=False, timeout=60,
                   extra_check_fn=None):
+    # Delay import to facilitate module use in limited virtualenvs.
+    import kombu
     c = kombu.Connection(hostname=hostname, port=port, userid=userid,
             password=password, ssl=ssl)
 
     start = time.time()
 
     while True:
         try:
             c.connection
@@ -74,23 +73,25 @@ def wait_for_amqp(hostname, port, userid
             extra_check_fn()
 
         if time.time() - start > timeout:
             raise Exception('Timeout reached waiting for AMQP')
 
         time.sleep(0.1)
 
 
-class IgnoreHostKeyPolicy(paramiko.MissingHostKeyPolicy):
-    def missing_host_key(self, client, hostname, key):
-        return
-
-
 def wait_for_ssh(hostname, port, timeout=60, extra_check_fn=None):
     """Wait for an SSH server to start on the specified host and port."""
+    # Delay import to facilitate module use in limited virtualenvs.
+    import paramiko
+
+    class IgnoreHostKeyPolicy(paramiko.MissingHostKeyPolicy):
+        def missing_host_key(self, client, hostname, key):
+            return
+
     start = time.time()
 
     while True:
         client = paramiko.SSHClient()
         client.set_missing_host_key_policy(IgnoreHostKeyPolicy())
         try:
             client.connect(hostname, port=port, timeout=0.1, allow_agent=False,
                            look_for_keys=False)
@@ -110,32 +111,38 @@ def wait_for_ssh(hostname, port, timeout
         if time.time() - start > timeout:
             raise Exception('Timeout reached waiting for SSH')
 
         time.sleep(0.1)
 
 
 def wait_for_kafka(hostport, timeout=60):
     """Wait for Kafka to start responding on the specified host:port string."""
+    # Delay import to facilitate module use in limited virtualenvs.
+    from kafka.client import KafkaClient
+
     start = time.time()
     while True:
         try:
             KafkaClient(hostport, client_id=b'dummy', timeout=1)
             return
         except Exception:
             pass
 
         if time.time() - start > timeout:
             raise Exception('Timeout reached waiting for Kafka')
 
         time.sleep(0.1)
 
 
 def wait_for_kafka_topic(hostport, topic, timeout=60):
     """Wait for a Kafka topic to become available."""
+    # Delay import to facilitate module use in limited virtualenvs.
+    from kafka.client import KafkaClient
+
     start = time.time()
     client = KafkaClient(hostport, client_id=b'dummy', timeout=1)
     while not client.has_metadata_for_topic(topic):
         if time.time() - start > timeout:
             raise Exception('timeout reached waiting for topic')
 
         time.sleep(0.1)
         client.load_metadata_for_topics()