testing: delay import modules to facilitate wider use (
bug 1363509); r?glob
We now have smaller, leaner development environments. Some
environments/virtualenvs don't have all the bells and whistles
like support for Docker, Kafka, SSH, etc.
In an upcoming commit, I'll be teaching the new hgdev environment
to have *optional* support for Docker. For this, I'll need to
import vcttesting.docker. However, this module imports util.py
which imports tons of random modules used for various service
tests. These modules are related to functionality unused by the
hgdev environment. Since we are striving for minimalism in our
new environments, we don't want to take on the Python package
dependencies.
This commit moves importing of uncommon modules to the functions
that use them so as to not require extra dependencies to import
util.py.
MozReview-Commit-ID: DMDpVCXkK0B
--- a/testing/vcttesting/bugzilla/__init__.py
+++ b/testing/vcttesting/bugzilla/__init__.py
@@ -1,16 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import base64
import bugsy
-import mechanize
import xmlrpclib
import yaml
from mozautomation.bugzilla_transports import bugzilla_transport
def create_xmlrpc_proxy(url, username, password):
"""Obtain an XMLRPC proxy to the Bugzilla service."""
@@ -124,16 +123,19 @@ class Bugzilla(object):
def update_user_login_denied_text(self, email, text):
return self.proxy.User.update({
'names': [email],
'login_denied_text': text,
})
def create_login_cookie(self):
+ # Delay import to facilitate module use in limited virtualenvs.
+ import mechanize
+
# We simulate a browser's HTML interaction with Bugzilla to obtain a
# login cookie. Is there a better way?
br = mechanize.Browser()
url = self.base_url + '/'
br.set_handle_robots(False)
br.open(url)
br.select_form(nr=0)
br.form['Bugzilla_login'] = self.username
--- a/testing/vcttesting/bugzilla/mach_commands.py
+++ b/testing/vcttesting/bugzilla/mach_commands.py
@@ -6,29 +6,32 @@ import os
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from vcttesting.bugzilla import Bugzilla
-from vcttesting.mozreview import MozReview
+
@CommandProvider
class BugzillaCommands(object):
def __init__(self, context):
if 'BUGZILLA_URL' in os.environ:
self.base_url = os.environ['BUGZILLA_URL']
username = os.environ['BUGZILLA_USERNAME']
password = os.environ['BUGZILLA_PASSWORD']
self.b = Bugzilla(self.base_url, username=username, password=password)
elif 'MOZREVIEW_HOME' in os.environ:
+ # Delay import to facilitate module use in limited virtualenvs.
+ from vcttesting.mozreview import MozReview
+
mr = MozReview(os.environ['MOZREVIEW_HOME'])
username = os.environ.get('BUGZILLA_USERNAME')
password = os.environ.get('BUGZILLA_PASSWORD')
self.b = mr.get_bugzilla(username=username, password=password)
else:
raise Exception('Do not know which Bugzilla instance to talk to. '
--- a/testing/vcttesting/util.py
+++ b/testing/vcttesting/util.py
@@ -4,19 +4,16 @@
from __future__ import absolute_import, unicode_literals
import os
import socket
import time
import concurrent.futures as futures
-from kafka.client import KafkaClient
-import kombu
-import paramiko
import requests
HERE = os.path.abspath(os.path.dirname(__file__))
ROOT = os.path.normpath(os.path.join(HERE, '..', '..'))
def get_available_port():
@@ -53,16 +50,18 @@ def wait_for_http(host, port, path='', t
if time.time() - start > timeout:
raise Exception('Timeout reached waiting for HTTP')
time.sleep(0.1)
def wait_for_amqp(hostname, port, userid, password, ssl=False, timeout=60,
extra_check_fn=None):
+ # Delay import to facilitate module use in limited virtualenvs.
+ import kombu
c = kombu.Connection(hostname=hostname, port=port, userid=userid,
password=password, ssl=ssl)
start = time.time()
while True:
try:
c.connection
@@ -74,23 +73,25 @@ def wait_for_amqp(hostname, port, userid
extra_check_fn()
if time.time() - start > timeout:
raise Exception('Timeout reached waiting for AMQP')
time.sleep(0.1)
-class IgnoreHostKeyPolicy(paramiko.MissingHostKeyPolicy):
- def missing_host_key(self, client, hostname, key):
- return
-
-
def wait_for_ssh(hostname, port, timeout=60, extra_check_fn=None):
"""Wait for an SSH server to start on the specified host and port."""
+ # Delay import to facilitate module use in limited virtualenvs.
+ import paramiko
+
+ class IgnoreHostKeyPolicy(paramiko.MissingHostKeyPolicy):
+ def missing_host_key(self, client, hostname, key):
+ return
+
start = time.time()
while True:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(IgnoreHostKeyPolicy())
try:
client.connect(hostname, port=port, timeout=0.1, allow_agent=False,
look_for_keys=False)
@@ -110,32 +111,38 @@ def wait_for_ssh(hostname, port, timeout
if time.time() - start > timeout:
raise Exception('Timeout reached waiting for SSH')
time.sleep(0.1)
def wait_for_kafka(hostport, timeout=60):
"""Wait for Kafka to start responding on the specified host:port string."""
+ # Delay import to facilitate module use in limited virtualenvs.
+ from kafka.client import KafkaClient
+
start = time.time()
while True:
try:
KafkaClient(hostport, client_id=b'dummy', timeout=1)
return
except Exception:
pass
if time.time() - start > timeout:
raise Exception('Timeout reached waiting for Kafka')
time.sleep(0.1)
def wait_for_kafka_topic(hostport, topic, timeout=60):
"""Wait for a Kafka topic to become available."""
+ # Delay import to facilitate module use in limited virtualenvs.
+ from kafka.client import KafkaClient
+
start = time.time()
client = KafkaClient(hostport, client_id=b'dummy', timeout=1)
while not client.has_metadata_for_topic(topic):
if time.time() - start > timeout:
raise Exception('timeout reached waiting for topic')
time.sleep(0.1)
client.load_metadata_for_topics()