mozreviewpulse: create project skeleton (1319659) r?smacleod
Set up an initial project structure for the new mozreviewpulse service.
Also set up a project dev environment with acceptance tests, unit tests,
and a Makefile for easy dev evironment setup.
MozReview-Commit-ID: I5cpAx3loFp
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/.editorconfig
@@ -0,0 +1,4 @@
+# Override for PyCharm Makefile
+[{Makefile, makefile, GNUmakefile}]
+indent_style = tab
+indent_size = 4
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/Makefile
@@ -0,0 +1,25 @@
+.PHONY = docker mountebank rabbitmq test clean
+
+PYTHON = python2.7
+
+default: docker venv
+
+docker: mountebank rabbitmq
+
+mountebank:
+ docker build -t mountebank docker-mountebank/
+
+rabbitmq:
+ docker pull rabbitmq:3-management
+
+venv:
+ virtualenv --no-site-packages --python=$(PYTHON) venv
+ venv/bin/pip install --upgrade pip
+ venv/bin/pip install -r requirements.txt
+ venv/bin/pip install -e .
+
+test:
+ venv/bin/pytest tests
+
+clean:
+ rm -r venv/
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/docker-mountebank/Dockerfile
@@ -0,0 +1,21 @@
+FROM node:alpine
+
+MAINTAINER Māris Fogels <mars@mozilla.com>
+
+ENV MOUNTEBANK_MAJOR_VERSION 1.6
+ENV MOUNTEBANK_VERSION 1.6.0
+ENV MOUNTEBANK_SHA256 0149ee5c1a7f1f02e0a46e748d16a7a8c7145c459139357e6f73e39f0e716308
+
+# So wget works over SSL
+RUN apk --no-cache add openssl
+
+# Install the mountebank nodejs service. See http://www.mbtest.org/docs/install
+RUN wget -c -O mountebank.tar.gz https://s3.amazonaws.com/mountebank/v${MOUNTEBANK_MAJOR_VERSION}/mountebank-v${MOUNTEBANK_VERSION}-npm.tar.gz \
+ && echo "${MOUNTEBANK_SHA256} mountebank.tar.gz" | sha256sum -c \
+ && tar xf mountebank.tar.gz \
+ && rm mountebank.tar.gz
+
+EXPOSE 2525
+
+ENTRYPOINT ["/mountebank/bin/mb"]
+CMD ["--help"]
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/mozreviewpulse/testing.py
@@ -0,0 +1,130 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Test helpers.
+"""
+import json
+from collections import namedtuple
+
+import requests
+
+
+class MountebankClient:
+
+ def __init__(self, host, port=2525, imposter_port=4000):
+ self.host = host
+ self.port = port
+ self.imposter_port = imposter_port
+
+ @property
+ def imposters_admin_url(self):
+ return self.get_endpoint_with_port(self.port, '/imposters')
+
+ @property
+ def stub_baseurl(self):
+ return self.get_endpoint_with_port(self.imposter_port)
+
+ def get_endpoint(self, path=''):
+ """Construct a URL for the imposter service with optional path."""
+ return self.get_endpoint_with_port(self.imposter_port, path)
+
+ def get_endpoint_with_port(self, port, path=''):
+ """Construct a service endpoint URL with port and optional path."""
+ return 'http://{0}:{1}{2}'.format(self.host, port, path)
+
+ def create_imposter(self, imposter_json):
+ """Take a dict and turn it into a service stub."""
+ response = requests.post(self.imposters_admin_url, json=imposter_json)
+ if response.status_code != 201:
+ raise RuntimeError(
+ "mountebank imposter creation failed: {0} {1}".format(
+ response.status_code, response.content
+ ))
+
+ def create_stub(self, stub_json):
+ """Create a single http stub using the default imposter port."""
+ self.create_imposter({
+ 'port': self.imposter_port,
+ 'protocol': 'http',
+ 'stubs': [stub_json]
+ })
+
+ def reset_imposters(self):
+ """Delete all imposters."""
+ requests.delete(self.imposters_admin_url)
+
+ def get_requests(self):
+ """Return a list of requests made to the imposter."""
+ url = self.imposters_admin_url + '/' + str(self.imposter_port)
+ return requests.get(url).json().get('requests')
+
+
+MBHostInfo = namedtuple('MBHostInfo', 'ip adminport imposterport')
+
+
+def run_mountebank_server(request, docker_client, record_requests=False):
+ """Run a mountebank service container in Docker."""
+
+ # We do not record requests by default because it can cause a long-running
+ # server to consume a lot of memory.
+ # See http://www.mbtest.org/docs/api/mocks
+ if record_requests:
+ command = 'start --mock'
+ else:
+ command = 'start'
+
+ ip = run_container(
+ request,
+ docker_client,
+ 'mountebank',
+ # Sent to the mountebank server
+ command=command,
+ # The port our imposters will communicate over
+ ports=[4000],
+ # So mountebank process logs got to STDOUT and STDERR.
+ tty=True
+ )
+ return MBHostInfo(ip, 2525, 4000)
+
+
+
+def run_container(fixture_request, docker_client, image, cleanup=True, **kwargs):
+ """Run and clean up a docker container."""
+ host_config = docker_client.create_host_config(publish_all_ports=True)
+
+ container = docker_client.create_container(
+ image=image, host_config=host_config, **kwargs)
+ docker_client.start(container=container["Id"])
+ container_info = docker_client.inspect_container(container.get('Id'))
+
+ ip = container_info["NetworkSettings"]["IPAddress"]
+
+ def _cleanup():
+ docker_client.remove_container(
+ container=container["Id"],
+ force=True
+ )
+ if cleanup:
+ fixture_request.addfinalizer(_cleanup)
+
+ return ip
+
+
+def pull_image(docker_client, image_name, tag, **kwargs):
+ """Pull or update a docker image."""
+ response = docker_client.pull(image_name, tag=tag, **kwargs)
+ # Grab the last line of output
+ lastline = response.splitlines().pop()
+ try:
+ result = json.loads(lastline)
+ except ValueError as e:
+ raise Exception("Bad JSON result from docker pull: {0}".format(lastline))
+
+ # Should get a dict of {'status': ...} on success and {'error': ...} on
+ # failure.
+ if 'error' in result:
+ raise Exception("Failed to pull image: {0}".format(lastline))
+
+ return image_name
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/mozreviewpulse/trystarter.py
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import taskcluster
+
+
+class TryStarter:
+ """Reads 'review request posted' events from pulse and starts Try jobs
+ on TaskCluster for the review requests' changset.
+ """
+
+ def __init__(self, queue, taskcluster_url, confirm_connections=True):
+ self.queue = queue
+ self.taskcluster_url = taskcluster_url
+
+ if confirm_connections:
+ self._ensure_taskcluster_connection(taskcluster_url)
+
+ def get(self):
+ """Return a single message from the Pulse service."""
+ return self.queue.get()
+
+ def _ensure_taskcluster_connection(self, taskcluster_url):
+ index = taskcluster.Index(options={'baseUrl': taskcluster_url})
+ # This will raise a subclass of TaskclusterFailure if things go wrong.
+ index.ping()
+
+ def process_messages(self):
+ """Process all Pulse messages and dispatch Try requests for them."""
+ pass
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/requirements.txt
@@ -0,0 +1,5 @@
+docker-py
+kombu
+mock
+pytest
+taskcluster
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/setup.py
@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup, find_packages
+
+setup(
+ name='mozreviewpulse',
+ version='0.1',
+ description='MozReview event listener service',
+ url='https://mozilla-version-control-tools.readthedocs.io/',
+ author='Mozilla',
+ author_email='dev-version-control@lists.mozilla.org',
+ license='MPL 2.0',
+ classifiers=[
+ 'Development Status :: 4 - Beta',
+ 'Intended Audience :: Developers',
+ 'Topic :: Software Development :: Build Tools',
+ 'Programming Language :: Python :: 2.7',
+ ],
+ packages=find_packages(),
+ install_requires=['taskcluster'],
+)
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/tests/conftest.py
@@ -0,0 +1,78 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+
+import kombu as kombu
+import pytest
+import docker as docker_py
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3 import Retry
+
+from mozreviewpulse.testing import MountebankClient, run_container, \
+ run_mountebank_server
+
+
+def wait_for_connection(url, retries=5):
+ """Helper to retry a HTTP request."""
+ s = requests.Session()
+ adapter = HTTPAdapter(max_retries=Retry(total=retries, backoff_factor=0.1))
+ s.mount('http://', adapter)
+ s.get(url)
+
+
+@pytest.fixture(scope='session')
+def docker():
+ client = docker_py.Client(os.environ.get('DOCKER_HOST', None))
+
+ # client readiness check
+ try:
+ client.info()
+ except requests.ConnectionError as e:
+ pytest.skip("Error connecting to docker daemon: {0}".format(e))
+
+ return client
+
+
+@pytest.fixture(scope='session')
+def mountebank_server(request, docker):
+ """Run a mountebank service container in Docker."""
+ mb_host_info = run_mountebank_server(request, docker, record_requests=True)
+ mb_url = 'http://{0}:{1}'.format(mb_host_info.ip, mb_host_info.adminport)
+ wait_for_connection(mb_url)
+ return mb_host_info
+
+
+@pytest.fixture(scope='function')
+def mountebank(request, mountebank_server):
+ client = MountebankClient(
+ mountebank_server.ip,
+ mountebank_server.adminport,
+ mountebank_server.imposterport)
+
+ request.addfinalizer(client.reset_imposters)
+
+ if request.cls:
+ # We are being used as a fixture for a unittest.TestCase.
+ request.cls.mountebank = client
+ request.cls.mountebank_host_info = mountebank_server
+ return client
+
+
+@pytest.fixture(scope='function')
+def pulse_server(request, docker):
+ """Run a Mozilla Pulse service container in Docker."""
+ # Use 'rabbitmq:3-management' so that the management plugin is installed and
+ # enabled. See https://hub.docker.com/_/rabbitmq/
+ ip = run_container(request, docker, 'rabbitmq:3-management')
+ return ip
+
+
+@pytest.fixture
+def pulse_conn(request, pulse_server):
+ conn = kombu.Connection(pulse_server, port=5672)
+ request.addfinalizer(conn.release)
+ # Wait for the service to come up
+ conn.ensure_connection(max_retries=10, interval_start=0.3, interval_step=0.3)
+ return conn
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/tests/test_service.py
@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Service-level tests.
+"""
+
+import json
+
+import pytest
+import taskcluster
+
+from mozreviewpulse.trystarter import TryStarter
+
+
+@pytest.fixture
+def pulse_queue(request, pulse_conn):
+ queue = pulse_conn.SimpleBuffer('somequeue')
+ request.addfinalizer(queue.close)
+ return queue
+
+
+@pytest.fixture
+def fake_taskcluster(mountebank):
+ # Return HTTP 200 for all requests
+ mountebank.create_stub(
+ {'responses': [
+ {'is': {
+ 'statusCode': 200,
+ }}
+ ]}
+ )
+ return mountebank
+
+
+def requests_to_path(mb_client, path):
+ """Yield mountebank requests that match the given path."""
+ for request in mb_client.get_requests():
+ if request['path'] == path:
+ yield request
+
+
+def test_listener_receives_messages_from_queue(pulse_queue):
+ pulse_listener = TryStarter(pulse_queue, None, confirm_connections=False)
+ payload = {'payload': 'hello'}
+ pulse_queue.put(payload)
+ message = pulse_listener.get()
+ assert message.body == json.dumps(payload)
+
+
+def test_test_starter_can_ping_taskcluster(fake_taskcluster):
+ # Arrange
+ # Explicitly say we want to confirm service connections
+ TryStarter(pulse_queue, fake_taskcluster.get_endpoint(), confirm_connections=True)
+
+ # Act
+ requests = fake_taskcluster.get_requests()
+
+ # Assert
+ assert len(requests) == 1
+ r = requests.pop()
+ assert r['path'] == '/ping'
+ assert r['method'] == 'GET'
+
+
+def test_review_request_starts_task_in_taskcluster(pulse_queue, fake_taskcluster):
+ # Consume pulse message that review request posted -> start tasks in taskcluster using HTTP api
+ # Arrange
+ test_starter = TryStarter(pulse_queue, fake_taskcluster.get_endpoint())
+ payload = {'payload': 'hello'}
+ pulse_queue.put(payload)
+
+ taskId = 1
+ tcq = taskcluster.Queue()
+ createTask_path = tcq.makeRoute('createTask', replDict={'taskId': taskId})
+
+ # Act
+ test_starter.process_messages()
+
+ # Assert
+ created_tasks = list(requests_to_path(
+ fake_taskcluster, createTask_path))
+ assert len(created_tasks) == 1
+ task = created_tasks.pop()
+ assert task['method'] == 'POST'
+
+
+def result_from_taskcluster_does_X_and_posts_back_to_reviewboard():
+ # Consume from pulse a taskcluster result -> fetch artifact from S3 -> Post result to RB
+ pass
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/mozreviewpulse/tests/test_trystarter.py
@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Project unit tests.
+"""
+
+import taskcluster
+from mock import Mock
+from pytest import raises
+
+from mozreviewpulse.trystarter import TryStarter
+
+
+def test_uses_custom_taskcluster_url():
+ with raises(taskcluster.exceptions.TaskclusterConnectionError) as excinfo:
+ TryStarter(Mock(), 'my_custom_url')
+ assert 'my_custom_url' in excinfo.value.superExc.message
+
+
+def test_connections_are_checked_on_init():
+ with raises(taskcluster.exceptions.TaskclusterConnectionError) as excinfo:
+ TryStarter(Mock(), 'my_custom_url', confirm_connections=True)
+ assert 'my_custom_url' in excinfo.value.superExc.message
+
+
+def test_connection_checks_on_init_can_be_skipped():
+ # This should not raise an exception
+ TryStarter(Mock(), 'my_custom_url', confirm_connections=False)
\ No newline at end of file