Bug 1384241 - Review comment: Move gevent-websocket code to separate module. r=gps draft
authorNick Alexander <nalexander@mozilla.com>
Wed, 26 Jul 2017 15:13:31 -0700
changeset 616494 3a5d971f34417aa2827c5cbc06b82fe470bdd8a8
parent 616493 38741a3dc0bd6430b33306c322d011f64ccd6ca2
child 616495 9c44680ab95a60ed60d202c1a2f5c6c4128ba837
push id70704
push usernalexander@mozilla.com
push dateThu, 27 Jul 2017 03:35:28 +0000
reviewersgps
bugs1384241
milestone56.0a1
Bug 1384241 - Review comment: Move gevent-websocket code to separate module. r=gps MozReview-Commit-ID: 8P3u21UHzzO
python/mozbuild/mozbuild/faster_daemon_server.py
python/mozbuild/mozbuild/mach_commands.py
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/faster_daemon_server.py
@@ -0,0 +1,112 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Run a WebSocket server that accepts connections, uses the |mach build
+faster| daemon to watch source directories, performs partial |mach
+build faster| builds, and broadcasts build output changes to connected
+WebSocket clients.
+'''
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import bottle
+import gevent
+import gevent.monkey
+import gevent.queue
+import json
+
+import mozbuild.faster_daemon as faster_daemon
+
+
+def message_to_json_string(sequence, message):
+    return json.dumps({
+        'sequence': sequence,
+        'type': 'change',
+        'change': {
+            'unrecognized': list(sorted(message.unrecognized)),
+            'inputs': list(sorted(message.input_to_outputs.keys())),
+            'outputs': list(sorted(message.output_to_inputs.keys())),
+        },
+    })
+
+
+def start_server_greenlet(daemon, port=8080, verbose=False):
+    '''
+    Start a WebSocket server greenlet listening on the given `port`,
+    watch changes from the given `daemon`, and broadcast observed
+    output changes as JSON to connected WebSocket clients.
+
+    Warning: sadly, using gevent and pywatchman requires gevent to
+    monkey patch the environment, which might interact with other
+    modules and code.
+
+    Returns a gevent greenlet.
+    '''
+
+    # Sadly, pywatchman requires gevent to monkey patch the environment.
+    gevent.monkey.patch_all()
+
+    # Each connected WebSocket has it's own gevent message queue.
+    message_queues = []
+
+    def producer():
+        sequence = 0
+
+        faster_daemon.print_line('watch', 'waiting for changes')
+
+        for change in daemon.output_changes():
+            if verbose:
+                faster_daemon.print_line('watch', 'sending sequence number {} to {} queues'
+                                         .format(sequence, len(message_queues)))
+            for message_queue in message_queues:
+                message_queue.put_nowait((sequence, change))
+            sequence += 1
+            gevent.sleep(0)
+
+    producer_greenlet = gevent.spawn(producer)
+
+    app = bottle.Bottle()
+
+    @app.route('/websocket')
+    def handle_websocket():
+        wsock = bottle.request.environ.get('wsgi.websocket')
+        if not wsock:
+            bottle.abort(400, 'Expected WebSocket request.')
+
+        message_queue = gevent.queue.Queue()
+        message_queues.append(message_queue)
+
+        if verbose:
+            faster_daemon.print_line('watch', 'adding queue, there are now {} queues'
+                                     .format(len(message_queues)))
+
+        try:
+            while True:
+                try:
+                    (sequence, message) = message_queue.get()
+                    wsock.send(message_to_json_string(sequence, message))
+                except WebSocketError:
+                    break
+        finally:
+            message_queues.remove(message_queue)
+            if verbose:
+                faster_daemon.print_line('watch', 'removed queue, there are now {} queues'
+                                         .format(len(message_queues)))
+
+    from gevent.pywsgi import WSGIServer
+    from geventwebsocket import WebSocketError
+    from geventwebsocket.handler import WebSocketHandler
+    server = WSGIServer(('0.0.0.0', port), app,
+                        handler_class=WebSocketHandler)
+    _server_greenlet = server.start()
+
+    if verbose:
+        faster_daemon.print_line('watch', 'listening on localhost:{}'.format(port))
+
+    # TODO: figure out how to package up the producer and the server
+    # greenlets into one greenlet, to kill the one greenlet when
+    # either of the underlying greenlets dies, and to kill both the
+    # underlying greenlets when that one greenlet is killed.
+    return producer_greenlet
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -308,119 +308,33 @@ class StoreDebugParamsAndWarnAction(argp
 
 @CommandProvider
 class Watch(MachCommandBase):
     """Interface to watch and re-build the tree."""
 
     @Command('watch', category='build', description='Watch and re-build the tree.')
     @CommandArgument('-v', '--verbose', action='store_true',
                      help='Verbose output for what commands the watcher is running.')
-    def watch(self, verbose=False):
+    @CommandArgument('-p', '--port',
+                     default=8080,
+                     help='Port to listen for WebSocket connections on.')
+    def watch(self, port=8080, verbose=False):
         """Watch and re-build the source tree."""
         self._activate_virtualenv()
         self.virtualenv_manager.install_pip_package('pywatchman==1.3.0')
         self.virtualenv_manager.install_pip_package('bottle==0.12.13')
         self.virtualenv_manager.install_pip_package('gevent-websocket==0.10.1')
 
-        import gevent
-        import gevent.monkey
-        import gevent.queue
-        import json
         import mozbuild.faster_daemon as faster_daemon
-        import pywatchman
-
-        # Sadly, pywatchman requires gevent monkey patching the environment.
-        gevent.monkey.patch_all()
+        import mozbuild.faster_daemon_server as server
 
         daemon = faster_daemon.Daemon(self.config_environment)
 
-        # Each connection has it's own gevent message queue.
-        message_queues = []
-
-        def producer():
-            sequence = 0
-
-            print('waiting for changes')
-            try:
-                for change in daemon.output_changes():
-                    if verbose:
-                        faster_daemon.print_line('watch','sending sequence number {} to {} queues'
-                                                 .format(sequence, len(message_queues)))
-                    for message_queue in message_queues:
-                        message_queue.put_nowait((sequence, change))
-                    sequence += 1
-                    gevent.sleep(0)
-
-            except pywatchman.CommandError as ex:
-                print('watchman:', ex.msg, file=sys.stderr)
-                sys.exit(1)
-
-            except pywatchman.SocketTimeout as ex:
-                print('watchman:', str(ex), file=sys.stderr)
-                sys.exit(2)
-
-            except KeyboardInterrupt:
-                # Suppress ugly stack trace when user hits Ctrl-C.
-                sys.exit(3)
-
-            return 0
-
-        producer_greenlet = gevent.spawn(producer)
-
-        from bottle import request, Bottle, abort
-        app = Bottle()
-
-        @app.route('/websocket')
-        def handle_websocket():
-            wsock = request.environ.get('wsgi.websocket')
-            if not wsock:
-                abort(400, 'Expected WebSocket request.')
-
-            message_queue = gevent.queue.Queue()
-            message_queues.append(message_queue)
-
-            if verbose:
-                faster_daemon.print_line('watch', 'adding queue, there are now {} queues'
-                                         .format(len(message_queues)))
-
-            try:
-                while True:
-                    try:
-                        (sequence, message) = message_queue.get()
-
-                        s = json.dumps(
-                            {
-                                'sequence': sequence,
-                                'type': 'change',
-                                'change': {
-                                    'unrecognized': list(sorted(message.unrecognized)),
-                                    'inputs': list(sorted(message.input_to_outputs.keys())),
-                                    'outputs': list(sorted(message.output_to_inputs.keys())),
-                                },
-                            })
-                        wsock.send(s)
-                    except WebSocketError:
-                        break
-            finally:
-                message_queues.remove(message_queue)
-                if verbose:
-                    faster_daemon.print_line('watch', 'removed queue, there are now {} queues'
-                                             .format(len(message_queues)))
-
-        from gevent.pywsgi import WSGIServer
-        from geventwebsocket import WebSocketError
-        from geventwebsocket.handler import WebSocketHandler
-        server = WSGIServer(('0.0.0.0', 8080), app,
-                            handler_class=WebSocketHandler)
-        server_greenlet = server.start()
-
-        if verbose:
-            faster_daemon.print_line('watch', 'listening on localhost:8080')
-
-        producer_greenlet.get()
+        greenlet = server.start_server_greenlet(daemon, port=port, verbose=verbose)
+        greenlet.get()
 
         return 0
 
 
 @CommandProvider
 class Build(MachCommandBase):
     """Interface to build the tree."""