Bug 1384241 - Part 2: Push changes to WebSocket listeners. r=gps draft
authorNick Alexander <nalexander@mozilla.com>
Wed, 26 Jul 2017 15:13:31 -0700
changeset 616893 c9726990a7529717f4d6b7bab0f35112eb2693e9
parent 616892 20fcd2c092fa4bf7c92192c37aa526701d977a08
child 639629 1b2116b8e729dc5d67c72c5d06630078865baecb
push id70849
push usernalexander@mozilla.com
push dateThu, 27 Jul 2017 16:40:09 +0000
reviewersgps
bugs1384241
milestone56.0a1
Bug 1384241 - Part 2: Push changes to WebSocket listeners. r=gps Pushing changes via a WebSocket service allows front-end code (i.e., the Firefox devtools) to react to file system input changes and build output changes without themselves polling the filesystem. This could grow in a number of directions, but it's intended right now for this narrow use case. MozReview-Commit-ID: 7lXNp5pq4IK
python/mozbuild/mozbuild/faster_daemon_server.py
python/mozbuild/mozbuild/mach_commands.py
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/faster_daemon_server.py
@@ -0,0 +1,112 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Run a WebSocket server that accepts connections, uses the |mach build
+faster| daemon to watch source directories, performs partial |mach
+build faster| builds, and broadcasts build output changes to connected
+WebSocket clients.
+'''
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import bottle
+import gevent
+import gevent.monkey
+import gevent.queue
+import json
+
+import mozbuild.faster_daemon as faster_daemon
+
+
+def message_to_json_string(sequence, message):
+    return json.dumps({
+        'sequence': sequence,
+        'type': 'change',
+        'change': {
+            'unrecognized': list(sorted(message.unrecognized)),
+            'inputs': list(sorted(message.input_to_outputs.keys())),
+            'outputs': list(sorted(message.output_to_inputs.keys())),
+        },
+    })
+
+
+def start_server_greenlet(daemon, port=8080, verbose=False):
+    '''
+    Start a WebSocket server greenlet listening on the given `port`,
+    watch changes from the given `daemon`, and broadcast observed
+    output changes as JSON to connected WebSocket clients.
+
+    Warning: sadly, using gevent and pywatchman requires gevent to
+    monkey patch the environment, which might interact with other
+    modules and code.
+
+    Returns a gevent greenlet.
+    '''
+
+    # Sadly, pywatchman requires gevent to monkey patch the environment.
+    gevent.monkey.patch_all()
+
+    # Each connected WebSocket has it's own gevent message queue.
+    message_queues = []
+
+    def producer():
+        sequence = 0
+
+        faster_daemon.print_line('watch', 'waiting for changes')
+
+        for change in daemon.output_changes():
+            if verbose:
+                faster_daemon.print_line('watch', 'sending sequence number {} to {} queues'
+                                         .format(sequence, len(message_queues)))
+            for message_queue in message_queues:
+                message_queue.put_nowait((sequence, change))
+            sequence += 1
+            gevent.sleep(0)
+
+    producer_greenlet = gevent.spawn(producer)
+
+    app = bottle.Bottle()
+
+    @app.route('/websocket')
+    def handle_websocket():
+        wsock = bottle.request.environ.get('wsgi.websocket')
+        if not wsock:
+            bottle.abort(400, 'Expected WebSocket request.')
+
+        message_queue = gevent.queue.Queue()
+        message_queues.append(message_queue)
+
+        if verbose:
+            faster_daemon.print_line('watch', 'adding queue, there are now {} queues'
+                                     .format(len(message_queues)))
+
+        try:
+            while True:
+                try:
+                    (sequence, message) = message_queue.get()
+                    wsock.send(message_to_json_string(sequence, message))
+                except WebSocketError:
+                    break
+        finally:
+            message_queues.remove(message_queue)
+            if verbose:
+                faster_daemon.print_line('watch', 'removed queue, there are now {} queues'
+                                         .format(len(message_queues)))
+
+    from gevent.pywsgi import WSGIServer
+    from geventwebsocket import WebSocketError
+    from geventwebsocket.handler import WebSocketHandler
+    server = WSGIServer(('0.0.0.0', port), app,
+                        handler_class=WebSocketHandler)
+    _server_greenlet = server.start()
+
+    if verbose:
+        faster_daemon.print_line('watch', 'listening on localhost:{}'.format(port))
+
+    # TODO: figure out how to package up the producer and the server
+    # greenlets into one greenlet, to kill the one greenlet when
+    # either of the underlying greenlets dies, and to kill both the
+    # underlying greenlets when that one greenlet is killed.
+    return producer_greenlet
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -306,29 +306,43 @@ class StoreDebugParamsAndWarnAction(argp
         setattr(namespace, self.dest, values)
 
 
 @CommandProvider
 class Watch(MachCommandBase):
     """Interface to watch and re-build the tree."""
 
     @Command('watch', category='build', description='Watch and re-build the tree.')
-    def watch(self):
+    @CommandArgument('-v', '--verbose', action='store_true',
+                     help='Verbose output for what commands the watcher is running.')
+    @CommandArgument('-p', '--port',
+                     default=8080,
+                     help='Port to listen for WebSocket connections on.')
+    def watch(self, port=8080, verbose=False):
         """Watch and re-build the source tree."""
         self._activate_virtualenv()
         try:
             self.virtualenv_manager.install_pip_package('pywatchman==1.3.0')
         except:
             print('Could not install pywatchman from pip.  See '
                   'https://developer.mozilla.org/en-US/docs/TODO_DOCUMENTATION_URL')
             return 1
 
-        from mozbuild.faster_daemon import Daemon
-        daemon = Daemon(self.config_environment)
-        return daemon.watch()
+        self.virtualenv_manager.install_pip_package('bottle==0.12.13')
+        self.virtualenv_manager.install_pip_package('gevent-websocket==0.10.1')
+
+        import mozbuild.faster_daemon as faster_daemon
+        import mozbuild.faster_daemon_server as server
+
+        daemon = faster_daemon.Daemon(self.config_environment)
+
+        greenlet = server.start_server_greenlet(daemon, port=port, verbose=verbose)
+        greenlet.get()
+
+        return 0
 
 
 @CommandProvider
 class Build(MachCommandBase):
     """Interface to build the tree."""
 
     @Command('build', category='build', description='Build the tree.')
     @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,