Bug 1384241 - Part 2: Push changes to WebSocket listeners. r=gps
Pushing changes via a WebSocket service allows front-end code (i.e.,
the Firefox devtools) to react to file system input changes and build
output changes without themselves polling the filesystem. This could
grow in a number of directions, but it's intended right now for this
narrow use case.
MozReview-Commit-ID: 7lXNp5pq4IK
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/faster_daemon_server.py
@@ -0,0 +1,112 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Run a WebSocket server that accepts connections, uses the |mach build
+faster| daemon to watch source directories, performs partial |mach
+build faster| builds, and broadcasts build output changes to connected
+WebSocket clients.
+'''
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import bottle
+import gevent
+import gevent.monkey
+import gevent.queue
+import json
+
+import mozbuild.faster_daemon as faster_daemon
+
+
+def message_to_json_string(sequence, message):
+ return json.dumps({
+ 'sequence': sequence,
+ 'type': 'change',
+ 'change': {
+ 'unrecognized': list(sorted(message.unrecognized)),
+ 'inputs': list(sorted(message.input_to_outputs.keys())),
+ 'outputs': list(sorted(message.output_to_inputs.keys())),
+ },
+ })
+
+
+def start_server_greenlet(daemon, port=8080, verbose=False):
+ '''
+ Start a WebSocket server greenlet listening on the given `port`,
+ watch changes from the given `daemon`, and broadcast observed
+ output changes as JSON to connected WebSocket clients.
+
+ Warning: sadly, using gevent and pywatchman requires gevent to
+ monkey patch the environment, which might interact with other
+ modules and code.
+
+ Returns a gevent greenlet.
+ '''
+
+ # Sadly, pywatchman requires gevent to monkey patch the environment.
+ gevent.monkey.patch_all()
+
+ # Each connected WebSocket has it's own gevent message queue.
+ message_queues = []
+
+ def producer():
+ sequence = 0
+
+ faster_daemon.print_line('watch', 'waiting for changes')
+
+ for change in daemon.output_changes():
+ if verbose:
+ faster_daemon.print_line('watch', 'sending sequence number {} to {} queues'
+ .format(sequence, len(message_queues)))
+ for message_queue in message_queues:
+ message_queue.put_nowait((sequence, change))
+ sequence += 1
+ gevent.sleep(0)
+
+ producer_greenlet = gevent.spawn(producer)
+
+ app = bottle.Bottle()
+
+ @app.route('/websocket')
+ def handle_websocket():
+ wsock = bottle.request.environ.get('wsgi.websocket')
+ if not wsock:
+ bottle.abort(400, 'Expected WebSocket request.')
+
+ message_queue = gevent.queue.Queue()
+ message_queues.append(message_queue)
+
+ if verbose:
+ faster_daemon.print_line('watch', 'adding queue, there are now {} queues'
+ .format(len(message_queues)))
+
+ try:
+ while True:
+ try:
+ (sequence, message) = message_queue.get()
+ wsock.send(message_to_json_string(sequence, message))
+ except WebSocketError:
+ break
+ finally:
+ message_queues.remove(message_queue)
+ if verbose:
+ faster_daemon.print_line('watch', 'removed queue, there are now {} queues'
+ .format(len(message_queues)))
+
+ from gevent.pywsgi import WSGIServer
+ from geventwebsocket import WebSocketError
+ from geventwebsocket.handler import WebSocketHandler
+ server = WSGIServer(('0.0.0.0', port), app,
+ handler_class=WebSocketHandler)
+ _server_greenlet = server.start()
+
+ if verbose:
+ faster_daemon.print_line('watch', 'listening on localhost:{}'.format(port))
+
+ # TODO: figure out how to package up the producer and the server
+ # greenlets into one greenlet, to kill the one greenlet when
+ # either of the underlying greenlets dies, and to kill both the
+ # underlying greenlets when that one greenlet is killed.
+ return producer_greenlet
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -306,29 +306,43 @@ class StoreDebugParamsAndWarnAction(argp
setattr(namespace, self.dest, values)
@CommandProvider
class Watch(MachCommandBase):
"""Interface to watch and re-build the tree."""
@Command('watch', category='build', description='Watch and re-build the tree.')
- def watch(self):
+ @CommandArgument('-v', '--verbose', action='store_true',
+ help='Verbose output for what commands the watcher is running.')
+ @CommandArgument('-p', '--port',
+ default=8080,
+ help='Port to listen for WebSocket connections on.')
+ def watch(self, port=8080, verbose=False):
"""Watch and re-build the source tree."""
self._activate_virtualenv()
try:
self.virtualenv_manager.install_pip_package('pywatchman==1.3.0')
except:
print('Could not install pywatchman from pip. See '
'https://developer.mozilla.org/en-US/docs/TODO_DOCUMENTATION_URL')
return 1
- from mozbuild.faster_daemon import Daemon
- daemon = Daemon(self.config_environment)
- return daemon.watch()
+ self.virtualenv_manager.install_pip_package('bottle==0.12.13')
+ self.virtualenv_manager.install_pip_package('gevent-websocket==0.10.1')
+
+ import mozbuild.faster_daemon as faster_daemon
+ import mozbuild.faster_daemon_server as server
+
+ daemon = faster_daemon.Daemon(self.config_environment)
+
+ greenlet = server.start_server_greenlet(daemon, port=port, verbose=verbose)
+ greenlet.get()
+
+ return 0
@CommandProvider
class Build(MachCommandBase):
"""Interface to build the tree."""
@Command('build', category='build', description='Build the tree.')
@CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,