new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/faster_daemon.py
@@ -0,0 +1,297 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Use pywatchman to watch source directories and perform partial |mach
+build faster| builds.
+'''
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import datetime
+import os
+import sys
+import time
+
+import mozpack.path as mozpath
+from mozpack.manifests import (
+ InstallManifest,
+)
+from mozpack.copier import (
+ FileCopier,
+ FileRegistrySubtree,
+)
+from mozpack.files import (
+ FileFinder,
+)
+
+# Watchman integration cribbed entirely from
+# https://github.com/facebook/watchman/blob/19aebfebb0b5b0b5174b3914a879370ffc5dac37/python/bin/watchman-wait
+import pywatchman
+
+
+def print_line(prefix, m, now=None):
+ now = now or datetime.datetime.utcnow()
+ print(b'[%s %sZ] %s' % (prefix, now.isoformat(), m))
+
+
+def print_copy_result(elapsed, destdir, result, verbose=True):
+ COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
+ 'Added/updated {updated}; ' \
+ 'Removed {rm_files} files and {rm_dirs} directories.'
+
+ print_line('watch', COMPLETE.format(
+ elapsed=elapsed,
+ dest=destdir,
+ existing=result.existing_files_count,
+ updated=result.updated_files_count,
+ rm_files=result.removed_files_count,
+ rm_dirs=result.removed_directories_count))
+
+
+class FasterBuildChange(object):
+ def __init__(self):
+ self.unrecognized = set()
+ self.input_to_outputs = {}
+ self.output_to_inputs = {}
+
+
+class Daemon(object):
+ def __init__(self, config_environment):
+ self.config_environment = config_environment
+ self._file_copier = None
+ self._client = None
+ self._subscriptions = []
+
+ @property
+ def defines(self):
+ defines = dict((name, self.config_environment.defines[name])
+ for name in self.config_environment.defines
+ if name not in self.config_environment.non_global_defines)
+ defines.update({
+ 'AB_CD': 'en-US',
+ 'BUILD_FASTER': 1,
+ })
+ defines.update({
+ 'BOOKMARKS_INCLUDE_DIR': mozpath.join(self.config_environment.topsrcdir,
+ 'browser', 'locales', 'en-US', 'profile'),
+ })
+ return defines
+
+ @property
+ def file_copier(self):
+ if self._file_copier is None:
+ file_copier = FileCopier()
+
+ finder = FileFinder(mozpath.join(self.config_environment.topobjdir, 'faster'))
+ for path, f in finder.find('*.track'):
+ manifest = InstallManifest(fileobj=f.open())
+
+ # Turn 'install_dist_bin_browser.track' into ['browser'].
+ parts = os.path.basename(os.path.splitext(path)[0]).split('_')
+ parts = parts[3:]
+
+ subtree = mozpath.join(*parts) if parts else None
+ manifest.populate_registry(FileRegistrySubtree(subtree, file_copier),
+ defines_override=self.defines)
+
+ self._file_copier = file_copier
+
+ return self._file_copier
+
+ def watch_set(self, depth=2):
+ file_copier = self.file_copier
+
+ # We want the set of watches to be "reasonable", so we watch
+ # (by default) $TOPSRCDIR/two/levels or $TOPSRCDIR/onelevel.
+ # Many folks root $TOPOBJDIR in $TOPSRCDIR, so we handle that
+ # situation specially. We definitely don't want to watch
+ # $TOPOBJDIR by mistake!
+ watches = set()
+ for input in file_copier.input_to_outputs_tree().keys():
+ rel = mozpath.commonprefix((input, self.config_environment.topobjdir))
+ if rel.startswith(self.config_environment.topobjdir):
+ continue
+
+ rel = mozpath.relpath(input, self.config_environment.topsrcdir)
+ if rel:
+ partial_paths = file_copier._partial_paths(rel)
+ if len(partial_paths) >= 2:
+ watch = partial_paths[-2]
+ elif len(partial_paths) >= 1:
+ watch = partial_paths[-1]
+ watches.add(watch)
+ else:
+ raise Exception("Refusing to watch input ({}) not rooted in topsrcdir ({})"
+ .format(input, self.config_environment.topsrcdir))
+
+ return watches
+
+ def subscribe_to_dir(self, name, dir_to_watch):
+ query = {
+ 'expression': ['true'],
+ 'fields': ['name'],
+ }
+ watch = self.client.query('watch-project', dir_to_watch)
+ if 'warning' in watch:
+ print('WARNING: ', watch['warning'], file=sys.stderr)
+
+ root = watch['watch']
+ if 'relative_path' in watch:
+ query['relative_root'] = watch['relative_path']
+
+ # Get the initial clock value so that we only get updates.
+ query['since'] = self.client.query('clock', root)['clock']
+
+ return self.client.query('subscribe', root, name, query)
+
+ def changed_files(self):
+ # in theory we can parse just the result variable here, but
+ # the client object will accumulate all subscription results
+ # over time, so we ask it to remove and return those values
+ # for each of the subscriptions
+
+ files = set()
+ for sub in self._subscriptions:
+ name = sub['subscribe']
+ data = self.client.getSubscription(name)
+ if data is None:
+ continue
+
+ for dat in data:
+ files |= set([mozpath.join(self.config_environment.topsrcdir, name, f)
+ for f in dat.get('files', [])])
+
+ files = set([f for f in files if os.path.isfile(f)])
+
+ return files
+
+ def incremental_copy(self, copier, force=False, verbose=True):
+ # Just like the 'repackage' target in browser/app/Makefile.in.
+ if 'cocoa' == self.config_environment.substs['MOZ_WIDGET_TOOLKIT']:
+ bundledir = mozpath.join(self.config_environment.topobjdir, 'dist',
+ self.config_environment.substs['MOZ_MACBUNDLE_NAME'],
+ 'Contents', 'Resources')
+ start = time.time()
+ result = copier.copy(bundledir,
+ skip_if_older=not force,
+ remove_unaccounted=False,
+ remove_all_directory_symlinks=False,
+ remove_empty_directories=False)
+ print_copy_result(time.time() - start, bundledir, result, verbose=verbose)
+
+ destdir = mozpath.join(self.config_environment.topobjdir, 'dist', 'bin')
+ start = time.time()
+ result = copier.copy(destdir,
+ skip_if_older=not force,
+ remove_unaccounted=False,
+ remove_all_directory_symlinks=False,
+ remove_empty_directories=False)
+ print_copy_result(time.time() - start, destdir, result, verbose=verbose)
+
+
+ def input_changes(self):
+ '''
+ Return an iterator of `FasterBuildChange` instances as inputs
+ to the faster build system change.
+ '''
+
+ self.client = pywatchman.client()
+ # TODO: restrict these capabilities to the minimal set.
+ self.client.capabilityCheck(required=['term-dirname', 'cmd-watch-project', 'wildmatch'])
+
+ for watch in sorted(self.watch_set()):
+ name = watch
+ path = mozpath.join(self.config_environment.topsrcdir, watch)
+ sub = self.subscribe_to_dir(name, path)
+ self._subscriptions.append(sub)
+
+ input_to_outputs = self.file_copier.input_to_outputs_tree()
+ for input, outputs in input_to_outputs.items():
+ if not outputs:
+ raise Exception("Refusing to watch input ({}) with no outputs".format(input))
+
+ while True:
+ try:
+ _watch_result = self.client.receive()
+
+ changed = self.changed_files()
+ if not changed:
+ continue
+
+ result = FasterBuildChange()
+
+ for change in changed:
+ change = mozpath.normpath(change)
+ if change in input_to_outputs:
+ result.input_to_outputs[change] = set(input_to_outputs[change])
+ else:
+ result.unrecognized.add(change)
+
+ for input, outputs in result.input_to_outputs.items():
+ for output in outputs:
+ if output not in result.output_to_inputs:
+ result.output_to_inputs[output] = set()
+ result.output_to_inputs[output].add(input)
+
+ yield result
+
+ except pywatchman.SocketTimeout:
+ # Let's check to see if we're still functional.
+ _version = self.client.query('version')
+
+ def output_changes(self, verbose=True):
+ '''
+ Return an iterator of `FasterBuildChange` instances as outputs
+ from the faster build system are updated.
+ '''
+ # TODO: consider how to really provide the debug diagnostics
+ # we want, since this print is not interleaved with the
+ # corresponding watch.
+ if verbose:
+ for watch in sorted(self.watch_set()):
+ path = mozpath.join(self.config_environment.topsrcdir, watch)
+ print_line('watch', 'Watching srcdir {}'.format(path))
+
+ for change in self.input_changes():
+ now = datetime.datetime.utcnow()
+
+ for unrecognized in sorted(change.unrecognized):
+ print_line('watch', '! {}'.format(unrecognized), now=now)
+
+ all_outputs = set()
+ for input in sorted(change.input_to_outputs):
+ outputs = change.input_to_outputs[input]
+
+ print_line('watch', '< {}'.format(input), now=now)
+ for output in sorted(outputs):
+ print_line('watch', '> {}'.format(output), now=now)
+ all_outputs |= outputs
+
+ if all_outputs:
+ partial_copier = FileCopier()
+ for output in all_outputs:
+ partial_copier.add(output, self.file_copier[output])
+
+ self.incremental_copy(partial_copier, force=True, verbose=verbose)
+ yield change
+
+ def watch(self, verbose=True):
+ try:
+ for change in self.output_changes(verbose=verbose):
+ pass
+
+ except pywatchman.CommandError as ex:
+ print('watchman:', ex.msg, file=sys.stderr)
+ sys.exit(1)
+
+ except pywatchman.SocketTimeout as ex:
+ print('watchman:', str(ex), file=sys.stderr)
+ sys.exit(2)
+
+ except KeyboardInterrupt:
+ # Suppress ugly stack trace when user hits Ctrl-C.
+ sys.exit(3)
+
+ return 0--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -302,16 +302,31 @@ class BuildOutputManager(OutputManager):
class StoreDebugParamsAndWarnAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
sys.stderr.write('The --debugparams argument is deprecated. Please ' +
'use --debugger-args instead.\n\n')
setattr(namespace, self.dest, values)
@CommandProvider
+class Watch(MachCommandBase):
+ """Interface to watch and re-build the tree."""
+
+ @Command('watch', category='build', description='Watch and re-build the tree.')
+ def watch(self):
+ """Watch and re-build the source tree."""
+ self._activate_virtualenv()
+ self.virtualenv_manager.install_pip_package('pywatchman==1.3.0')
+
+ from mozbuild.faster_daemon import Daemon
+ daemon = Daemon(self.config_environment)
+ return daemon.watch()
+
+
+@CommandProvider
class Build(MachCommandBase):
"""Interface to build the tree."""
@Command('build', category='build', description='Build the tree.')
@CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,
help='Number of concurrent jobs to run. Default is the number of CPUs.')
@CommandArgument('-C', '--directory', default=None,
help='Change to a subdirectory of the build directory first.')