Bug 1384241 - Add |mach watch|: pywatchman based incremental |mach build faster| shell. r=gps
There's a natural follow-on that I haven't time to explore right now:
I want the faster make backend to also write a "unified chrome
manifest" that maps outputs
(browser/chrome/browser/content/browser/ext-utils.js) to chrome:// or
resource:// URLs (chrome://content/browser/ext-utils.js or similar).
MozReview-Commit-ID: LDQmm8KD57I
--- a/python/mozbuild/mozbuild/backend/fastermake.py
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -154,12 +154,28 @@ class FasterMakeBackend(CommonBackend, P
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
+ # For artifact builds only, write a single unified manifest for consumption by |mach watch|.
+ if self.environment.is_artifact_build:
+ unified_manifest = InstallManifest()
+ for base, install_manifest in self._install_manifests.iteritems():
+ # Expect 'dist/bin/**', which includes 'dist/bin' with no trailing slash.
+ assert base.startswith('dist/bin')
+ base = base[len('dist/bin'):]
+ if base and base[0] == '/':
+ base = base[1:]
+ unified_manifest.add_entries_from(install_manifest, base=base)
+
+ with self._write_file(
+ mozpath.join(self.environment.topobjdir, 'faster',
+ 'unified_install_dist_bin')) as fh:
+ unified_manifest.write(fileobj=fh)
+
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -753,16 +753,21 @@ class MachCommandConditions(object):
"""Must have a mercurial source checkout."""
return getattr(cls, 'substs', {}).get('VCS_CHECKOUT_TYPE') == 'hg'
@staticmethod
def is_git(cls):
"""Must have a git source checkout."""
return getattr(cls, 'substs', {}).get('VCS_CHECKOUT_TYPE') == 'git'
+ @staticmethod
+ def is_artifact_build(cls):
+ """Must be an artifact build."""
+ return getattr(cls, 'substs', {}).get('MOZ_ARTIFACT_BUILDS')
+
class PathArgument(object):
"""Parse a filesystem path argument and transform it in various ways."""
def __init__(self, arg, topsrcdir, topobjdir, cwd=None):
self.arg = arg
self.topsrcdir = topsrcdir
self.topobjdir = topobjdir
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/faster_daemon.py
@@ -0,0 +1,280 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Use pywatchman to watch source directories and perform partial |mach
+build faster| builds.
+'''
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import datetime
+import sys
+import time
+
+import mozbuild.util
+import mozpack.path as mozpath
+from mozpack.manifests import (
+ InstallManifest,
+)
+from mozpack.copier import (
+ FileCopier,
+)
+
+# Watchman integration cribbed entirely from
+# https://github.com/facebook/watchman/blob/19aebfebb0b5b0b5174b3914a879370ffc5dac37/python/bin/watchman-wait
+import pywatchman
+
+
+def print_line(prefix, m, now=None):
+ now = now or datetime.datetime.utcnow()
+ print(b'[%s %sZ] %s' % (prefix, now.isoformat(), m))
+
+
+def print_copy_result(elapsed, destdir, result, verbose=True):
+ COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
+ 'Added/updated {updated}; ' \
+ 'Removed {rm_files} files and {rm_dirs} directories.'
+
+ print_line('watch', COMPLETE.format(
+ elapsed=elapsed,
+ dest=destdir,
+ existing=result.existing_files_count,
+ updated=result.updated_files_count,
+ rm_files=result.removed_files_count,
+ rm_dirs=result.removed_directories_count))
+
+
+class FasterBuildException(Exception):
+ def __init__(self, message, cause):
+ Exception.__init__(self, message)
+ self.cause = cause
+
+
+class FasterBuildChange(object):
+ def __init__(self):
+ self.unrecognized = set()
+ self.input_to_outputs = {}
+ self.output_to_inputs = {}
+
+
+class Daemon(object):
+ def __init__(self, config_environment):
+ self.config_environment = config_environment
+ self._client = None
+
+ @property
+ def defines(self):
+ defines = dict(self.config_environment.acdefines)
+ # These additions work around warts in the build system: see
+ # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/config/faster/rules.mk#92-93
+ # and
+ # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/python/mozbuild/mozbuild/backend/tup.py#244-253.
+ defines.update({
+ 'AB_CD': 'en-US',
+ 'BUILD_FASTER': '1',
+ })
+ defines.update({
+ 'BOOKMARKS_INCLUDE_DIR': mozpath.join(self.config_environment.topsrcdir,
+ 'browser', 'locales', 'en-US', 'profile'),
+ })
+ return defines
+
+ @mozbuild.util.memoized_property
+ def file_copier(self):
+ # TODO: invalidate the file copier when the build system
+ # itself changes, i.e., the underlying unified manifest
+ # changes.
+ file_copier = FileCopier()
+
+ unified_manifest = InstallManifest(
+ mozpath.join(self.config_environment.topobjdir,
+ 'faster', 'unified_install_dist_bin'))
+
+ unified_manifest.populate_registry(file_copier, defines_override=self.defines)
+
+ return file_copier
+
+ def subscribe_to_topsrcdir(self):
+ self.subscribe_to_dir('topsrcdir', self.config_environment.topsrcdir)
+
+ def subscribe_to_dir(self, name, dir_to_watch):
+ query = {
+ 'empty_on_fresh_instance': True,
+ 'expression': [
+ 'allof',
+ ['type', 'f'],
+ ['not',
+ ['anyof',
+ ['dirname', '.hg'],
+ ['name', '.hg', 'wholename'],
+ ['dirname', '.git'],
+ ['name', '.git', 'wholename'],
+ ],
+ ],
+ ],
+ 'fields': ['name'],
+ }
+ watch = self.client.query('watch-project', dir_to_watch)
+ if 'warning' in watch:
+ print('WARNING: ', watch['warning'], file=sys.stderr)
+
+ root = watch['watch']
+ if 'relative_path' in watch:
+ query['relative_root'] = watch['relative_path']
+
+ # Get the initial clock value so that we only get updates.
+ # Wait 30s to allow for slow Windows IO. See
+ # https://facebook.github.io/watchman/docs/cmd/clock.html.
+ query['since'] = self.client.query('clock', root, {'sync_timeout': 30000})['clock']
+
+ return self.client.query('subscribe', root, name, query)
+
+ def changed_files(self):
+ # In theory we can parse just the result variable here, but
+ # the client object will accumulate all subscription results
+ # over time, so we ask it to remove and return those values.
+ files = set()
+
+ data = self.client.getSubscription('topsrcdir')
+ if data:
+ for dat in data:
+ files |= set([mozpath.normpath(mozpath.join(self.config_environment.topsrcdir, f))
+ for f in dat.get('files', [])])
+
+ return files
+
+ def incremental_copy(self, copier, force=False, verbose=True):
+ # Just like the 'repackage' target in browser/app/Makefile.in.
+ if 'cocoa' == self.config_environment.substs['MOZ_WIDGET_TOOLKIT']:
+ bundledir = mozpath.join(self.config_environment.topobjdir, 'dist',
+ self.config_environment.substs['MOZ_MACBUNDLE_NAME'],
+ 'Contents', 'Resources')
+ start = time.time()
+ result = copier.copy(bundledir,
+ skip_if_older=not force,
+ remove_unaccounted=False,
+ remove_all_directory_symlinks=False,
+ remove_empty_directories=False)
+ print_copy_result(time.time() - start, bundledir, result, verbose=verbose)
+
+ destdir = mozpath.join(self.config_environment.topobjdir, 'dist', 'bin')
+ start = time.time()
+ result = copier.copy(destdir,
+ skip_if_older=not force,
+ remove_unaccounted=False,
+ remove_all_directory_symlinks=False,
+ remove_empty_directories=False)
+ print_copy_result(time.time() - start, destdir, result, verbose=verbose)
+
+ def input_changes(self, verbose=True):
+ '''
+ Return an iterator of `FasterBuildChange` instances as inputs
+ to the faster build system change.
+ '''
+
+ # TODO: provide the debug diagnostics we want: this print is
+ # not immediately before the watch.
+ if verbose:
+ print_line('watch', 'Connecting to watchman')
+ # TODO: figure out why a large timeout is required for the
+ # client, and a robust strategy for retrying timed out
+ # requests.
+ self.client = pywatchman.client(timeout=5.0)
+
+ try:
+ if verbose:
+ print_line('watch', 'Checking watchman capabilities')
+ # TODO: restrict these capabilities to the minimal set.
+ self.client.capabilityCheck(required=[
+ 'clock-sync-timeout',
+ 'cmd-watch-project',
+ 'term-dirname',
+ 'wildmatch',
+ ])
+
+ if verbose:
+ print_line('watch', 'Subscribing to {}'.format(self.config_environment.topsrcdir))
+ self.subscribe_to_topsrcdir()
+ if verbose:
+ print_line('watch', 'Watching {}'.format(self.config_environment.topsrcdir))
+
+ input_to_outputs = self.file_copier.input_to_outputs_tree()
+ for input, outputs in input_to_outputs.items():
+ if not outputs:
+ raise Exception("Refusing to watch input ({}) with no outputs".format(input))
+
+ while True:
+ try:
+ _watch_result = self.client.receive()
+
+ changed = self.changed_files()
+ if not changed:
+ continue
+
+ result = FasterBuildChange()
+
+ for change in changed:
+ if change in input_to_outputs:
+ result.input_to_outputs[change] = set(input_to_outputs[change])
+ else:
+ result.unrecognized.add(change)
+
+ for input, outputs in result.input_to_outputs.items():
+ for output in outputs:
+ if output not in result.output_to_inputs:
+ result.output_to_inputs[output] = set()
+ result.output_to_inputs[output].add(input)
+
+ yield result
+
+ except pywatchman.SocketTimeout:
+ # Let's check to see if we're still functional.
+ _version = self.client.query('version')
+
+ except pywatchman.CommandError as e:
+ # Abstract away pywatchman errors.
+ raise FasterBuildException(e, 'Command error using pywatchman to watch {}'.format(
+ self.config_environment.topsrcdir))
+
+ except pywatchman.SocketTimeout as e:
+ # Abstract away pywatchman errors.
+ raise FasterBuildException(e, 'Socket timeout using pywatchman to watch {}'.format(
+ self.config_environment.topsrcdir))
+
+ finally:
+ self.client.close()
+
+ def output_changes(self, verbose=True):
+ '''
+ Return an iterator of `FasterBuildChange` instances as outputs
+ from the faster build system are updated.
+ '''
+ for change in self.input_changes(verbose=verbose):
+ now = datetime.datetime.utcnow()
+
+ for unrecognized in sorted(change.unrecognized):
+ print_line('watch', '! {}'.format(unrecognized), now=now)
+
+ all_outputs = set()
+ for input in sorted(change.input_to_outputs):
+ outputs = change.input_to_outputs[input]
+
+ print_line('watch', '< {}'.format(input), now=now)
+ for output in sorted(outputs):
+ print_line('watch', '> {}'.format(output), now=now)
+ all_outputs |= outputs
+
+ if all_outputs:
+ partial_copier = FileCopier()
+ for output in all_outputs:
+ partial_copier.add(output, self.file_copier[output])
+
+ self.incremental_copy(partial_copier, force=True, verbose=verbose)
+ yield change
+
+ def watch(self, verbose=True):
+ for change in self.output_changes(verbose=verbose):
+ pass
+
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -302,16 +302,55 @@ class BuildOutputManager(OutputManager):
class StoreDebugParamsAndWarnAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
sys.stderr.write('The --debugparams argument is deprecated. Please ' +
'use --debugger-args instead.\n\n')
setattr(namespace, self.dest, values)
@CommandProvider
+class Watch(MachCommandBase):
+ """Interface to watch and re-build the tree."""
+
+ @Command('watch', category='post-build', description='Watch and re-build the tree.',
+ conditions=[conditions.is_firefox])
+ @CommandArgument('-v', '--verbose', action='store_true',
+ help='Verbose output for what commands the watcher is running.')
+ def watch(self, verbose=False):
+ """Watch and re-build the source tree."""
+
+ if not conditions.is_artifact_build(self):
+ print('mach watch requires an artifact build. See '
+ 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Simple_Firefox_build')
+ return 1
+
+ if not self.substs.get('WATCHMAN', None):
+ print('mach watch requires watchman to be installed. See '
+ 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching')
+ return 1
+
+ self._activate_virtualenv()
+ try:
+ self.virtualenv_manager.install_pip_package('pywatchman==1.3.0')
+ except Exception:
+ print('Could not install pywatchman from pip. See '
+ 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching')
+ return 1
+
+ from mozbuild.faster_daemon import Daemon
+ daemon = Daemon(self.config_environment)
+
+ try:
+ return daemon.watch()
+ except KeyboardInterrupt:
+ # Suppress ugly stack trace when user hits Ctrl-C.
+ sys.exit(3)
+
+
+@CommandProvider
class Build(MachCommandBase):
"""Interface to build the tree."""
@Command('build', category='build', description='Build the tree.')
@CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,
help='Number of concurrent jobs to run. Default is the number of CPUs.')
@CommandArgument('-C', '--directory', default=None,
help='Change to a subdirectory of the build directory first.')
--- a/python/mozbuild/mozpack/manifests.py
+++ b/python/mozbuild/mozpack/manifests.py
@@ -195,26 +195,17 @@ class InstallManifest(object):
def __neq__(self, other):
return not self.__eq__(other)
def __ior__(self, other):
if not isinstance(other, InstallManifest):
raise ValueError('Can only | with another instance of InstallManifest.')
- # We must copy source files to ourselves so extra dependencies from
- # the preprocessor are taken into account. Ideally, we would track
- # which source file each entry came from. However, this is more
- # complicated and not yet implemented. The current implementation
- # will result in over invalidation, possibly leading to performance
- # loss.
- self._source_files |= other._source_files
-
- for dest in sorted(other._dests):
- self._add_entry(dest, other._dests[dest])
+ self.add_entries_from(other)
return self
def _encode_field_entry(self, data):
"""Converts an object into a format that can be stored in the manifest file.
Complex data types, such as ``dict``, need to be converted into a text
representation before they can be written to a file.
@@ -326,16 +317,36 @@ class InstallManifest(object):
))
def _add_entry(self, dest, entry):
if dest in self._dests:
raise ValueError('Item already in manifest: %s' % dest)
self._dests[dest] = entry
+ def add_entries_from(self, other, base=''):
+ """
+ Copy data from another mozpack.copier.InstallManifest
+ instance, adding an optional base prefix to the destination.
+
+ This allows to merge two manifests into a single manifest, or
+ two take the tagged union of two manifests.
+ """
+ # We must copy source files to ourselves so extra dependencies from
+ # the preprocessor are taken into account. Ideally, we would track
+ # which source file each entry came from. However, this is more
+ # complicated and not yet implemented. The current implementation
+ # will result in over invalidation, possibly leading to performance
+ # loss.
+ self._source_files |= other._source_files
+
+ for dest in sorted(other._dests):
+ new_dest = mozpath.join(base, dest) if base else dest
+ self._add_entry(new_dest, other._dests[dest])
+
def populate_registry(self, registry, defines_override={},
link_policy='symlink'):
"""Populate a mozpack.copier.FileRegistry instance with data from us.
The caller supplied a FileRegistry instance (or at least something that
conforms to its interface) and that instance is populated with data
from this manifest.