Bug 1384241 - Add |mach watch|: pywatchman based incremental |mach build faster| shell. r=gps draft
authorNick Alexander <nalexander@mozilla.com>
Wed, 26 Jul 2017 20:34:09 -0700
changeset 617908 13dbeac2fbfa78741be3718fd5305a8ae0d698a8
parent 617789 de26dd98836ec49d0abc93052e84b6a90b3e248f
child 639901 57e1df30b8985920bbfee76783de6be43289be4e
push id71149
push usernalexander@mozilla.com
push dateFri, 28 Jul 2017 22:59:27 +0000
reviewersgps
bugs1384241
milestone56.0a1
Bug 1384241 - Add |mach watch|: pywatchman based incremental |mach build faster| shell. r=gps There's a natural follow-on that I haven't time to explore right now: I want the faster make backend to also write a "unified chrome manifest" that maps outputs (browser/chrome/browser/content/browser/ext-utils.js) to chrome:// or resource:// URLs (chrome://content/browser/ext-utils.js or similar). MozReview-Commit-ID: LDQmm8KD57I
python/mozbuild/mozbuild/backend/fastermake.py
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/faster_daemon.py
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozpack/manifests.py
--- a/python/mozbuild/mozbuild/backend/fastermake.py
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -154,12 +154,28 @@ class FasterMakeBackend(CommonBackend, P
         mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
 
         for base, install_manifest in self._install_manifests.iteritems():
             with self._write_file(
                     mozpath.join(self.environment.topobjdir, 'faster',
                                  'install_%s' % base.replace('/', '_'))) as fh:
                 install_manifest.write(fileobj=fh)
 
+        # For artifact builds only, write a single unified manifest for consumption by |mach watch|.
+        if self.environment.is_artifact_build:
+            unified_manifest = InstallManifest()
+            for base, install_manifest in self._install_manifests.iteritems():
+                # Expect 'dist/bin/**', which includes 'dist/bin' with no trailing slash.
+                assert base.startswith('dist/bin')
+                base = base[len('dist/bin'):]
+                if base and base[0] == '/':
+                    base = base[1:]
+                unified_manifest.add_entries_from(install_manifest, base=base)
+
+            with self._write_file(
+                    mozpath.join(self.environment.topobjdir, 'faster',
+                                 'unified_install_dist_bin')) as fh:
+                unified_manifest.write(fileobj=fh)
+
         with self._write_file(
                 mozpath.join(self.environment.topobjdir, 'faster',
                              'Makefile')) as fh:
             mk.dump(fh, removal_guard=False)
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -753,16 +753,21 @@ class MachCommandConditions(object):
         """Must have a mercurial source checkout."""
         return getattr(cls, 'substs', {}).get('VCS_CHECKOUT_TYPE') == 'hg'
 
     @staticmethod
     def is_git(cls):
         """Must have a git source checkout."""
         return getattr(cls, 'substs', {}).get('VCS_CHECKOUT_TYPE') == 'git'
 
+    @staticmethod
+    def is_artifact_build(cls):
+        """Must be an artifact build."""
+        return getattr(cls, 'substs', {}).get('MOZ_ARTIFACT_BUILDS')
+
 
 class PathArgument(object):
     """Parse a filesystem path argument and transform it in various ways."""
 
     def __init__(self, arg, topsrcdir, topobjdir, cwd=None):
         self.arg = arg
         self.topsrcdir = topsrcdir
         self.topobjdir = topobjdir
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/faster_daemon.py
@@ -0,0 +1,280 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Use pywatchman to watch source directories and perform partial |mach
+build faster| builds.
+'''
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import datetime
+import sys
+import time
+
+import mozbuild.util
+import mozpack.path as mozpath
+from mozpack.manifests import (
+    InstallManifest,
+)
+from mozpack.copier import (
+    FileCopier,
+)
+
+# Watchman integration cribbed entirely from
+# https://github.com/facebook/watchman/blob/19aebfebb0b5b0b5174b3914a879370ffc5dac37/python/bin/watchman-wait
+import pywatchman
+
+
+def print_line(prefix, m, now=None):
+    now = now or datetime.datetime.utcnow()
+    print(b'[%s %sZ] %s' % (prefix, now.isoformat(), m))
+
+
+def print_copy_result(elapsed, destdir, result, verbose=True):
+    COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
+        'Added/updated {updated}; ' \
+        'Removed {rm_files} files and {rm_dirs} directories.'
+
+    print_line('watch', COMPLETE.format(
+        elapsed=elapsed,
+        dest=destdir,
+        existing=result.existing_files_count,
+        updated=result.updated_files_count,
+        rm_files=result.removed_files_count,
+        rm_dirs=result.removed_directories_count))
+
+
+class FasterBuildException(Exception):
+    def __init__(self, message, cause):
+        Exception.__init__(self, message)
+        self.cause = cause
+
+
+class FasterBuildChange(object):
+    def __init__(self):
+        self.unrecognized = set()
+        self.input_to_outputs = {}
+        self.output_to_inputs = {}
+
+
+class Daemon(object):
+    def __init__(self, config_environment):
+        self.config_environment = config_environment
+        self._client = None
+
+    @property
+    def defines(self):
+        defines = dict(self.config_environment.acdefines)
+        # These additions work around warts in the build system: see
+        # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/config/faster/rules.mk#92-93
+        # and
+        # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/python/mozbuild/mozbuild/backend/tup.py#244-253.
+        defines.update({
+            'AB_CD': 'en-US',
+            'BUILD_FASTER': '1',
+        })
+        defines.update({
+            'BOOKMARKS_INCLUDE_DIR': mozpath.join(self.config_environment.topsrcdir,
+                                                  'browser', 'locales', 'en-US', 'profile'),
+        })
+        return defines
+
+    @mozbuild.util.memoized_property
+    def file_copier(self):
+        # TODO: invalidate the file copier when the build system
+        # itself changes, i.e., the underlying unified manifest
+        # changes.
+        file_copier = FileCopier()
+
+        unified_manifest = InstallManifest(
+            mozpath.join(self.config_environment.topobjdir,
+                         'faster', 'unified_install_dist_bin'))
+
+        unified_manifest.populate_registry(file_copier, defines_override=self.defines)
+
+        return file_copier
+
+    def subscribe_to_topsrcdir(self):
+        self.subscribe_to_dir('topsrcdir', self.config_environment.topsrcdir)
+
+    def subscribe_to_dir(self, name, dir_to_watch):
+        query = {
+            'empty_on_fresh_instance': True,
+            'expression': [
+                'allof',
+                ['type', 'f'],
+                ['not',
+                 ['anyof',
+                  ['dirname', '.hg'],
+                  ['name', '.hg', 'wholename'],
+                  ['dirname', '.git'],
+                  ['name', '.git', 'wholename'],
+                 ],
+                ],
+            ],
+            'fields': ['name'],
+        }
+        watch = self.client.query('watch-project', dir_to_watch)
+        if 'warning' in watch:
+            print('WARNING: ', watch['warning'], file=sys.stderr)
+
+        root = watch['watch']
+        if 'relative_path' in watch:
+            query['relative_root'] = watch['relative_path']
+
+        # Get the initial clock value so that we only get updates.
+        # Wait 30s to allow for slow Windows IO.  See
+        # https://facebook.github.io/watchman/docs/cmd/clock.html.
+        query['since'] = self.client.query('clock', root, {'sync_timeout': 30000})['clock']
+
+        return self.client.query('subscribe', root, name, query)
+
+    def changed_files(self):
+        # In theory we can parse just the result variable here, but
+        # the client object will accumulate all subscription results
+        # over time, so we ask it to remove and return those values.
+        files = set()
+
+        data = self.client.getSubscription('topsrcdir')
+        if data:
+            for dat in data:
+                files |= set([mozpath.normpath(mozpath.join(self.config_environment.topsrcdir, f))
+                              for f in dat.get('files', [])])
+
+        return files
+
+    def incremental_copy(self, copier, force=False, verbose=True):
+        # Just like the 'repackage' target in browser/app/Makefile.in.
+        if 'cocoa' == self.config_environment.substs['MOZ_WIDGET_TOOLKIT']:
+            bundledir = mozpath.join(self.config_environment.topobjdir, 'dist',
+                                     self.config_environment.substs['MOZ_MACBUNDLE_NAME'],
+                                     'Contents', 'Resources')
+            start = time.time()
+            result = copier.copy(bundledir,
+                                 skip_if_older=not force,
+                                 remove_unaccounted=False,
+                                 remove_all_directory_symlinks=False,
+                                 remove_empty_directories=False)
+            print_copy_result(time.time() - start, bundledir, result, verbose=verbose)
+
+        destdir = mozpath.join(self.config_environment.topobjdir, 'dist', 'bin')
+        start = time.time()
+        result = copier.copy(destdir,
+                             skip_if_older=not force,
+                             remove_unaccounted=False,
+                             remove_all_directory_symlinks=False,
+                             remove_empty_directories=False)
+        print_copy_result(time.time() - start, destdir, result, verbose=verbose)
+
+    def input_changes(self, verbose=True):
+        '''
+        Return an iterator of `FasterBuildChange` instances as inputs
+        to the faster build system change.
+        '''
+
+        # TODO: provide the debug diagnostics we want: this print is
+        # not immediately before the watch.
+        if verbose:
+            print_line('watch', 'Connecting to watchman')
+        # TODO: figure out why a large timeout is required for the
+        # client, and a robust strategy for retrying timed out
+        # requests.
+        self.client = pywatchman.client(timeout=5.0)
+
+        try:
+            if verbose:
+                print_line('watch', 'Checking watchman capabilities')
+            # TODO: restrict these capabilities to the minimal set.
+            self.client.capabilityCheck(required=[
+                'clock-sync-timeout',
+                'cmd-watch-project',
+                'term-dirname',
+                'wildmatch',
+            ])
+
+            if verbose:
+                print_line('watch', 'Subscribing to {}'.format(self.config_environment.topsrcdir))
+            self.subscribe_to_topsrcdir()
+            if verbose:
+                print_line('watch', 'Watching {}'.format(self.config_environment.topsrcdir))
+
+            input_to_outputs = self.file_copier.input_to_outputs_tree()
+            for input, outputs in input_to_outputs.items():
+                if not outputs:
+                    raise Exception("Refusing to watch input ({}) with no outputs".format(input))
+
+            while True:
+                try:
+                    _watch_result = self.client.receive()
+
+                    changed = self.changed_files()
+                    if not changed:
+                        continue
+
+                    result = FasterBuildChange()
+
+                    for change in changed:
+                        if change in input_to_outputs:
+                            result.input_to_outputs[change] = set(input_to_outputs[change])
+                        else:
+                            result.unrecognized.add(change)
+
+                    for input, outputs in result.input_to_outputs.items():
+                        for output in outputs:
+                            if output not in result.output_to_inputs:
+                                result.output_to_inputs[output] = set()
+                            result.output_to_inputs[output].add(input)
+
+                    yield result
+
+                except pywatchman.SocketTimeout:
+                    # Let's check to see if we're still functional.
+                    _version = self.client.query('version')
+
+        except pywatchman.CommandError as e:
+            # Abstract away pywatchman errors.
+            raise FasterBuildException(e, 'Command error using pywatchman to watch {}'.format(
+                self.config_environment.topsrcdir))
+
+        except pywatchman.SocketTimeout as e:
+            # Abstract away pywatchman errors.
+            raise FasterBuildException(e, 'Socket timeout using pywatchman to watch {}'.format(
+                self.config_environment.topsrcdir))
+
+        finally:
+            self.client.close()
+
+    def output_changes(self, verbose=True):
+        '''
+        Return an iterator of `FasterBuildChange` instances as outputs
+        from the faster build system are updated.
+        '''
+        for change in self.input_changes(verbose=verbose):
+            now = datetime.datetime.utcnow()
+
+            for unrecognized in sorted(change.unrecognized):
+                print_line('watch', '! {}'.format(unrecognized), now=now)
+
+            all_outputs = set()
+            for input in sorted(change.input_to_outputs):
+                outputs = change.input_to_outputs[input]
+
+                print_line('watch', '< {}'.format(input), now=now)
+                for output in sorted(outputs):
+                    print_line('watch', '> {}'.format(output), now=now)
+                all_outputs |= outputs
+
+            if all_outputs:
+                partial_copier = FileCopier()
+                for output in all_outputs:
+                    partial_copier.add(output, self.file_copier[output])
+
+                self.incremental_copy(partial_copier, force=True, verbose=verbose)
+                yield change
+
+    def watch(self, verbose=True):
+        for change in self.output_changes(verbose=verbose):
+            pass
+
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -302,16 +302,55 @@ class BuildOutputManager(OutputManager):
 class StoreDebugParamsAndWarnAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         sys.stderr.write('The --debugparams argument is deprecated. Please ' +
                          'use --debugger-args instead.\n\n')
         setattr(namespace, self.dest, values)
 
 
 @CommandProvider
+class Watch(MachCommandBase):
+    """Interface to watch and re-build the tree."""
+
+    @Command('watch', category='post-build', description='Watch and re-build the tree.',
+             conditions=[conditions.is_firefox])
+    @CommandArgument('-v', '--verbose', action='store_true',
+                     help='Verbose output for what commands the watcher is running.')
+    def watch(self, verbose=False):
+        """Watch and re-build the source tree."""
+
+        if not conditions.is_artifact_build(self):
+            print('mach watch requires an artifact build. See '
+                  'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Simple_Firefox_build')
+            return 1
+
+        if not self.substs.get('WATCHMAN', None):
+            print('mach watch requires watchman to be installed. See '
+                  'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching')
+            return 1
+
+        self._activate_virtualenv()
+        try:
+            self.virtualenv_manager.install_pip_package('pywatchman==1.3.0')
+        except Exception:
+            print('Could not install pywatchman from pip. See '
+                  'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching')
+            return 1
+
+        from mozbuild.faster_daemon import Daemon
+        daemon = Daemon(self.config_environment)
+
+        try:
+            return daemon.watch()
+        except KeyboardInterrupt:
+            # Suppress ugly stack trace when user hits Ctrl-C.
+            sys.exit(3)
+
+
+@CommandProvider
 class Build(MachCommandBase):
     """Interface to build the tree."""
 
     @Command('build', category='build', description='Build the tree.')
     @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,
         help='Number of concurrent jobs to run. Default is the number of CPUs.')
     @CommandArgument('-C', '--directory', default=None,
         help='Change to a subdirectory of the build directory first.')
--- a/python/mozbuild/mozpack/manifests.py
+++ b/python/mozbuild/mozpack/manifests.py
@@ -195,26 +195,17 @@ class InstallManifest(object):
 
     def __neq__(self, other):
         return not self.__eq__(other)
 
     def __ior__(self, other):
         if not isinstance(other, InstallManifest):
             raise ValueError('Can only | with another instance of InstallManifest.')
 
-        # We must copy source files to ourselves so extra dependencies from
-        # the preprocessor are taken into account. Ideally, we would track
-        # which source file each entry came from. However, this is more
-        # complicated and not yet implemented. The current implementation
-        # will result in over invalidation, possibly leading to performance
-        # loss.
-        self._source_files |= other._source_files
-
-        for dest in sorted(other._dests):
-            self._add_entry(dest, other._dests[dest])
+        self.add_entries_from(other)
 
         return self
 
     def _encode_field_entry(self, data):
         """Converts an object into a format that can be stored in the manifest file.
 
         Complex data types, such as ``dict``, need to be converted into a text
         representation before they can be written to a file.
@@ -326,16 +317,36 @@ class InstallManifest(object):
         ))
 
     def _add_entry(self, dest, entry):
         if dest in self._dests:
             raise ValueError('Item already in manifest: %s' % dest)
 
         self._dests[dest] = entry
 
+    def add_entries_from(self, other, base=''):
+        """
+        Copy data from another mozpack.copier.InstallManifest
+        instance, adding an optional base prefix to the destination.
+
+        This allows to merge two manifests into a single manifest, or
+        two take the tagged union of two manifests.
+        """
+        # We must copy source files to ourselves so extra dependencies from
+        # the preprocessor are taken into account. Ideally, we would track
+        # which source file each entry came from. However, this is more
+        # complicated and not yet implemented. The current implementation
+        # will result in over invalidation, possibly leading to performance
+        # loss.
+        self._source_files |= other._source_files
+
+        for dest in sorted(other._dests):
+            new_dest = mozpath.join(base, dest) if base else dest
+            self._add_entry(new_dest, other._dests[dest])
+
     def populate_registry(self, registry, defines_override={},
                           link_policy='symlink'):
         """Populate a mozpack.copier.FileRegistry instance with data from us.
 
         The caller supplied a FileRegistry instance (or at least something that
         conforms to its interface) and that instance is populated with data
         from this manifest.