Bug 1416052 - Generate a make file to include in client.mk; r=nalexander draft
authorGregory Szorc <gps@mozilla.com>
Thu, 09 Nov 2017 22:27:16 -0800
changeset 697343 b1380e8cda858a7c5144c14d2626f4687541acfb
parent 697342 d86f00b120dd5fadacc7d27df3b0aaa39afb7e78
child 697344 35488ba16bbfaab0327d6a3a6fec2108c1659cbb
push id88985
push userbmo:gps@mozilla.com
push dateMon, 13 Nov 2017 23:04:28 +0000
reviewersnalexander
bugs1416052
milestone59.0a1
Bug 1416052 - Generate a make file to include in client.mk; r=nalexander Currently, client.mk calls `mach environment` to obtain a make file to be evaluated in the context of client.mk. The reason it is implemented this way is because client.mk could be an entrypoint to the build system. With recent changes that require the use of mach to use client.mk, we are now guaranteed to have Python code running before client.mk is invoked. This means we don't need to invoke `mach` from client.mk. This commit ports the code for generating a client.mk suitable make file from `mach environment` to the build dispatcher. We now write out a new .mozconfig-client-mk file in the objdir. client.mk is changed to cat this file and to include it as a native make file. The OBJDIR environment variable is also set so client.mk knows where to read the auto-generated file from. This commit should be backwards compatible. Hopefully it is obvious, but this new make file is only temporary. As soon as the remaining mozconfig logic is moved out of client.mk, we should be able to simplify down to a single "include" in client.mk. MozReview-Commit-ID: BEfWo76Z1qA
client.mk
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/mach_commands.py
--- a/client.mk
+++ b/client.mk
@@ -44,18 +44,18 @@ define CR
 
 endef
 
 # As $(shell) doesn't preserve newlines, use sed to replace them with an
 # unlikely sequence (||), which is then replaced back to newlines by make
 # before evaluation. $(shell) replacing newlines with spaces, || is always
 # followed by a space (since sed doesn't remove newlines), except on the
 # last line, so replace both '|| ' and '||'.
-MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell $(TOPSRCDIR)/mach environment --format=client.mk | sed 's/$$/||/')))
-$(eval $(MOZCONFIG_CONTENT))
+MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell cat $(OBJDIR)/.mozconfig-client-mk | sed 's/$$/||/')))
+include $(OBJDIR)/.mozconfig-client-mk
 
 export FOUND_MOZCONFIG
 
 # As '||' was used as a newline separator, it means it's not occurring in
 # lines themselves. It can thus safely be used to replaces normal spaces,
 # to then replace newlines with normal spaces. This allows to get a list
 # of mozconfig output lines.
 MOZCONFIG_OUT_LINES := $(subst $(CR), ,$(subst $(NULL) $(NULL),||,$(MOZCONFIG_CONTENT)))
@@ -103,18 +103,17 @@ ifndef MACH
 $(error client.mk must be used via `mach`. Try running \
 `./mach $(firstword $(MAKECMDGOALS) $(.DEFAULT_GOAL))`)
 endif
 
 # Define mkdir
 include $(TOPSRCDIR)/config/makefiles/makeutils.mk
 include $(TOPSRCDIR)/config/makefiles/autotargets.mk
 
-# For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES
-# from mach environment --format=client.mk output.
+# For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES.
 MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES))
 $(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG) $(call mkdir_deps,$(OBJDIR)) $(OBJDIR)/CLOBBER
 	$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@
 
 # Include that makefile so that it is created. This should not actually change
 # the environment since MOZCONFIG_CONTENT, which MOZCONFIG_OUT_LINES derives
 # from, has already been eval'ed.
 include $(OBJDIR)/.mozconfig.mk
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -45,16 +45,17 @@ from ..testing import (
 from ..compilation.warnings import (
     WarningsCollector,
     WarningsDatabase,
 )
 from ..shellutil import (
     quote as shell_quote,
 )
 from ..util import (
+    FileAvoidWrite,
     mkdir,
     resolve_target_to_make,
 )
 
 
 FINDER_SLOW_MESSAGE = '''
 ===================
 PERFORMANCE WARNING
@@ -1318,16 +1319,34 @@ class BuildDriver(MozbuildObject):
 
     def _run_client_mk(self, target=None, line_handler=None, jobs=0,
                        verbose=None, keep_going=False, append_env=None):
         append_env = dict(append_env or {})
         append_env['TOPSRCDIR'] = self.topsrcdir
 
         append_env['CONFIG_GUESS'] = self.resolve_config_guess()
 
+        mozconfig = self.mozconfig
+        mozconfig_client_mk = os.path.join(self.topobjdir,
+                                           '.mozconfig-client-mk')
+        with FileAvoidWrite(mozconfig_client_mk) as fh:
+            for arg in mozconfig['make_extra'] or []:
+                fh.write(arg)
+                fh.write(b'\n')
+            if mozconfig['make_flags']:
+                fh.write(b'MOZ_MAKE_FLAGS=%s\n' % b' '.join(mozconfig['make_flags']))
+            objdir = mozpath.normsep(self.topobjdir)
+            fh.write(b'MOZ_OBJDIR=%s\n' % objdir)
+            fh.write(b'OBJDIR=%s\n' % objdir)
+            if mozconfig['path']:
+                fh.write(b'FOUND_MOZCONFIG=%s\n' %
+                         mozpath.normsep(mozconfig['path']))
+
+        append_env['OBJDIR'] = mozpath.normsep(self.topobjdir)
+
         return self._run_make(srcdir=True,
                               filename='client.mk',
                               allow_parallel=False,
                               ensure_exit_code=False,
                               print_directory=False,
                               target=target,
                               line_handler=line_handler,
                               log=False,
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1016,17 +1016,17 @@ class Makefiles(MachCommandBase):
                 if f == 'Makefile.in':
                     yield os.path.join(root, f)
 
 @CommandProvider
 class MachDebug(MachCommandBase):
     @Command('environment', category='build-dev',
         description='Show info about the mach and build environment.')
     @CommandArgument('--format', default='pretty',
-        choices=['pretty', 'client.mk', 'configure', 'json'],
+        choices=['pretty', 'configure', 'json'],
         help='Print data in the given format.')
     @CommandArgument('--output', '-o', type=str,
         help='Output to the given file.')
     @CommandArgument('--verbose', '-v', action='store_true',
         help='Print verbose output.')
     def environment(self, format, output=None, verbose=False):
         func = getattr(self, '_environment_%s' % format.replace('.', '_'))
 
@@ -1084,29 +1084,16 @@ class MachDebug(MachCommandBase):
                 print('config substitutions:', file=out)
                 for k in sorted(config.substs):
                     print('\t%s: %s' % (k, config.substs[k]), file=out)
 
                 print('config defines:', file=out)
                 for k in sorted(config.defines):
                     print('\t%s' % k, file=out)
 
-    def _environment_client_mk(self, out, verbose):
-        if self.mozconfig['make_extra']:
-            for arg in self.mozconfig['make_extra']:
-                print(arg, file=out)
-        if self.mozconfig['make_flags']:
-            print('MOZ_MAKE_FLAGS=%s' % ' '.join(self.mozconfig['make_flags']))
-        objdir = mozpath.normsep(self.topobjdir)
-        print('MOZ_OBJDIR=%s' % objdir, file=out)
-        print('OBJDIR=%s' % objdir, file=out)
-        if self.mozconfig['path']:
-            print('FOUND_MOZCONFIG=%s' % mozpath.normsep(self.mozconfig['path']),
-                file=out)
-
     def _environment_json(self, out, verbose):
         import json
         class EnvironmentEncoder(json.JSONEncoder):
             def default(self, obj):
                 if isinstance(obj, MozbuildObject):
                     result = {
                         'topsrcdir': obj.topsrcdir,
                         'topobjdir': obj.topobjdir,