Bug 1402012 - Update buildconfig.py to use PartialConfigEnvironment; r?glandium draft
authorMike Shal <mshal@mozilla.com>
Thu, 24 Aug 2017 22:52:01 -0400
changeset 672665 b0171dc25abd1d2573173bdfe701abf1559c8046
parent 672664 d0b6fa22dbd267233e2695286bb0d84546ba6184
child 733875 d9d0a2a1f301b641f09c22bf00dbbb44db13ff2b
push id82326
push userbmo:mshal@mozilla.com
push dateFri, 29 Sep 2017 15:41:13 +0000
reviewersglandium
bugs1402012
milestone58.0a1
Bug 1402012 - Update buildconfig.py to use PartialConfigEnvironment; r?glandium By using the PartialConfigEnvironment, the clients of buildconfig will depend on config.statusd/ files instead of config.status directly. Clients can access substs and defines using buildconfig.substs['FOO'] or buildconfig.defines['BAR'], and then collect file-level dependencies for make using buildconfig.get_dependencies(). All GENERATED_FILES rules already make use of this because file_generate.py automatically includes these dependencies (along with all python modules loaded). As a result of this commit, re-running configure will no longer cause the world to be rebuilt. Although config.status is updated, no build steps use config.status directly and instead depend on values in config.statusd/, which are written with FileAvoidWrite. Since those files are not official targets according to the make backend, make won't try to continually rebuild the backend when those files are out of date. And since they are FileAvoidWrite, make will only re-run dependent steps if the actual configure value has changed. As a result of using JSON to load data from the config.statusd directory, substs can be unicode (instead of a bare string type). generate_certdata.py converts the subst manually to a string so the value can be exported to the environment without issue on Windows. Additionally, patching the buildconfig.substs dict no longer works, so the unit-symbolstore.py test was modified to patch the underlying buildconfig.substs._dict instead. The other files that needed to be modified make use of all the defines for the preprocessor. Those that are used during 'mach build' now use buildconfig.defines['ALLDEFINES'], which maps to a special FileAvoidWrite file generated for the PartialConfigEnvironment. MozReview-Commit-ID: 2pJ4s3TVeS8
build/buildconfig.py
js/src/builtin/embedjs.py
mobile/android/base/generate_build_config.py
python/mozbuild/mozbuild/action/file_generate.py
python/mozbuild/mozbuild/action/generate_symbols_file.py
security/generate_certdata.py
toolkit/crashreporter/tools/unit-symbolstore.py
toolkit/mozapps/installer/find-dupes.py
toolkit/mozapps/installer/packager.py
--- a/build/buildconfig.py
+++ b/build/buildconfig.py
@@ -1,20 +1,19 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import sys
 from mozbuild.base import MozbuildObject
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
 
 config = MozbuildObject.from_environment()
+partial_config = PartialConfigEnvironment(config.topobjdir)
 
-for var in ('topsrcdir', 'topobjdir', 'defines', 'non_global_defines',
-            'substs'):
+for var in ('topsrcdir', 'topobjdir'):
     value = getattr(config, var)
     setattr(sys.modules[__name__], var, value)
 
-substs = dict(substs)
-
-for var in os.environ:
-    if var not in ('CPP', 'CXXCPP', 'SHELL') and var in substs:
-        substs[var] = os.environ[var]
+for var in ('defines', 'substs', 'get_dependencies'):
+    value = getattr(partial_config, var)
+    setattr(sys.modules[__name__], var, value)
--- a/js/src/builtin/embedjs.py
+++ b/js/src/builtin/embedjs.py
@@ -127,18 +127,17 @@ def messages(jsmsg):
       defines.append("#define %s %i" % (match.group(1), len(defines)))
     else:
       # Make sure that MSG_DEF isn't preceded by whitespace
       assert not line.strip().startswith("MSG_DEF")
   return '\n'.join(defines)
 
 def get_config_defines(buildconfig):
   # Collect defines equivalent to ACDEFINES and add MOZ_DEBUG_DEFINES.
-  env = {key: value for key, value in buildconfig.defines.iteritems()
-         if key not in buildconfig.non_global_defines}
+  env = buildconfig.defines['ALLDEFINES']
   for define in buildconfig.substs['MOZ_DEBUG_DEFINES']:
     env[define] = 1
   return env
 
 def process_inputs(namespace, c_out, msg_file, inputs):
   deps = [path for path in inputs if path.endswith(".h") or path.endswith(".h.js")]
   sources = [path for path in inputs if path.endswith(".js") and not path.endswith(".h.js")]
   assert len(deps) + len(sources) == len(inputs)
--- a/mobile/android/base/generate_build_config.py
+++ b/mobile/android/base/generate_build_config.py
@@ -28,18 +28,18 @@ import sys
 import buildconfig
 
 from mozbuild import preprocessor
 from mozbuild.android_version_code import android_version_code
 
 
 def _defines():
     CONFIG = defaultdict(lambda: None)
-    CONFIG.update(buildconfig.substs)
-    DEFINES = dict(buildconfig.defines)
+    CONFIG.update(buildconfig.substs.iteritems())
+    DEFINES = dict(buildconfig.defines.iteritems())
 
     for var in ('MOZ_ANDROID_ACTIVITY_STREAM'
                 'MOZ_ANDROID_ANR_REPORTER',
                 'MOZ_ANDROID_BEAM',
                 'MOZ_ANDROID_DOWNLOADS_INTEGRATION',
                 'MOZ_ANDROID_DOWNLOAD_CONTENT_SERVICE',
                 'MOZ_ANDROID_EXCLUDE_FONTS',
                 'MOZ_ANDROID_GCM',
--- a/python/mozbuild/mozbuild/action/file_generate.py
+++ b/python/mozbuild/mozbuild/action/file_generate.py
@@ -80,16 +80,20 @@ def main(argv):
                 deps = set()
 
             # Only write out the dependencies if the script was successful
             if not ret:
                 # Add dependencies on any python modules that were imported by
                 # the script.
                 deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
                                                  buildconfig.topobjdir))
+                # Add dependencies on any buildconfig items that were accessed
+                # by the script.
+                deps |= set(buildconfig.get_dependencies())
+
                 mk = Makefile()
                 mk.create_rule([args.output_file]).add_dependencies(deps)
                 with FileAvoidWrite(args.dep_file) as dep_file:
                     mk.dump(dep_file)
         # Even when our file's contents haven't changed, we want to update
         # the file's mtime so make knows this target isn't still older than
         # whatever prerequisite caused it to be built this time around.
         try:
--- a/python/mozbuild/mozbuild/action/generate_symbols_file.py
+++ b/python/mozbuild/mozbuild/action/generate_symbols_file.py
@@ -17,17 +17,17 @@ def generate_symbols_file(output, *args)
     parser = argparse.ArgumentParser()
     parser.add_argument('input')
     parser.add_argument('-D', action=DefinesAction)
     parser.add_argument('-U', action='append', default=[])
     args = parser.parse_args(args)
     input = os.path.abspath(args.input)
 
     pp = Preprocessor()
-    pp.context.update(buildconfig.defines)
+    pp.context.update(buildconfig.defines['ALLDEFINES'])
     if args.D:
         pp.context.update(args.D)
     for undefine in args.U:
         if undefine in pp.context:
             del pp.context[undefine]
     # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
     if buildconfig.substs['MOZ_DEBUG']:
         pp.context['DEBUG'] = '1'
--- a/security/generate_certdata.py
+++ b/security/generate_certdata.py
@@ -4,12 +4,12 @@
 # and moz.build `GENERATED_FILES` semantics.
 
 import buildconfig
 import os
 import subprocess
 
 def main(output, *inputs):
     env=dict(os.environ)
-    env['PERL'] = buildconfig.substs['PERL']
+    env['PERL'] = str(buildconfig.substs['PERL'])
     output.write(subprocess.check_output([buildconfig.substs['PYTHON'],
                  inputs[0], inputs[2]], env=env))
     return None
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -145,17 +145,17 @@ class TestCopyDebug(HelperMixin, unittes
         d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
                                                   symbol_path=self.symbol_dir,
                                                   copy_debug=True,
                                                   archs="abc xyz")
         d.CopyDebug = mock_copy_debug
         d.Process(os.path.join(self.test_dir, add_extension(["foo"])[0]))
         self.assertEqual(1, len(copied))
 
-    @patch.dict('buildconfig.substs', {'MAKECAB': 'makecab'})
+    @patch.dict('buildconfig.substs._dict', {'MAKECAB': 'makecab'})
     def test_copy_debug_copies_binaries(self):
         """
         Test that CopyDebug copies binaries as well on Windows.
         """
         test_file = os.path.join(self.test_dir, 'foo.dll')
         write_dll(test_file)
         code_file = 'foo.dll'
         code_id = 'abc123'
--- a/toolkit/mozapps/installer/find-dupes.py
+++ b/toolkit/mozapps/installer/find-dupes.py
@@ -97,17 +97,17 @@ def main():
     parser.add_argument('directory',
                         help='The directory to check for duplicates in')
 
     args = parser.parse_args()
 
     allowed_dupes = []
     for filename in args.dupes_files:
         pp = Preprocessor()
-        pp.context.update(buildconfig.defines)
+        pp.context.update(buildconfig.defines['ALLDEFINES'])
         if args.D:
             pp.context.update(args.D)
         for undefine in args.U:
             if undefine in pp.context:
                 del pp.context[undefine]
         pp.out = StringIO()
         pp.do_filter('substitution')
         pp.do_include(filename)
--- a/toolkit/mozapps/installer/packager.py
+++ b/toolkit/mozapps/installer/packager.py
@@ -221,17 +221,17 @@ def main():
                         help='Manifest file name')
     parser.add_argument('source', help='Source directory')
     parser.add_argument('destination', help='Destination directory')
     parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
                         default=[],
                         help='Extra files not to be considered as resources')
     args = parser.parse_args()
 
-    defines = dict(buildconfig.defines)
+    defines = dict(buildconfig.defines['ALLDEFINES'])
     if args.ignore_errors:
         errors.ignore_errors()
 
     if args.defines:
         for name, value in [split_define(d) for d in args.defines]:
             defines[name] = value
 
     compress = {