Bug 1402012 - Update buildconfig.py to use PartialConfigEnvironment; r?glandium
By using the PartialConfigEnvironment, the clients of buildconfig will
depend on config.statusd/ files instead of config.status directly.
Clients can access substs and defines using buildconfig.substs['FOO'] or
buildconfig.defines['BAR'], and then collect file-level dependencies for
make using buildconfig.get_dependencies(). All GENERATED_FILES rules
already make use of this because file_generate.py automatically includes
these dependencies (along with all python modules loaded).
As a result of this commit, re-running configure will no longer cause
the world to be rebuilt. Although config.status is updated, no build
steps use config.status directly and instead depend on values in
config.statusd/, which are written with FileAvoidWrite. Since those
files are not official targets according to the make backend, make won't
try to continually rebuild the backend when those files are out of date.
And since they are FileAvoidWrite, make will only re-run dependent steps
if the actual configure value has changed.
As a result of using JSON to load data from the config.statusd
directory, substs can be unicode (instead of a bare string type).
generate_certdata.py converts the subst manually to a string so the
value can be exported to the environment without issue on Windows.
Additionally, patching the buildconfig.substs dict no longer works, so
the unit-symbolstore.py test was modified to patch the underlying
buildconfig.substs._dict instead.
The other files that needed to be modified make use of all the defines
for the preprocessor. Those that are used during 'mach build' now use
buildconfig.defines['ALLDEFINES'], which maps to a special
FileAvoidWrite file generated for the PartialConfigEnvironment.
MozReview-Commit-ID: 2pJ4s3TVeS8
--- a/build/buildconfig.py
+++ b/build/buildconfig.py
@@ -1,20 +1,19 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import sys
from mozbuild.base import MozbuildObject
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
config = MozbuildObject.from_environment()
+partial_config = PartialConfigEnvironment(config.topobjdir)
-for var in ('topsrcdir', 'topobjdir', 'defines', 'non_global_defines',
- 'substs'):
+for var in ('topsrcdir', 'topobjdir'):
value = getattr(config, var)
setattr(sys.modules[__name__], var, value)
-substs = dict(substs)
-
-for var in os.environ:
- if var not in ('CPP', 'CXXCPP', 'SHELL') and var in substs:
- substs[var] = os.environ[var]
+for var in ('defines', 'substs', 'get_dependencies'):
+ value = getattr(partial_config, var)
+ setattr(sys.modules[__name__], var, value)
--- a/js/src/builtin/embedjs.py
+++ b/js/src/builtin/embedjs.py
@@ -127,18 +127,17 @@ def messages(jsmsg):
defines.append("#define %s %i" % (match.group(1), len(defines)))
else:
# Make sure that MSG_DEF isn't preceded by whitespace
assert not line.strip().startswith("MSG_DEF")
return '\n'.join(defines)
def get_config_defines(buildconfig):
# Collect defines equivalent to ACDEFINES and add MOZ_DEBUG_DEFINES.
- env = {key: value for key, value in buildconfig.defines.iteritems()
- if key not in buildconfig.non_global_defines}
+ env = buildconfig.defines['ALLDEFINES']
for define in buildconfig.substs['MOZ_DEBUG_DEFINES']:
env[define] = 1
return env
def process_inputs(namespace, c_out, msg_file, inputs):
deps = [path for path in inputs if path.endswith(".h") or path.endswith(".h.js")]
sources = [path for path in inputs if path.endswith(".js") and not path.endswith(".h.js")]
assert len(deps) + len(sources) == len(inputs)
--- a/mobile/android/base/generate_build_config.py
+++ b/mobile/android/base/generate_build_config.py
@@ -28,18 +28,18 @@ import sys
import buildconfig
from mozbuild import preprocessor
from mozbuild.android_version_code import android_version_code
def _defines():
CONFIG = defaultdict(lambda: None)
- CONFIG.update(buildconfig.substs)
- DEFINES = dict(buildconfig.defines)
+ CONFIG.update(buildconfig.substs.iteritems())
+ DEFINES = dict(buildconfig.defines.iteritems())
for var in ('MOZ_ANDROID_ACTIVITY_STREAM'
'MOZ_ANDROID_ANR_REPORTER',
'MOZ_ANDROID_BEAM',
'MOZ_ANDROID_DOWNLOADS_INTEGRATION',
'MOZ_ANDROID_DOWNLOAD_CONTENT_SERVICE',
'MOZ_ANDROID_EXCLUDE_FONTS',
'MOZ_ANDROID_GCM',
--- a/python/mozbuild/mozbuild/action/file_generate.py
+++ b/python/mozbuild/mozbuild/action/file_generate.py
@@ -80,16 +80,20 @@ def main(argv):
deps = set()
# Only write out the dependencies if the script was successful
if not ret:
# Add dependencies on any python modules that were imported by
# the script.
deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
buildconfig.topobjdir))
+ # Add dependencies on any buildconfig items that were accessed
+ # by the script.
+ deps |= set(buildconfig.get_dependencies())
+
mk = Makefile()
mk.create_rule([args.output_file]).add_dependencies(deps)
with FileAvoidWrite(args.dep_file) as dep_file:
mk.dump(dep_file)
# Even when our file's contents haven't changed, we want to update
# the file's mtime so make knows this target isn't still older than
# whatever prerequisite caused it to be built this time around.
try:
--- a/python/mozbuild/mozbuild/action/generate_symbols_file.py
+++ b/python/mozbuild/mozbuild/action/generate_symbols_file.py
@@ -17,17 +17,17 @@ def generate_symbols_file(output, *args)
parser = argparse.ArgumentParser()
parser.add_argument('input')
parser.add_argument('-D', action=DefinesAction)
parser.add_argument('-U', action='append', default=[])
args = parser.parse_args(args)
input = os.path.abspath(args.input)
pp = Preprocessor()
- pp.context.update(buildconfig.defines)
+ pp.context.update(buildconfig.defines['ALLDEFINES'])
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
# Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
if buildconfig.substs['MOZ_DEBUG']:
pp.context['DEBUG'] = '1'
--- a/security/generate_certdata.py
+++ b/security/generate_certdata.py
@@ -4,12 +4,12 @@
# and moz.build `GENERATED_FILES` semantics.
import buildconfig
import os
import subprocess
def main(output, *inputs):
env=dict(os.environ)
- env['PERL'] = buildconfig.substs['PERL']
+ env['PERL'] = str(buildconfig.substs['PERL'])
output.write(subprocess.check_output([buildconfig.substs['PYTHON'],
inputs[0], inputs[2]], env=env))
return None
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -145,17 +145,17 @@ class TestCopyDebug(HelperMixin, unittes
d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
symbol_path=self.symbol_dir,
copy_debug=True,
archs="abc xyz")
d.CopyDebug = mock_copy_debug
d.Process(os.path.join(self.test_dir, add_extension(["foo"])[0]))
self.assertEqual(1, len(copied))
- @patch.dict('buildconfig.substs', {'MAKECAB': 'makecab'})
+ @patch.dict('buildconfig.substs._dict', {'MAKECAB': 'makecab'})
def test_copy_debug_copies_binaries(self):
"""
Test that CopyDebug copies binaries as well on Windows.
"""
test_file = os.path.join(self.test_dir, 'foo.dll')
write_dll(test_file)
code_file = 'foo.dll'
code_id = 'abc123'
--- a/toolkit/mozapps/installer/find-dupes.py
+++ b/toolkit/mozapps/installer/find-dupes.py
@@ -97,17 +97,17 @@ def main():
parser.add_argument('directory',
help='The directory to check for duplicates in')
args = parser.parse_args()
allowed_dupes = []
for filename in args.dupes_files:
pp = Preprocessor()
- pp.context.update(buildconfig.defines)
+ pp.context.update(buildconfig.defines['ALLDEFINES'])
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
pp.out = StringIO()
pp.do_filter('substitution')
pp.do_include(filename)
--- a/toolkit/mozapps/installer/packager.py
+++ b/toolkit/mozapps/installer/packager.py
@@ -221,17 +221,17 @@ def main():
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
- defines = dict(buildconfig.defines)
+ defines = dict(buildconfig.defines['ALLDEFINES'])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {