Bug 1402012 - Create config.statusd directory; r?glandium
The config.statusd directory is created alongside config.status, which
contains the same information but is split across many files instead of
all in a single file. This allows the build system to track dependencies
on individual configure values.
MozReview-Commit-ID: 2DbwKCJuNSX
--- a/configure.py
+++ b/configure.py
@@ -12,16 +12,17 @@ import sys
import textwrap
base_dir = os.path.abspath(os.path.dirname(__file__))
sys.path.insert(0, os.path.join(base_dir, 'python', 'mozbuild'))
from mozbuild.configure import ConfigureSandbox
from mozbuild.makeutil import Makefile
from mozbuild.pythonutil import iter_modules_in_path
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
from mozbuild.util import (
indented_repr,
encode,
)
def main(argv):
config = {}
@@ -85,16 +86,19 @@ def config_status(config):
if __name__ == '__main__':
from mozbuild.util import patch_main
patch_main()
from mozbuild.config_status import config_status
args = dict([(name, globals()[name]) for name in __all__])
config_status(**args)
'''))
+ partial_config = PartialConfigEnvironment(config['TOPOBJDIR'])
+ partial_config.write_vars(sanitized_config)
+
# Write out a depfile so Make knows to re-run configure when relevant Python
# changes.
mk = Makefile()
rule = mk.create_rule()
rule.add_targets(["$(OBJDIR)/config.status"])
rule.add_dependencies(itertools.chain(config['ALL_CONFIGURE_PATHS'],
iter_modules_in_path(config['TOPOBJDIR'],
config['TOPSRCDIR'])))
--- a/python/mozbuild/mozbuild/backend/configenvironment.py
+++ b/python/mozbuild/mozbuild/backend/configenvironment.py
@@ -1,23 +1,25 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import os
import sys
+import json
-from collections import Iterable
+from collections import Iterable, OrderedDict
from types import StringTypes, ModuleType
import mozpack.path as mozpath
from mozbuild.util import (
+ FileAvoidWrite,
memoized_property,
ReadOnlyDict,
)
from mozbuild.shellutil import quote as shell_quote
if sys.version_info.major == 2:
text_type = unicode
@@ -206,8 +208,159 @@ class ConfigEnvironment(object):
return ReadOnlyDict(acdefines)
@staticmethod
def from_config_status(path):
config = BuildConfig.from_config_status(path)
return ConfigEnvironment(config.topsrcdir, config.topobjdir,
config.defines, config.non_global_defines, config.substs, path)
+
+
+class PartialConfigDict(object):
+ """Facilitates mapping the config.statusd defines & substs with dict-like access.
+
+ This allows a buildconfig client to use buildconfig.defines['FOO'] (and
+ similar for substs), where the value of FOO is delay-loaded until it is
+ needed.
+ """
+ def __init__(self, config_statusd, typ, environ_override=False):
+ self._dict = {}
+ self._datadir = mozpath.join(config_statusd, typ)
+ self._config_track = mozpath.join(self._datadir, 'config.track')
+ self._files = set()
+ self._environ_override = environ_override
+
+ def _load_config_track(self):
+ existing_files = set()
+ try:
+ with open(self._config_track) as fh:
+ existing_files.update(fh.read().splitlines())
+ except IOError:
+ pass
+ return existing_files
+
+ def _write_file(self, key, value):
+ filename = mozpath.join(self._datadir, key)
+ with FileAvoidWrite(filename) as fh:
+ json.dump(value, fh, indent=4)
+ return filename
+
+ def _fill_group(self, values):
+ # Clear out any cached values. This is mostly for tests that will check
+ # the environment, write out a new set of variables, and then check the
+ # environment again. Normally only configure ends up calling this
+ # function, and other consumers create their own
+ # PartialConfigEnvironments in new python processes.
+ self._dict = {}
+
+ existing_files = self._load_config_track()
+
+ new_files = set()
+ for k, v in values.iteritems():
+ new_files.add(self._write_file(k, v))
+
+ for filename in existing_files - new_files:
+ # We can't actually os.remove() here, since make would not see that the
+ # file has been removed and that the target needs to be updated. Instead
+ # we just overwrite the file with a value of None, which is equivalent
+ # to a non-existing file.
+ with FileAvoidWrite(filename) as fh:
+ json.dump(None, fh)
+
+ with FileAvoidWrite(self._config_track) as fh:
+ for f in sorted(new_files):
+ fh.write('%s\n' % f)
+
+ def __getitem__(self, key):
+ if self._environ_override:
+ if (key not in ('CPP', 'CXXCPP', 'SHELL')) and (key in os.environ):
+ return os.environ[key]
+
+ if key not in self._dict:
+ data = None
+ try:
+ filename = mozpath.join(self._datadir, key)
+ self._files.add(filename)
+ with open(filename) as f:
+ data = json.load(f)
+ except IOError:
+ pass
+ self._dict[key] = data
+
+ if self._dict[key] is None:
+ raise KeyError("'%s'" % key)
+ return self._dict[key]
+
+ def __setitem__(self, key, value):
+ self._dict[key] = value
+
+ def get(self, key, default=None):
+ return self[key] if key in self else default
+
+ def __contains__(self, key):
+ try:
+ return self[key] is not None
+ except KeyError:
+ return False
+
+ def iteritems(self):
+ existing_files = self._load_config_track()
+ for f in existing_files:
+ # The track file contains filenames, and the basename is the
+ # variable name.
+ var = mozpath.basename(f)
+ yield var, self[var]
+
+
+class PartialConfigEnvironment(object):
+ """Allows access to individual config.status items via config.statusd/* files.
+
+ This class is similar to the full ConfigEnvironment, which uses
+ config.status, except this allows access and tracks dependencies to
+ individual configure values. It is intended to be used during the build
+ process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and
+ anything else that may need to access specific substs or defines.
+
+ Creating a PartialConfigEnvironment requires only the topobjdir, which is
+ needed to distinguish between the top-level environment and the js/src
+ environment.
+
+ The PartialConfigEnvironment automatically defines one additional subst variable
+ from all the defines not appearing in non_global_defines:
+ - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
+ preprocessor command lines. The order in which defines were given
+ when creating the ConfigEnvironment is preserved.
+
+ and one additional define from all the defines as a dictionary:
+ - ALLDEFINES contains all of the global defines as a dictionary. This is
+ intended to be used instead of the defines structure from config.status so
+ that scripts can depend directly on its value.
+ """
+ def __init__(self, topobjdir):
+ config_statusd = mozpath.join(topobjdir, 'config.statusd')
+ self.substs = PartialConfigDict(config_statusd, 'substs', environ_override=True)
+ self.defines = PartialConfigDict(config_statusd, 'defines')
+ self.topobjdir = topobjdir
+
+ def write_vars(self, config):
+ substs = config['substs'].copy()
+ defines = config['defines'].copy()
+
+ global_defines = [
+ name for name in config['defines']
+ if name not in config['non_global_defines']
+ ]
+ acdefines = ' '.join(['-D%s=%s' % (name,
+ shell_quote(config['defines'][name]).replace('$', '$$'))
+ for name in sorted(global_defines)])
+ substs['ACDEFINES'] = acdefines
+
+ all_defines = OrderedDict()
+ for k in global_defines:
+ all_defines[k] = config['defines'][k]
+ defines['ALLDEFINES'] = all_defines
+
+ self.substs._fill_group(substs)
+ self.defines._fill_group(defines)
+
+ def get_dependencies(self):
+ return ['$(wildcard %s)' % f for f in self.substs._files | self.defines._files]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py
@@ -0,0 +1,162 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import unittest
+from mozunit import main
+from tempfile import mkdtemp
+from shutil import rmtree
+
+import mozpack.path as mozpath
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
+
+config = {
+ 'defines': {
+ 'MOZ_FOO': '1',
+ 'MOZ_BAR': '2',
+ 'MOZ_NON_GLOBAL': '3',
+ },
+ 'substs': {
+ 'MOZ_SUBST_1': '1',
+ 'MOZ_SUBST_2': '2',
+ 'CPP': 'cpp',
+ },
+ 'non_global_defines': [
+ 'MOZ_NON_GLOBAL',
+ ],
+}
+
+
+class TestPartial(unittest.TestCase):
+ def setUp(self):
+ self._old_env = dict(os.environ)
+
+ def tearDown(self):
+ os.environ.clear()
+ os.environ.update(self._old_env)
+
+ def _objdir(self):
+ objdir = mkdtemp()
+ self.addCleanup(rmtree, objdir)
+ return objdir
+
+ def test_auto_substs(self):
+ '''Test the automatically set values of ACDEFINES, and ALLDEFINES
+ '''
+ env = PartialConfigEnvironment(self._objdir())
+ env.write_vars(config)
+ self.assertEqual(env.substs['ACDEFINES'], '-DMOZ_BAR=2 -DMOZ_FOO=1')
+ self.assertEqual(env.defines['ALLDEFINES'], {
+ 'MOZ_BAR': '2',
+ 'MOZ_FOO': '1',
+ })
+
+ def test_remove_subst(self):
+ '''Test removing a subst from the config. The file should be overwritten with 'None'
+ '''
+ env = PartialConfigEnvironment(self._objdir())
+ path = mozpath.join(env.topobjdir, 'config.statusd', 'substs', 'MYSUBST')
+ myconfig = config.copy()
+ env.write_vars(myconfig)
+ with self.assertRaises(KeyError):
+ x = env.substs['MYSUBST']
+ self.assertFalse(os.path.exists(path))
+
+ myconfig['substs']['MYSUBST'] = 'new'
+ env.write_vars(myconfig)
+
+ self.assertEqual(env.substs['MYSUBST'], 'new')
+ self.assertTrue(os.path.exists(path))
+
+ del myconfig['substs']['MYSUBST']
+ env.write_vars(myconfig)
+ with self.assertRaises(KeyError):
+ x = env.substs['MYSUBST']
+ # Now that the subst is gone, the file still needs to be present so that
+ # make can update dependencies correctly. Overwriting the file with
+ # 'None' is the same as deleting it as far as the
+ # PartialConfigEnvironment is concerned, but make can't track a
+ # dependency on a file that doesn't exist.
+ self.assertTrue(os.path.exists(path))
+
+ def _assert_deps(self, env, deps):
+ deps = sorted(['$(wildcard %s)' % (mozpath.join(env.topobjdir, 'config.statusd', d)) for d in deps])
+ self.assertEqual(sorted(env.get_dependencies()), deps)
+
+ def test_dependencies(self):
+ '''Test getting dependencies on defines and substs.
+ '''
+ env = PartialConfigEnvironment(self._objdir())
+ env.write_vars(config)
+ self._assert_deps(env, [])
+
+ self.assertEqual(env.defines['MOZ_FOO'], '1')
+ self._assert_deps(env, ['defines/MOZ_FOO'])
+
+ self.assertEqual(env.defines['MOZ_BAR'], '2')
+ self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR'])
+
+ # Getting a define again shouldn't add a redundant dependency
+ self.assertEqual(env.defines['MOZ_FOO'], '1')
+ self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR'])
+
+ self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
+ self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1'])
+
+ with self.assertRaises(KeyError):
+ x = env.substs['NON_EXISTENT']
+ self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1', 'substs/NON_EXISTENT'])
+ self.assertEqual(env.substs.get('NON_EXISTENT'), None)
+
+ def test_set_subst(self):
+ '''Test setting a subst
+ '''
+ env = PartialConfigEnvironment(self._objdir())
+ env.write_vars(config)
+
+ self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
+ env.substs['MOZ_SUBST_1'] = 'updated'
+ self.assertEqual(env.substs['MOZ_SUBST_1'], 'updated')
+
+ # A new environment should pull the result from the file again.
+ newenv = PartialConfigEnvironment(env.topobjdir)
+ self.assertEqual(newenv.substs['MOZ_SUBST_1'], '1')
+
+ def test_env_override(self):
+ '''Test overriding a subst with an environment variable
+ '''
+ env = PartialConfigEnvironment(self._objdir())
+ env.write_vars(config)
+
+ self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
+ self.assertEqual(env.substs['CPP'], 'cpp')
+
+ # Reset the environment and set some environment variables.
+ env = PartialConfigEnvironment(env.topobjdir)
+ os.environ['MOZ_SUBST_1'] = 'subst 1 environ'
+ os.environ['CPP'] = 'cpp environ'
+
+ # The MOZ_SUBST_1 should be overridden by the environment, while CPP is
+ # a special variable and should not.
+ self.assertEqual(env.substs['MOZ_SUBST_1'], 'subst 1 environ')
+ self.assertEqual(env.substs['CPP'], 'cpp')
+
+ def test_update(self):
+ '''Test calling update on the substs or defines pseudo dicts
+ '''
+ env = PartialConfigEnvironment(self._objdir())
+ env.write_vars(config)
+
+ mysubsts = {'NEW': 'new'}
+ mysubsts.update(env.substs.iteritems())
+ self.assertEqual(mysubsts['NEW'], 'new')
+ self.assertEqual(mysubsts['CPP'], 'cpp')
+
+ mydefines = {'DEBUG': '1'}
+ mydefines.update(env.defines.iteritems())
+ self.assertEqual(mydefines['DEBUG'], '1')
+ self.assertEqual(mydefines['MOZ_FOO'], '1')
+
+if __name__ == "__main__":
+ main()
--- a/python/mozbuild/mozbuild/test/python.ini
+++ b/python/mozbuild/mozbuild/test/python.ini
@@ -1,13 +1,14 @@
[action/test_buildlist.py]
[action/test_generate_browsersearch.py]
[action/test_package_fennec_apk.py]
[backend/test_build.py]
[backend/test_configenvironment.py]
+[backend/test_partialconfigenvironment.py]
[backend/test_recursivemake.py]
[backend/test_test_manifest.py]
[backend/test_visualstudio.py]
[codecoverage/test_lcov_rewrite.py]
[compilation/test_warnings.py]
[configure/lint.py]
[configure/test_checks_configure.py]
[configure/test_compile_checks.py]