Bug 1283052 - Remove some of the magic around mozconfig detection. r?gps
The mozconfig detection logic has bitten us on many occasions in the
past. The following changes are made to tentatively improve the
situation:
- The API is modified such that autodetection of the mozconfig has
to be a conscious decision made by the caller, and not triggered
any time there is no mozconfig given, which could be a conscious
decision of the opposite.
- mozinfo.json now stores the actual mozconfig (or lack thereof) used
during configure.
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -110,16 +110,18 @@ def mozconfig(current_project, mozconfig
loader = MozconfigLoader(build_env.topsrcdir)
current_project = current_project[0] if current_project else None
mozconfig = mozconfig[0] if mozconfig else None
mozconfig = loader.find_mozconfig(env={'MOZCONFIG': mozconfig})
mozconfig = loader.read_mozconfig(mozconfig, moz_build_app=current_project)
return mozconfig
+set_config('MOZCONFIG', depends(mozconfig)(lambda m: m['path']))
+
# Hacks related to old-configure
# ==============================
@depends('--help')
def old_configure_assignments(help):
return []
--- a/configure.py
+++ b/configure.py
@@ -45,16 +45,17 @@ def config_status(config):
if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR')
}
sanitized_config['defines'] = {
k: sanitized_bools(v) for k, v in config['DEFINES'].iteritems()
}
sanitized_config['non_global_defines'] = config['non_global_defines']
sanitized_config['topsrcdir'] = config['TOPSRCDIR']
sanitized_config['topobjdir'] = config['TOPOBJDIR']
+ sanitized_config['mozconfig'] = config.get('MOZCONFIG')
# Create config.status. Eventually, we'll want to just do the work it does
# here, when we're able to skip configure tests/use cached results/not rely
# on autoconf.
print("Creating config.status", file=sys.stderr)
encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
with codecs.open('config.status', 'w', encoding) as fh:
fh.write('#!%s\n' % config['PYTHON'])
@@ -63,17 +64,17 @@ def config_status(config):
# for True, False and None are true, false and null, which don't exist.
# Define them.
fh.write('true, false, null = True, False, None\n')
for k, v in sanitized_config.iteritems():
fh.write('%s = ' % k)
json.dump(v, fh, sort_keys=True, indent=4, ensure_ascii=False)
fh.write('\n')
fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
- "'non_global_defines', 'substs']")
+ "'non_global_defines', 'substs', 'mozconfig']")
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
fh.write('''
if __name__ == '__main__':
args = dict([(name, globals()[name]) for name in __all__])
from mozbuild.config_status import config_status
config_status(**args)
''')
--- a/python/mozbuild/mozbuild/backend/configenvironment.py
+++ b/python/mozbuild/mozbuild/backend/configenvironment.py
@@ -29,16 +29,17 @@ class BuildConfig(object):
def __init__(self):
self.topsrcdir = None
self.topobjdir = None
self.defines = {}
self.non_global_defines = []
self.substs = {}
self.files = []
+ self.mozconfig = None
@classmethod
def from_config_status(cls, path):
"""Create an instance from a config.status file."""
code_cache = cls._CODE_CACHE
mtime = os.path.getmtime(path)
# cache the compiled code as it can be reused
@@ -101,26 +102,27 @@ class ConfigEnvironment(object):
ConfigEnvironment expects a "top_srcdir" subst to be set with the top
source directory, in msys format on windows. It is used to derive a
"srcdir" subst when treating config files. It can either be an absolute
path or a path relative to the topobjdir.
"""
def __init__(self, topsrcdir, topobjdir, defines=None,
- non_global_defines=None, substs=None, source=None):
+ non_global_defines=None, substs=None, source=None, mozconfig=None):
if not source:
source = mozpath.join(topobjdir, 'config.status')
self.source = source
self.defines = ReadOnlyDict(defines or {})
self.non_global_defines = non_global_defines or []
self.substs = dict(substs or {})
self.topsrcdir = mozpath.abspath(topsrcdir)
self.topobjdir = mozpath.abspath(topobjdir)
+ self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None
self.lib_prefix = self.substs.get('LIB_PREFIX', '')
if 'LIB_SUFFIX' in self.substs:
self.lib_suffix = '.%s' % self.substs['LIB_SUFFIX']
self.dll_prefix = self.substs.get('DLL_PREFIX', '')
self.dll_suffix = self.substs.get('DLL_SUFFIX', '')
if self.substs.get('IMPORT_LIB_SUFFIX'):
self.import_prefix = self.lib_prefix
self.import_suffix = '.%s' % self.substs['IMPORT_LIB_SUFFIX']
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -68,50 +68,49 @@ class ObjdirMismatchException(BadEnviron
class MozbuildObject(ProcessExecutionMixin):
"""Base class providing basic functionality useful to many modules.
Modules in this package typically require common functionality such as
accessing the current config, getting the location of the source directory,
running processes, etc. This classes provides that functionality. Other
modules can inherit from this class to obtain this functionality easily.
"""
- def __init__(self, topsrcdir, settings, log_manager, topobjdir=None):
+ def __init__(self, topsrcdir, settings, log_manager, topobjdir=None,
+ mozconfig=MozconfigLoader.AUTODETECT):
"""Create a new Mozbuild object instance.
Instances are bound to a source directory, a ConfigSettings instance,
and a LogManager instance. The topobjdir may be passed in as well. If
it isn't, it will be calculated from the active mozconfig.
"""
self.topsrcdir = mozpath.normsep(topsrcdir)
self.settings = settings
self.populate_logger()
self.log_manager = log_manager
self._make = None
self._topobjdir = mozpath.normsep(topobjdir) if topobjdir else topobjdir
- self._mozconfig = None
+ self._mozconfig = mozconfig
self._config_guess_output = None
self._config_environment = None
self._virtualenv_manager = None
@classmethod
def from_environment(cls, cwd=None, detect_virtualenv_mozinfo=True):
"""Create a MozbuildObject by detecting the proper one from the env.
This examines environment state like the current working directory and
creates a MozbuildObject from the found source directory, mozconfig, etc.
The role of this function is to identify a topsrcdir, topobjdir, and
mozconfig file.
If the current working directory is inside a known objdir, we always
- use the topsrcdir and mozconfig associated with that objdir. If no
- mozconfig is associated with that objdir, we fall back to looking for
- the mozconfig in the usual places.
+ use the topsrcdir and mozconfig associated with that objdir.
If the current working directory is inside a known srcdir, we use that
topsrcdir and look for mozconfigs using the default mechanism, which
looks inside environment variables.
If the current Python interpreter is running from a virtualenv inside
an objdir, we use that as our objdir.
@@ -121,17 +120,17 @@ class MozbuildObject(ProcessExecutionMix
mozinfo.json file relative to the virtualenv directory. This was
added to facilitate testing. Callers likely shouldn't change the
default.
"""
cwd = cwd or os.getcwd()
topsrcdir = None
topobjdir = None
- mozconfig = None
+ mozconfig = MozconfigLoader.AUTODETECT
def load_mozinfo(path):
info = json.load(open(path, 'rt'))
topsrcdir = info.get('topsrcdir')
topobjdir = os.path.dirname(path)
mozconfig = info.get('mozconfig')
return topsrcdir, topobjdir, mozconfig
@@ -191,17 +190,18 @@ class MozbuildObject(ProcessExecutionMix
if topsrcdir == topobjdir:
raise BadEnvironmentException('The object directory appears '
'to be the same as your source directory (%s). This build '
'configuration is not supported.' % topsrcdir)
# If we can't resolve topobjdir, oh well. The constructor will figure
# it out via config.guess.
- return cls(topsrcdir, None, None, topobjdir=topobjdir)
+ return cls(topsrcdir, None, None, topobjdir=topobjdir,
+ mozconfig=mozconfig)
@staticmethod
def resolve_mozconfig_topobjdir(topsrcdir, mozconfig, default=None):
topobjdir = mozconfig['topobjdir'] or default
if not topobjdir:
return None
if '@CONFIG_GUESS@' in topobjdir:
@@ -232,19 +232,19 @@ class MozbuildObject(ProcessExecutionMix
return self._virtualenv_manager
@property
def mozconfig(self):
"""Returns information about the current mozconfig file.
This a dict as returned by MozconfigLoader.read_mozconfig()
"""
- if self._mozconfig is None:
+ if self._mozconfig is MozconfigLoader.AUTODETECT:
loader = MozconfigLoader(self.topsrcdir)
- self._mozconfig = loader.read_mozconfig(
+ self._mozconfig = loader.read_mozconfig(path=self._mozconfig,
moz_build_app=os.environ.get('MOZ_CURRENT_PROJECT'))
return self._mozconfig
@property
def config_environment(self):
"""Returns the ConfigEnvironment for the current build configuration.
--- a/python/mozbuild/mozbuild/config_status.py
+++ b/python/mozbuild/mozbuild/config_status.py
@@ -57,17 +57,18 @@ files by running:
mach build-backend --backend=VisualStudio
===============================
'''.strip()
def config_status(topobjdir='.', topsrcdir='.', defines=None,
- non_global_defines=None, substs=None, source=None):
+ non_global_defines=None, substs=None, source=None,
+ mozconfig=None):
'''Main function, providing config.status functionality.
Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS
variables.
Without the -n option, this program acts as config.status and considers
the current directory as the top object directory, even when config.status
is in a different directory. It will, however, treat the directory
@@ -108,17 +109,18 @@ def config_status(topobjdir='.', topsrcd
help='do everything except writing files out.')
options = parser.parse_args()
# Without -n, the current directory is meant to be the top object directory
if not options.not_topobjdir:
topobjdir = os.path.abspath('.')
env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
- non_global_defines=non_global_defines, substs=substs, source=source)
+ non_global_defines=non_global_defines, substs=substs,
+ source=source, mozconfig=mozconfig)
# mozinfo.json only needs written if configure changes and configure always
# passes this environment variable.
if 'WRITE_MOZINFO' in os.environ:
write_mozinfo(os.path.join(topobjdir, 'mozinfo.json'), env, os.environ)
cpu_start = time.clock()
time_start = time.time()
--- a/python/mozbuild/mozbuild/mozconfig.py
+++ b/python/mozbuild/mozbuild/mozconfig.py
@@ -75,16 +75,18 @@ class MozconfigLoader(object):
DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh')
IGNORE_SHELL_VARIABLES = {'_'}
ENVIRONMENT_VARIABLES = {
'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'MOZ_OBJDIR',
}
+ AUTODETECT = object()
+
def __init__(self, topsrcdir):
self.topsrcdir = topsrcdir
@property
def _loader_script(self):
our_dir = os.path.abspath(os.path.dirname(__file__))
return os.path.join(our_dir, 'mozconfig_loader')
@@ -185,26 +187,26 @@ class MozconfigLoader(object):
raise MozconfigFindException(
MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir))
return None
def read_mozconfig(self, path=None, moz_build_app=None):
"""Read the contents of a mozconfig into a data structure.
- This takes the path to a mozconfig to load. If it is not defined, we
- will try to find a mozconfig from the environment using
+ This takes the path to a mozconfig to load. If the given path is
+ AUTODETECT, will try to find a mozconfig from the environment using
find_mozconfig().
mozconfig files are shell scripts. So, we can't just parse them.
Instead, we run the shell script in a wrapper which allows us to record
state from execution. Thus, the output from a mozconfig is a friendly
static data structure.
"""
- if path is None:
+ if path is self.AUTODETECT:
path = self.find_mozconfig()
result = {
'path': path,
'topobjdir': None,
'configure_args': None,
'make_flags': None,
'make_extra': None,
--- a/python/mozbuild/mozbuild/mozinfo.py
+++ b/python/mozbuild/mozbuild/mozinfo.py
@@ -5,17 +5,17 @@
# This module produces a JSON file that provides basic build info and
# configuration metadata.
from __future__ import absolute_import
import os
import re
import json
-import mozbuild.mozconfig as mozconfig
+
def build_dict(config, env=os.environ):
"""
Build a dict containing data about the build configuration from
the environment.
"""
substs = config.substs
@@ -24,19 +24,18 @@ def build_dict(config, env=os.environ):
missing = [r for r in required if r not in substs]
if missing:
raise Exception("Missing required environment variables: %s" %
', '.join(missing))
d = {}
d['topsrcdir'] = config.topsrcdir
- the_mozconfig = mozconfig.MozconfigLoader(config.topsrcdir).find_mozconfig(env)
- if the_mozconfig:
- d['mozconfig'] = the_mozconfig
+ if config.mozconfig:
+ d['mozconfig'] = config.mozconfig
# os
o = substs["OS_TARGET"]
known_os = {"Linux": "linux",
"WINNT": "win",
"Darwin": "mac",
"Android": "b2g" if substs.get("MOZ_WIDGET_TOOLKIT") == "gonk" else "android"}
if o in known_os:
--- a/python/mozbuild/mozbuild/test/test_mozinfo.py
+++ b/python/mozbuild/mozbuild/test/test_mozinfo.py
@@ -240,17 +240,18 @@ class TestWriteMozinfo(unittest.TestCase
TARGET_CPU='i386',
MOZ_WIDGET_TOOLKIT='windows',
))
tempdir = tempfile.tempdir
c.topsrcdir = tempdir
with NamedTemporaryFile(dir=os.path.normpath(c.topsrcdir)) as mozconfig:
mozconfig.write('unused contents')
mozconfig.flush()
- write_mozinfo(self.f, c, {'MOZCONFIG': mozconfig.name})
+ c.mozconfig = mozconfig.name
+ write_mozinfo(self.f, c)
with open(self.f) as f:
d = json.load(f)
self.assertEqual('win', d['os'])
self.assertEqual('x86', d['processor'])
self.assertEqual('windows', d['toolkit'])
self.assertEqual(tempdir, d['topsrcdir'])
self.assertEqual(mozconfig.name, d['mozconfig'])
self.assertEqual(32, d['bits'])