Bug 1256571 - Change the execution model of python configure. r?chmanchester
So far, everything was essentially executed at "declaration". This
made the sandbox code simpler, but to improve on the tooling around
python configure (for tests and introspection), we need to have more
flexibility, which executing everything at declaration doesn't give.
With this change, only @depends functions depending on --help, as
well as templates, are executed at the moment the moz.configure
files are included in the sandbox. The remainder is executed at the
end.
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -153,25 +153,28 @@ def add_old_configure_arg(arg):
if arg:
args.append(arg)
option(env='PYTHON', nargs=1, help='Python interpreter')
# Setup python virtualenv
# ==============================================================
-@depends('PYTHON', check_build_environment, mozconfig)
+@depends('PYTHON', check_build_environment, mozconfig, '--help')
@imports('os')
@imports('sys')
@imports('subprocess')
@imports(_from='mozbuild.configure.util', _import='LineIO')
@imports(_from='mozbuild.virtualenv', _import='VirtualenvManager')
@imports(_from='mozbuild.virtualenv', _import='verify_python_version')
@imports('distutils.sysconfig')
-def virtualenv_python(env_python, build_env, mozconfig):
+def virtualenv_python(env_python, build_env, mozconfig, help):
+ if help:
+ return
+
python = env_python[0] if env_python else None
# Ideally we'd rely on the mozconfig injection from mozconfig_options,
# but we'd rather avoid the verbosity when we need to reexecute with
# a different python.
if mozconfig['path']:
if 'PYTHON' in mozconfig['env']['added']:
python = mozconfig['env']['added']['PYTHON']
--- a/python/mozbuild/mozbuild/configure/__init__.py
+++ b/python/mozbuild/mozbuild/configure/__init__.py
@@ -124,16 +124,19 @@ class ConfigureSandbox(dict):
# Store options added with `imply_option`, and the reason they were
# added (which can either have been given to `imply_option`, or
# inferred.
self._implied_options = []
# Store all results from _prepare_function
self._prepared_functions = set()
+ # Queue of functions to execute, with their arguments
+ self._execution_queue = []
+
self._helper = CommandLineHelper(environ, argv)
assert isinstance(config, dict)
self._config = config
if logger is None:
logger = moz_logger = logging.getLogger('moz.configure')
logger.setLevel(logging.DEBUG)
@@ -166,17 +169,20 @@ class ConfigureSandbox(dict):
self._help.add(self._help_option)
elif moz_logger:
handler = logging.FileHandler('config.log', mode='w', delay=True)
handler.setFormatter(formatter)
logger.addHandler(handler)
def include_file(self, path):
'''Include one file in the sandbox. Users of this class probably want
- to use `run` instead.'''
+
+ Note: this will execute all template invocations, as well as @depends
+ functions that depend on '--help', but nothing else.
+ '''
if self._paths:
path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
path = mozpath.normpath(path)
if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
raise ConfigureError(
'Cannot include `%s` because it is not in a subdirectory '
'of `%s`' % (path, mozpath.dirname(self._paths[0])))
@@ -192,48 +198,48 @@ class ConfigureSandbox(dict):
code = compile(source, path, 'exec')
exec(code, self)
self._paths.pop(-1)
def run(self, path=None):
- '''Executes the given file within the sandbox, and ensure the overall
- consistency of the executed script.'''
+ '''Executes the given file within the sandbox, as well as everything
+ pending from any other included file, and ensure the overall
+ consistency of the executed script(s).'''
if path:
self.include_file(path)
for option in self._options.itervalues():
# All options must be referenced by some @depends function
if option not in self._seen:
raise ConfigureError(
'Option `%s` is not handled ; reference it with a @depends'
% option.option
)
- # When running with --help, few options are handled but we still
- # want to find the unknown ones below, so handle them all now. We
- # however don't run any of the @depends function that depend on
- # them.
- if self._help:
- self._helper.handle(option)
+ self._value_for(option)
# All implied options should exist.
for implied_option in self._implied_options:
raise ConfigureError(
'`%s`, emitted from `%s` line %d, is unknown.'
% (implied_option.option, implied_option.caller[1],
implied_option.caller[2]))
# All options should have been removed (handled) by now.
for arg in self._helper:
without_value = arg.split('=', 1)[0]
raise InvalidOptionError('Unknown option: %s' % without_value)
+ # Run the execution queue
+ for func, args in self._execution_queue:
+ func(*args)
+
if self._help:
with LineIO(self.log_impl.info) as out:
self._help.usage(out)
def __getitem__(self, key):
impl = '%s_impl' % key
func = getattr(self, impl, None)
if func:
@@ -286,16 +292,19 @@ class ConfigureSandbox(dict):
for arg in dependencies:
if isinstance(arg, DependsFunction):
_, deps = self._depends[arg]
if self._help_option not in deps:
raise ConfigureError(
"`%s` depends on '--help' and `%s`. "
"`%s` must depend on '--help'"
% (func.__name__, arg.__name__, arg.__name__))
+ elif self._help:
+ raise ConfigureError("Missing @depends for `%s`: '--help'" %
+ func.__name__)
resolved_args = [self._value_for(d) for d in dependencies]
return func(*resolved_args)
@memoize
def _value_for_option(self, option):
implied = {}
for implied_option in self._implied_options[:]:
@@ -358,18 +367,16 @@ class ConfigureSandbox(dict):
if option.name:
self._options[option.name] = option
if option.env:
self._options[option.env] = option
if self._help:
self._help.add(option)
- self._value_for(option)
-
return option
def depends_impl(self, *args):
'''Implementation of @depends()
This function is a decorator. It returns a function that subsequently
takes a function and returns a dummy function. The dummy function
identifies the actual function for the sandbox, while preventing
further function calls from within the sandbox.
@@ -412,18 +419,24 @@ class ConfigureSandbox(dict):
def decorator(func):
if inspect.isgeneratorfunction(func):
raise ConfigureError(
'Cannot decorate generator functions with @depends')
func, glob = self._prepare_function(func)
dummy = wraps(func)(DependsFunction())
self._depends[dummy] = func, dependencies
- if not self._help or self._help_option in dependencies:
+
+ # Only @depends functions with a dependency on '--help' are
+ # executed immediately. Everything else is queued for later
+ # execution.
+ if self._help_option in dependencies:
self._value_for(dummy)
+ elif not self._help:
+ self._execution_queue.append((self._value_for, (dummy,)))
return dummy
return decorator
def include_impl(self, what):
'''Implementation of include().
Allows to include external files for execution in the sandbox.
@@ -567,28 +580,30 @@ class ConfigureSandbox(dict):
def set_config_impl(self, name, value):
'''Implementation of set_config().
Set the configuration items with the given name to the given value.
Both `name` and `value` can be references to @depends functions,
in which case the result from these functions is used. If the result
of either function is None, the configuration item is not set.
'''
- self._resolve_and_set(self._config, name, value)
+ self._execution_queue.append((
+ self._resolve_and_set, (self._config, name, value)))
def set_define_impl(self, name, value):
'''Implementation of set_define().
Set the define with the given name to the given value. Both `name` and
`value` can be references to @depends functions, in which case the
result from these functions is used. If the result of either function
is None, the define is not set. If the result is False, the define is
explicitly undefined (-U).
'''
defines = self._config.setdefault('DEFINES', {})
- self._resolve_and_set(defines, name, value)
+ self._execution_queue.append((
+ self._resolve_and_set, (defines, name, value)))
def imply_option_impl(self, option, value, reason=None):
'''Implementation of imply_option().
Injects additional options as if they had been passed on the command
line. The `option` argument is a string as in option()'s `name` or
`env`. The option must be declared after `imply_option` references it.
The `value` argument indicates the value to pass to the option.
It can be:
--- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
@@ -159,16 +159,17 @@ class TestChecksConfigure(unittest.TestC
sandbox = FindProgramSandbox(config, environ, [prog] + args, out, out)
base_dir = os.path.join(topsrcdir, 'build', 'moz.configure')
sandbox.include_file(os.path.join(base_dir, 'util.configure'))
sandbox.include_file(os.path.join(base_dir, 'checks.configure'))
status = 0
try:
exec(command, sandbox)
+ sandbox.run()
except SystemExit as e:
status = e.code
return config, out.getvalue(), status
def test_check_prog(self):
config, out, status = self.get_result(
'check_prog("FOO", ("known-a",))')