Bug 1256571 - Change the execution model of python configure. r?chmanchester draft
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 12 Apr 2016 21:32:38 +0900
changeset 350232 821a498713ec4292af22b3b5402782b3d5b7d892
parent 350231 b30bf73443fb2b957846852a7c96ef1d6509a7b6
child 518273 5ed98bebc114bc3a4ae1e3b6f3eb0589d17f38bb
push id15274
push userbmo:mh+mozilla@glandium.org
push dateWed, 13 Apr 2016 01:01:28 +0000
reviewerschmanchester
bugs1256571
milestone48.0a1
Bug 1256571 - Change the execution model of python configure. r?chmanchester So far, everything was essentially executed at "declaration". This made the sandbox code simpler, but to improve on the tooling around python configure (for tests and introspection), we need to have more flexibility, which executing everything at declaration doesn't give. With this change, only @depends functions depending on --help, as well as templates, are executed at the moment the moz.configure files are included in the sandbox. The remainder is executed at the end.
build/moz.configure/init.configure
python/mozbuild/mozbuild/configure/__init__.py
python/mozbuild/mozbuild/test/configure/test_checks_configure.py
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -153,25 +153,28 @@ def add_old_configure_arg(arg):
         if arg:
             args.append(arg)
 
 
 option(env='PYTHON', nargs=1, help='Python interpreter')
 
 # Setup python virtualenv
 # ==============================================================
-@depends('PYTHON', check_build_environment, mozconfig)
+@depends('PYTHON', check_build_environment, mozconfig, '--help')
 @imports('os')
 @imports('sys')
 @imports('subprocess')
 @imports(_from='mozbuild.configure.util', _import='LineIO')
 @imports(_from='mozbuild.virtualenv', _import='VirtualenvManager')
 @imports(_from='mozbuild.virtualenv', _import='verify_python_version')
 @imports('distutils.sysconfig')
-def virtualenv_python(env_python, build_env, mozconfig):
+def virtualenv_python(env_python, build_env, mozconfig, help):
+    if help:
+        return
+
     python = env_python[0] if env_python else None
 
     # Ideally we'd rely on the mozconfig injection from mozconfig_options,
     # but we'd rather avoid the verbosity when we need to reexecute with
     # a different python.
     if mozconfig['path']:
         if 'PYTHON' in mozconfig['env']['added']:
             python = mozconfig['env']['added']['PYTHON']
--- a/python/mozbuild/mozbuild/configure/__init__.py
+++ b/python/mozbuild/mozbuild/configure/__init__.py
@@ -124,16 +124,19 @@ class ConfigureSandbox(dict):
         # Store options added with `imply_option`, and the reason they were
         # added (which can either have been given to `imply_option`, or
         # inferred.
         self._implied_options = []
 
         # Store all results from _prepare_function
         self._prepared_functions = set()
 
+        # Queue of functions to execute, with their arguments
+        self._execution_queue = []
+
         self._helper = CommandLineHelper(environ, argv)
 
         assert isinstance(config, dict)
         self._config = config
 
         if logger is None:
             logger = moz_logger = logging.getLogger('moz.configure')
             logger.setLevel(logging.DEBUG)
@@ -166,17 +169,20 @@ class ConfigureSandbox(dict):
             self._help.add(self._help_option)
         elif moz_logger:
             handler = logging.FileHandler('config.log', mode='w', delay=True)
             handler.setFormatter(formatter)
             logger.addHandler(handler)
 
     def include_file(self, path):
         '''Include one file in the sandbox. Users of this class probably want
-        to use `run` instead.'''
+
+        Note: this will execute all template invocations, as well as @depends
+        functions that depend on '--help', but nothing else.
+        '''
 
         if self._paths:
             path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
             path = mozpath.normpath(path)
             if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
                 raise ConfigureError(
                     'Cannot include `%s` because it is not in a subdirectory '
                     'of `%s`' % (path, mozpath.dirname(self._paths[0])))
@@ -192,48 +198,48 @@ class ConfigureSandbox(dict):
 
         code = compile(source, path, 'exec')
 
         exec(code, self)
 
         self._paths.pop(-1)
 
     def run(self, path=None):
-        '''Executes the given file within the sandbox, and ensure the overall
-        consistency of the executed script.'''
+        '''Executes the given file within the sandbox, as well as everything
+        pending from any other included file, and ensure the overall
+        consistency of the executed script(s).'''
         if path:
             self.include_file(path)
 
         for option in self._options.itervalues():
             # All options must be referenced by some @depends function
             if option not in self._seen:
                 raise ConfigureError(
                     'Option `%s` is not handled ; reference it with a @depends'
                     % option.option
                 )
 
-            # When running with --help, few options are handled but we still
-            # want to find the unknown ones below, so handle them all now. We
-            # however don't run any of the @depends function that depend on
-            # them.
-            if self._help:
-                self._helper.handle(option)
+            self._value_for(option)
 
         # All implied options should exist.
         for implied_option in self._implied_options:
             raise ConfigureError(
                 '`%s`, emitted from `%s` line %d, is unknown.'
                 % (implied_option.option, implied_option.caller[1],
                    implied_option.caller[2]))
 
         # All options should have been removed (handled) by now.
         for arg in self._helper:
             without_value = arg.split('=', 1)[0]
             raise InvalidOptionError('Unknown option: %s' % without_value)
 
+        # Run the execution queue
+        for func, args in self._execution_queue:
+            func(*args)
+
         if self._help:
             with LineIO(self.log_impl.info) as out:
                 self._help.usage(out)
 
     def __getitem__(self, key):
         impl = '%s_impl' % key
         func = getattr(self, impl, None)
         if func:
@@ -286,16 +292,19 @@ class ConfigureSandbox(dict):
             for arg in dependencies:
                 if isinstance(arg, DependsFunction):
                     _, deps = self._depends[arg]
                     if self._help_option not in deps:
                         raise ConfigureError(
                             "`%s` depends on '--help' and `%s`. "
                             "`%s` must depend on '--help'"
                             % (func.__name__, arg.__name__, arg.__name__))
+        elif self._help:
+            raise ConfigureError("Missing @depends for `%s`: '--help'" %
+                                 func.__name__)
 
         resolved_args = [self._value_for(d) for d in dependencies]
         return func(*resolved_args)
 
     @memoize
     def _value_for_option(self, option):
         implied = {}
         for implied_option in self._implied_options[:]:
@@ -358,18 +367,16 @@ class ConfigureSandbox(dict):
         if option.name:
             self._options[option.name] = option
         if option.env:
             self._options[option.env] = option
 
         if self._help:
             self._help.add(option)
 
-        self._value_for(option)
-
         return option
 
     def depends_impl(self, *args):
         '''Implementation of @depends()
         This function is a decorator. It returns a function that subsequently
         takes a function and returns a dummy function. The dummy function
         identifies the actual function for the sandbox, while preventing
         further function calls from within the sandbox.
@@ -412,18 +419,24 @@ class ConfigureSandbox(dict):
 
         def decorator(func):
             if inspect.isgeneratorfunction(func):
                 raise ConfigureError(
                     'Cannot decorate generator functions with @depends')
             func, glob = self._prepare_function(func)
             dummy = wraps(func)(DependsFunction())
             self._depends[dummy] = func, dependencies
-            if not self._help or self._help_option in dependencies:
+
+            # Only @depends functions with a dependency on '--help' are
+            # executed immediately. Everything else is queued for later
+            # execution.
+            if self._help_option in dependencies:
                 self._value_for(dummy)
+            elif not self._help:
+                self._execution_queue.append((self._value_for, (dummy,)))
 
             return dummy
 
         return decorator
 
     def include_impl(self, what):
         '''Implementation of include().
         Allows to include external files for execution in the sandbox.
@@ -567,28 +580,30 @@ class ConfigureSandbox(dict):
 
     def set_config_impl(self, name, value):
         '''Implementation of set_config().
         Set the configuration items with the given name to the given value.
         Both `name` and `value` can be references to @depends functions,
         in which case the result from these functions is used. If the result
         of either function is None, the configuration item is not set.
         '''
-        self._resolve_and_set(self._config, name, value)
+        self._execution_queue.append((
+            self._resolve_and_set, (self._config, name, value)))
 
     def set_define_impl(self, name, value):
         '''Implementation of set_define().
         Set the define with the given name to the given value. Both `name` and
         `value` can be references to @depends functions, in which case the
         result from these functions is used. If the result of either function
         is None, the define is not set. If the result is False, the define is
         explicitly undefined (-U).
         '''
         defines = self._config.setdefault('DEFINES', {})
-        self._resolve_and_set(defines, name, value)
+        self._execution_queue.append((
+            self._resolve_and_set, (defines, name, value)))
 
     def imply_option_impl(self, option, value, reason=None):
         '''Implementation of imply_option().
         Injects additional options as if they had been passed on the command
         line. The `option` argument is a string as in option()'s `name` or
         `env`. The option must be declared after `imply_option` references it.
         The `value` argument indicates the value to pass to the option.
         It can be:
--- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
@@ -159,16 +159,17 @@ class TestChecksConfigure(unittest.TestC
         sandbox = FindProgramSandbox(config, environ, [prog] + args, out, out)
         base_dir = os.path.join(topsrcdir, 'build', 'moz.configure')
         sandbox.include_file(os.path.join(base_dir, 'util.configure'))
         sandbox.include_file(os.path.join(base_dir, 'checks.configure'))
 
         status = 0
         try:
             exec(command, sandbox)
+            sandbox.run()
         except SystemExit as e:
             status = e.code
 
         return config, out.getvalue(), status
 
     def test_check_prog(self):
         config, out, status = self.get_result(
             'check_prog("FOO", ("known-a",))')