Bug 1417684 - Reimplement make detection in moz.configure; r?build
Previously, moz.configure assembled a list of possible make binary
names and did a check_prog, taking the first one that resolved. This
logic was inadequate and littered with bugs.
Until very recently, moz.configure was always invoked with client.mk.
And client.mk exposed MAKE to the configure invocation. And this MAKE
was derived by Python code in mozbuild that knows how to find a
suitable make. So even though moz.configure had logic for resolving
MAKE, chances are that mozbuild was really doing all the work.
A bug in the old implementation was that the MAKE environment
variable was supplemented by other candidate binaries. Generally
speaking, if a variable is passed into configure, it should be
used explicitly. But the previous code didn't do that. The new code
only looks at MAKE if it is defined.
The old code was also failing to look for mozmake on Windows. We
actually require mozmake on Windows. Basically, the previous code
was relying on the fact that MAKE was defined and pointing to
mozmake for it to work. If we were to undefine MAKE and invoke the
old code, builds on Windows would break.
The old code was also not validating the make binary. The Python
code in base.py for locating a make binary (and client.mk until
recently) invokes baseconfig.mk to validate the make binary. So,
it was possible for the old moz.configure code to use a make binary
that can't actually be used with the build system.
The new moz.configure code is much more robust.
If MAKE is defined, it only looks at that binary. Otherwise, it
falls back to searching for various make binary names. The search
order is similar to base.py. The lone exception is mozmake is last
instead of between "make" and "gnumake." Since base.py's logic has
been invoking client.mk and that make becomes MAKE and is used by
moz.configure, I'm not anticipating any significant change in
behavior.
We also validate the make binary in moz.configure. The logic is
very similar to that in base.py. Basically, we invoke baseconfig.mk
(which does most validation) and special case the Xcode license
check failure. Eventually, we can likely move more logic from
baseconfig.mk into moz.configure. And once we no longer run
client.mk to invoke the build system, we can remove the "find make"
code from base.py and drive everything from moz.configure. But
these are for other commits.
To prove the new code works, we stop passing the mozbuild derived
MAKE into configure. moz.configure (or the mozconfig) is now in
the driver's seat for finding a make binary.
We still may not actually use the make found by moz.configure for
invoking client.mk or the build backend. But this was always a
problem (at least for `mach build`). We'll need to fix this
in subsequent commits.
MozReview-Commit-ID: 1qEdWSkdmGf
--- a/moz.configure
+++ b/moz.configure
@@ -350,28 +350,87 @@ def perl_version_check(min_version):
perl_version_check('5.006')
# GNU make detection
# ==============================================================
option(env='MAKE', nargs=1, help='Path to GNU make')
-@depends('MAKE', host)
-def possible_makes(make, host):
- candidates = []
+@template
+@imports('subprocess')
+def make_info():
+ def validate_make(env, path):
+ prog = find_program(path)
+
+ if not prog:
+ return None, False
+
+ baseconfig_mk = os.path.join(env.topsrcdir, 'config', 'baseconfig.mk')
+
+ cmd = [prog, '-f', baseconfig_mk]
+ if host.kernel == 'WINNT':
+ cmd.append('HOST_OS_ARCH=WINNT')
+
+ xcode_license_error = False
+
+ try:
+ log.debug('Executing %s' % cmd)
+ out = subprocess.check_output(cmd, cwd=env.topsrcdir,
+ stderr=subprocess.STDOUT)
+ log.debug('Execution success')
+ return prog, False
+ except subprocess.CalledProcessError as e:
+ log.debug('Execution failed: %s' % e.output)
+
+ # The `make` provided by Xcode will emit an error until the Xcode
+ # license terms are accepted. This is the make we want to use on
+ # MacOS but it isn't usable yet. So we need to handle it specially.
+ if 'Agreeing to the Xcode' in out:
+ return prog, True
+ else:
+ return None, False
+
+ return validate_make
+
+
+make_info = make_info()
+
+
+@depends('MAKE', check_build_environment, host)
+@checking('for make', callback=lambda x: x.prog)
+def make_prog(make, env, host):
+ # If MAKE is defined, use it and only it.
if make:
- candidates.append(make[0])
- if host.kernel == 'WINNT':
- candidates.extend(('make', 'gmake'))
+ candidates = [make[0]]
else:
- candidates.extend(('gmake', 'make'))
- return candidates
+ candidates = ['gmake', 'make', 'gnumake']
+
+ # MozillaBuild uses a custom build of GNU make.
+ if host.kernel == 'WINNT':
+ candidates.append('mozmake')
+
+ for candidate in candidates:
+ log.debug('make: trying %s', candidate)
+ prog, xcode_license_error = make_info(env, candidate)
-check_prog('GMAKE', possible_makes)
+ if not prog:
+ continue
+
+ if xcode_license_error:
+ die('Xcode requires an acceptance of its license agreement in '
+ 'order to build. Please run Xcode and accept its license.')
+
+ return namespace(
+ prog=prog,
+ )
+
+ die('no usable make implementation found')
+
+set_config('GMAKE', depends(make_prog)(lambda m: m.prog))
# tup detection
# ==============================================================
@depends(build_backends)
def tup_progs(build_backends):
for backend in build_backends:
if 'Tup' in backend:
return ['tup']
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1098,17 +1098,16 @@ class BuildDriver(MozbuildObject):
# yet.
# TODO we should probably call ensure_configure() here or
# somewhere above.
try:
config = self.config_environment
except Exception:
config_rc = run_configure(self.topsrcdir,
self.topobjdir,
- self._make_path(),
self.run_process,
line_handler=output.on_line)
if config_rc != 0:
return config_rc
config = self.config_environment
active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
@@ -1282,17 +1281,17 @@ class BuildDriver(MozbuildObject):
return 1
def on_line(line):
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
line_handler = line_handler or on_line
status = run_configure(self.topsrcdir, self.topobjdir,
- self._make_path(), self.run_process,
+ self.run_process,
options=options, line_handler=line_handler)
if not status:
print('Configure complete!')
print('Be sure to run |mach build| to pick up any changes');
return status
@@ -1387,17 +1386,17 @@ class BuildDriver(MozbuildObject):
def _run_client_mk(self, target=None, line_handler=None, jobs=0,
verbose=None, keep_going=False, append_env=None):
append_env = dict(append_env or {})
append_env['TOPSRCDIR'] = self.topsrcdir
append_env['CONFIG_GUESS'] = self.resolve_config_guess()
append_env['OBJDIR'] = mozpath.normsep(self.topobjdir)
res = ensure_configure(self.topsrcdir, self.topobjdir, self.log,
- self._make_path(), self.run_process)
+ self.run_process)
configure_ran, configure_exit = res
if configure_exit:
return configure_exit
return self._run_make(srcdir=True,
filename='client.mk',
allow_parallel=False,
--- a/python/mozbuild/mozbuild/controller/configure.py
+++ b/python/mozbuild/mozbuild/controller/configure.py
@@ -146,54 +146,47 @@ def _configure_needed(topsrcdir, topobjd
# The build backend must be strictly newer than all of configure's
# dependencies.
makefile = os.path.join(topobjdir, 'Makefile')
return not is_file_target_current([makefile], list(depends))
-def ensure_configure(topsrcdir, topobjdir, log, make, run_process):
+def ensure_configure(topsrcdir, topobjdir, log, run_process):
"""Ensure configure is up to date, running if necessary.
Returns a 2-tuple of (bool whether ran, int exit code).
"""
# Process configure.in files.
configures = _ensure_configure_files(topsrcdir, log)
if not _configure_needed(topsrcdir, topobjdir, log, configures):
log(logging.INFO, 'configure',
{'msg': 'configure and build backend up to date'},
'{msg}')
return False, None
- return True, run_configure(topsrcdir, topobjdir, make, run_process)
+ return True, run_configure(topsrcdir, topobjdir, run_process)
-def run_configure(topsrcdir, topobjdir, make, run_process, options=None,
+def run_configure(topsrcdir, topobjdir, run_process, options=None,
line_handler=None):
"""Run configure, even if it doesn't need to run.
Returns the integer exit code of configure.
"""
log_name = None if line_handler else 'configure'
options = options or []
configure = mozpath.join(topsrcdir, 'configure')
command = [configure] + options
- append_env = {
- # This is a holdover from client.mk. Remove it once configure's
- # make detection is reasonable.
- 'MAKE': ' '.join(make),
- }
-
res = run_process(args=command,
cwd=topobjdir,
- append_env=append_env,
line_handler=line_handler,
log_name=log_name,
ensure_exit_code=False)
# This is a holdover from client.mk. It should be done from within
# configure/config.status itself.
if not res:
now = time.time()