Bug 1454771 - Move tup invocation from Makefile.in to mach.
MozReview-Commit-ID: HkhK4oe93Vm
--- a/Makefile.in
+++ b/Makefile.in
@@ -144,28 +144,16 @@ install-manifests: $(addprefix install-,
# config/faster/rules.mk)
ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
install-manifests: faster
.PHONY: faster
faster: install-dist/idl
$(MAKE) -C faster FASTER_RECURSIVE_MAKE=1
endif
-.PHONY: tup
-tup:
- $(call BUILDSTATUS,TIERS $(if $(MOZ_ARTIFACT_BUILDS),artifact )tup)
-ifdef MOZ_ARTIFACT_BUILDS
- $(call BUILDSTATUS,TIER_START artifact)
- $(MAKE) recurse_artifact
- $(call BUILDSTATUS,TIER_FINISH artifact)
-endif
- $(call BUILDSTATUS,TIER_START tup)
- @$(TUP) $(if $(MOZ_AUTOMATION),--quiet) $(if $(findstring s,$(filter-out --%,$(MAKEFLAGS))),,--verbose) $(topobjdir)
- $(call BUILDSTATUS,TIER_FINISH tup)
-
.PHONY: $(addprefix install-,$(install_manifests))
$(addprefix install-,$(install_manifests)): install-%: $(install_manifest_depends)
ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
@# If we're using the hybrid FasterMake/RecursiveMake backend, we want
@# to ensure the FasterMake end doesn't have install manifests for the
@# same directory, because that would blow up
$(if $(wildcard _build_manifests/install/$(subst /,_,$*)),$(if $(wildcard faster/install_$(subst /,_,$*)*),$(error FasterMake and RecursiveMake ends of the hybrid build system want to handle $*)))
endif
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -189,17 +189,17 @@ class BuildBackend(LoggingMixin):
This is the main method used by child classes to react to build
metadata.
"""
def consume_finished(self):
"""Called when consume() has completed handling all objects."""
- def build(self, config, output, jobs, verbose):
+ def build(self, config, output, jobs, verbose, what=None):
"""Called when 'mach build' is executed.
This should return the status value of a subprocess, where 0 denotes
success and any other value is an error code. A return value of None
indicates that the default 'make -f client.mk' should run.
"""
return None
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -10,27 +10,29 @@ import sys
import mozpack.path as mozpath
from mozbuild.base import MozbuildObject
from mozbuild.backend.base import PartialBackend, HybridBackend
from mozbuild.backend.recursivemake import RecursiveMakeBackend
from mozbuild.shellutil import quote as shell_quote
from mozbuild.util import OrderedDefaultDict
from collections import defaultdict
+import multiprocessing
from mozpack.files import (
FileFinder,
)
from .common import CommonBackend
from ..frontend.data import (
ChromeManifestEntry,
ComputedFlags,
ContextDerived,
Defines,
+ DirectoryTraversal,
FinalTargetFiles,
FinalTargetPreprocessedFiles,
GeneratedFile,
GeneratedSources,
HostDefines,
HostSources,
JARManifest,
ObjdirFiles,
@@ -187,17 +189,17 @@ class BackendTupfile(object):
return True
return False
@property
def diff(self):
return self.fh.diff
-class TupOnly(CommonBackend, PartialBackend):
+class TupBackend(CommonBackend):
"""Backend that generates Tupfiles for the tup build system.
"""
def _init(self):
CommonBackend._init(self)
self._backend_files = {}
self._cmd = MozbuildObject.from_environment()
@@ -218,16 +220,32 @@ class TupOnly(CommonBackend, PartialBack
# The preprocessor including source-repo.h and buildid.h creates
# dependencies that aren't specified by moz.build and cause errors
# in Tup. Express these as a group dependency.
self._early_generated_files = '$(MOZ_OBJ_ROOT)/<early-generated-files>'
self._built_in_addons = set()
self._built_in_addons_file = 'dist/bin/browser/chrome/browser/content/browser/built_in_addons.json'
+ def build(self, config, output, jobs, verbose, what=None):
+ if not what:
+ what = [self.environment.topobjdir]
+ args = [self.environment.substs['TUP'], 'upd'] + what
+ if self.environment.substs.get('MOZ_AUTOMATION'):
+ args += ['--quiet']
+ if verbose:
+ args += ['--verbose']
+ if jobs > 0:
+ args += ['-j%d' % jobs]
+ else:
+ args += ['-j%d' % multiprocessing.cpu_count()]
+ return config.run_process(args=args,
+ line_handler=output.on_line,
+ ensure_exit_code=False)
+
def _get_backend_file(self, relobjdir):
objdir = mozpath.normpath(mozpath.join(self.environment.topobjdir, relobjdir))
if objdir not in self._backend_files:
self._backend_files[objdir] = \
BackendTupfile(objdir, self.environment,
self.environment.topsrcdir, self.environment.topobjdir)
return self._backend_files[objdir]
@@ -428,22 +446,18 @@ class TupOnly(CommonBackend, PartialBack
elif isinstance(obj, StaticLibrary):
backend_file.static_lib = obj
elif isinstance(obj, SharedLibrary):
backend_file.shared_lib = obj
elif isinstance(obj, HostProgram):
pass
elif isinstance(obj, Program):
backend_file.program = obj
-
- # The top-level Makefile.in still contains our driver target and some
- # things related to artifact builds, so as a special case ensure the
- # make backend generates a Makefile there.
- if obj.objdir == self.environment.topobjdir:
- return False
+ elif isinstance(obj, DirectoryTraversal):
+ pass
return True
def consume_finished(self):
CommonBackend.consume_finished(self)
# The approach here is similar to fastermake.py, but we
# simply write out the resulting files here.
@@ -823,16 +837,8 @@ class TupOnly(CommonBackend, PartialBack
extra_outputs=[self._installed_files],
check_unchanged=True,
)
backend_file.sources['.cpp'].extend(u[0] for u in unified_source_mapping)
backend_file.sources['.cpp'].extend(sorted(global_define_files))
test_backend_file = self._get_backend_file('dom/bindings/test')
test_backend_file.sources['.cpp'].extend(sorted('../%sBinding.cpp' % s for s in webidls.all_test_stems()))
-
-
-class TupBackend(HybridBackend(TupOnly, RecursiveMakeBackend)):
- def build(self, config, output, jobs, verbose):
- status = config._run_make(directory=self.environment.topobjdir, target='tup',
- line_handler=output.on_line, log=False, print_directory=False,
- ensure_exit_code=False, num_jobs=jobs, silent=not verbose)
- return status
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -965,25 +965,74 @@ class BuildDriver(MozbuildObject):
return 1
if directory is not None:
disable_extra_make_dependencies=True
directory = mozpath.normsep(directory)
if directory.startswith('/'):
directory = directory[1:]
- status = None
+ def backend_out_of_date(backend_file):
+ dep_file = '%s.in' % backend_file
+ if not os.path.exists(backend_file):
+ return True
+ if not os.path.exists(dep_file):
+ return True
+
+ dep_files = None
+ with open(dep_file, 'r') as fh:
+ dep_files = fh.read().splitlines()
+ if not dep_files:
+ return True
+
+ mtime = os.path.getmtime(backend_file)
+ for f in dep_files:
+ if os.path.getmtime(f) > mtime:
+ return True
+
+ return False
+
+ def maybe_invoke_backend(active_backend):
+ # Attempt to bypass the make-oriented logic below. Note this
+ # will only succeed in case we're building with a non-make
+ # backend (Tup), and otherwise be harmless.
+ if active_backend:
+ if 'Make' in active_backend:
+ return None
+ if backend_out_of_date(mozpath.join(self.topobjdir,
+ 'backend.%sBackend' %
+ active_backend)):
+ print('Build configuration changed. Regenerating backend.')
+ args = [config.substs['PYTHON'],
+ mozpath.join(self.topobjdir, 'config.status')]
+ self.run_process(args, cwd=self.topobjdir)
+ backend_cls = get_backend_class(active_backend)(config)
+ return backend_cls.build(self, output, jobs, verbose, what)
+ return None
+
monitor.start_resource_recording()
- if what:
- top_make = os.path.join(self.topobjdir, 'Makefile')
- if not os.path.exists(top_make):
- print('Your tree has not been configured yet. Please run '
- '|mach build| with no arguments.')
- return 1
+
+ config = None
+ try:
+ config = self.config_environment
+ except Exception:
+ pass
+ if config is None:
+ config_rc = self.configure(buildstatus_messages=True,
+ line_handler=output.on_line)
+ if config_rc != 0:
+ return config_rc
+
+ config = self.config_environment
+
+ status = maybe_invoke_backend(config.substs.get('BUILD_BACKENDS',
+ [None])[0])
+
+ if what and status is None:
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
if directory is not None:
make_dir = os.path.join(self.topobjdir, directory)
make_target = target
@@ -1042,55 +1091,28 @@ class BuildDriver(MozbuildObject):
status = self._run_make(directory=make_dir, target=make_target,
line_handler=output.on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'},
keep_going=keep_going)
if status != 0:
break
- else:
- # Try to call the default backend's build() method. This will
- # run configure to determine BUILD_BACKENDS if it hasn't run
- # yet.
- config = None
- try:
- config = self.config_environment
- except Exception:
- config_rc = self.configure(buildstatus_messages=True,
- line_handler=output.on_line)
- if config_rc != 0:
- return config_rc
- # Even if configure runs successfully, we may have trouble
- # getting the config_environment for some builds, such as
- # OSX Universal builds. These have to go through client.mk
- # regardless.
- try:
- config = self.config_environment
- except Exception:
- pass
-
- if config:
- active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
- if active_backend:
- backend_cls = get_backend_class(active_backend)(config)
- status = backend_cls.build(self, output, jobs, verbose)
-
+ elif status is None:
# If the backend doesn't specify a build() method, then just
# call client.mk directly.
- if status is None:
- status = self._run_client_mk(line_handler=output.on_line,
- jobs=jobs,
- verbose=verbose,
- keep_going=keep_going)
+ status = self._run_client_mk(line_handler=output.on_line,
+ jobs=jobs,
+ verbose=verbose,
+ keep_going=keep_going)
- self.log(logging.WARNING, 'warning_summary',
- {'count': len(monitor.warnings_database)},
- '{count} compiler warnings present.')
+ self.log(logging.WARNING, 'warning_summary',
+ {'count': len(monitor.warnings_database)},
+ '{count} compiler warnings present.')
# Try to run the active build backend's post-build step, if possible.
try:
config = self.config_environment
active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
if active_backend:
backend_cls = get_backend_class(active_backend)(config)
new_status = backend_cls.post_build(self, output, jobs, verbose, status)