Bug 1469067 - Build host programs in their final locations rather than copying them to dist/host/bin. draft
authorChris Manchester <cmanchester@mozilla.com>
Mon, 18 Jun 2018 14:22:20 -0700
changeset 808266 57f11fc363b88afee58d3daa372ccafb7c38d337
parent 808182 9b74b9f2939a7ae3a0ea6e711dc32ed5203e03ff
push id113330
push userbmo:cmanchester@mozilla.com
push dateMon, 18 Jun 2018 22:12:59 +0000
bugs1469067
milestone62.0a1
Bug 1469067 - Build host programs in their final locations rather than copying them to dist/host/bin. MozReview-Commit-ID: BrSou1ee2qV
config/makefiles/target_binaries.mk
config/rules.mk
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/backend/tup.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/test/frontend/data/host-program-paths/final-target/moz.build
python/mozbuild/mozbuild/test/frontend/data/host-program-paths/installed/moz.build
python/mozbuild/mozbuild/test/frontend/data/host-program-paths/moz.build
python/mozbuild/mozbuild/test/frontend/data/host-program-paths/not-installed/moz.build
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/config/makefiles/target_binaries.mk
+++ b/config/makefiles/target_binaries.mk
@@ -17,18 +17,18 @@ endif
 
 ifdef SHARED_LIBRARY
 SHARED_LIBRARY_FILES = $(SHARED_LIBRARY)
 SHARED_LIBRARY_DEST ?= $(FINAL_TARGET)
 SHARED_LIBRARY_TARGET = target
 INSTALL_TARGETS += SHARED_LIBRARY
 endif # SHARED_LIBRARY
 
-ifneq (,$(strip $(HOST_SIMPLE_PROGRAMS)$(HOST_PROGRAM)))
-HOST_PROGRAMS_EXECUTABLES = $(HOST_SIMPLE_PROGRAMS) $(HOST_PROGRAM) $(HOST_RUST_PROGRAMS)
+ifneq (,$(strip $(HOST_SIMPLE_PROGRAMS)))
+HOST_PROGRAMS_EXECUTABLES = $(HOST_SIMPLE_PROGRAMS) $(HOST_RUST_PROGRAMS)
 HOST_PROGRAMS_DEST ?= $(DIST)/host/bin
 HOST_PROGRAMS_TARGET = host
 INSTALL_TARGETS += HOST_PROGRAMS
 endif
 
 endif # !NO_DIST_INSTALL
 
 # EOF
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -570,32 +570,32 @@ endif # WINNT && !GNU_CC
 
 ifdef ENABLE_STRIP
 	$(STRIP) $(STRIP_FLAGS) $@
 endif
 ifdef MOZ_POST_PROGRAM_COMMAND
 	$(MOZ_POST_PROGRAM_COMMAND) $@
 endif
 
-$(HOST_PROGRAM): $(HOST_PROGOBJS) $(HOST_LIBS) $(HOST_EXTRA_DEPS) $(GLOBAL_DEPS)
+$(HOST_PROGRAM): $(HOST_PROGOBJS) $(HOST_LIBS) $(HOST_EXTRA_DEPS) $(GLOBAL_DEPS) $(call mkdir_deps,$(DEPTH)/dist/host/bin)
 	$(REPORT_BUILD)
 ifeq (_WINNT,$(GNU_CC)_$(HOST_OS_ARCH))
 	$(LINKER) -NOLOGO -OUT:$@ -PDB:$(HOST_PDBFILE) $(HOST_OBJS) $(WIN32_EXE_LDFLAGS) $(HOST_LDFLAGS) $(HOST_LIBS) $(HOST_EXTRA_LIBS)
 ifdef MSMANIFEST_TOOL
 	@if test -f $@.manifest; then \
-		if test -f '$(srcdir)/$@.manifest'; then \
-			echo 'Embedding manifest from $(srcdir)/$@.manifest and $@.manifest'; \
-			$(MT) -NOLOGO -MANIFEST '$(win_srcdir)/$@.manifest' $@.manifest -OUTPUTRESOURCE:$@\;1; \
+		if test -f '$(srcdir)/$(notdir $@).manifest'; then \
+			echo 'Embedding manifest from $(srcdir)/$(notdir $@).manifest and $@.manifest'; \
+			$(MT) -NOLOGO -MANIFEST '$(win_srcdir)/$(notdir $@).manifest' $@.manifest -OUTPUTRESOURCE:$@\;1; \
 		else \
 			echo 'Embedding manifest from $@.manifest'; \
 			$(MT) -NOLOGO -MANIFEST $@.manifest -OUTPUTRESOURCE:$@\;1; \
 		fi; \
-	elif test -f '$(srcdir)/$@.manifest'; then \
-		echo 'Embedding manifest from $(srcdir)/$@.manifest'; \
-		$(MT) -NOLOGO -MANIFEST '$(win_srcdir)/$@.manifest' -OUTPUTRESOURCE:$@\;1; \
+	elif test -f '$(srcdir)/$(notdir $@).manifest'; then \
+		echo 'Embedding manifest from $(srcdir)/$(notdir $@).manifest'; \
+		$(MT) -NOLOGO -MANIFEST '$(win_srcdir)/$(notdir $@).manifest' -OUTPUTRESOURCE:$@\;1; \
 	fi
 endif	# MSVC with manifest tool
 else
 ifeq ($(HOST_CPP_PROG_LINK),1)
 	$(HOST_CXX) -o $@ $(HOST_CXX_LDFLAGS) $(HOST_LDFLAGS) $(HOST_PROGOBJS) $(HOST_LIBS) $(HOST_EXTRA_LIBS)
 else
 	$(HOST_CC) -o $@ $(HOST_C_LDFLAGS) $(HOST_LDFLAGS) $(HOST_PROGOBJS) $(HOST_LIBS) $(HOST_EXTRA_LIBS)
 endif # HOST_CPP_PROG_LINK
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -628,17 +628,17 @@ class RecursiveMakeBackend(CommonBackend
             self._process_rust_tests(obj, backend_file)
 
         elif isinstance(obj, Program):
             self._process_program(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
             self._no_skip['syms'].add(backend_file.relobjdir)
 
         elif isinstance(obj, HostProgram):
-            self._process_host_program(obj.program, backend_file)
+            self._process_host_program(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
 
         elif isinstance(obj, SimpleProgram):
             self._process_simple_program(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
             self._no_skip['syms'].add(backend_file.relobjdir)
 
         elif isinstance(obj, HostSimpleProgram):
@@ -1101,17 +1101,18 @@ class RecursiveMakeBackend(CommonBackend
         ))
 
     def _process_program(self, obj, backend_file):
         backend_file.write('PROGRAM = %s\n' % self._pretty_path(obj.output_path, backend_file))
         if not obj.cxx_link and not self.environment.bin_suffix:
             backend_file.write('PROG_IS_C_ONLY_%s := 1\n' % obj.program)
 
     def _process_host_program(self, program, backend_file):
-        backend_file.write('HOST_PROGRAM = %s\n' % program)
+        backend_file.write('HOST_PROGRAM = %s\n' %
+                           self._pretty_path(program.output_path, backend_file))
 
     def _process_rust_program_base(self, obj, backend_file,
                                    target_variable,
                                    target_cargo_variable):
         backend_file.write_once('CARGO_FILE := %s\n' % obj.cargo_file)
         backend_file.write_once('CARGO_TARGET_DIR := .\n')
         backend_file.write('%s += %s\n' % (target_variable, obj.location))
         backend_file.write('%s += %s\n' % (target_cargo_variable, obj.name))
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -458,17 +458,22 @@ class TupBackend(CommonBackend):
     def _gen_host_programs(self, backend_file):
         for p in backend_file.host_programs:
             self._gen_host_program(backend_file, p)
 
 
     def _gen_host_program(self, backend_file, prog):
         _, _, _, extra_libs, _ = self._expand_libs(prog)
         objs = prog.objs
-        outputs = [prog.program]
+
+        if isinstance(prog, HostSimpleProgram):
+            outputs = [prog.name]
+        else:
+            outputs = [mozpath.relpath(prog.output_path.full_path,
+                                       backend_file.objdir)]
         host_libs = []
         for lib in prog.linked_libraries:
             if isinstance(lib, HostLibrary):
                 host_libs.append(lib)
         host_libs = self._lib_paths(backend_file.objdir, host_libs)
 
         inputs = objs + host_libs
         use_cxx = any(f.endswith(('.cc', '.cpp')) for f in prog.source_files())
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -507,16 +507,20 @@ class Program(BaseProgram):
     KIND = 'target'
 
 
 class HostProgram(HostMixin, BaseProgram):
     """Context derived container object for HOST_PROGRAM"""
     SUFFIX_VAR = 'HOST_BIN_SUFFIX'
     KIND = 'host'
 
+    @property
+    def install_target(self):
+        return 'dist/host/bin'
+
 
 class SimpleProgram(BaseProgram):
     """Context derived container object for each program in SIMPLE_PROGRAMS"""
     SUFFIX_VAR = 'BIN_SUFFIX'
     KIND = 'target'
 
     def source_files(self):
         for srcs in self.sources.values():
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/host-program-paths/final-target/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+FINAL_TARGET = 'final/target'
+HostProgram('final-target')
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/host-program-paths/installed/moz.build
@@ -0,0 +1,4 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+HostProgram('dist-host-bin')
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/host-program-paths/moz.build
@@ -0,0 +1,12 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+@template
+def HostProgram(name):
+    HOST_PROGRAM = name
+
+DIRS += [
+    'final-target',
+    'installed',
+    'not-installed',
+]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/host-program-paths/not-installed/moz.build
@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+DIST_INSTALL = False
+HostProgram('not-installed')
\ No newline at end of file
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -19,16 +19,17 @@ from mozbuild.frontend.data import (
     ConfigFileSubstitution,
     Defines,
     DirectoryTraversal,
     Exports,
     FinalTargetPreprocessedFiles,
     GeneratedFile,
     GeneratedSources,
     HostDefines,
+    HostProgram,
     HostRustLibrary,
     HostRustProgram,
     HostSources,
     IPDLCollection,
     JARManifest,
     LinkageMultipleRustLibrariesError,
     LocalInclude,
     LocalizedFiles,
@@ -70,16 +71,17 @@ class TestEmitterBasic(unittest.TestCase
     def tearDown(self):
         os.environ.clear()
         os.environ.update(self._old_env)
 
     def reader(self, name, enable_tests=False, extra_substs=None):
         substs = dict(
             ENABLE_TESTS='1' if enable_tests else '',
             BIN_SUFFIX='.prog',
+            HOST_BIN_SUFFIX='.hostprog',
             OS_TARGET='WINNT',
             COMPILE_ENVIRONMENT='1',
             STL_FLAGS=['-I/path/to/topobjdir/dist/stl_wrappers'],
             VISIBILITY_FLAGS=['-include',
                               '$(topsrcdir)/config/gcc_hidden.h'],
             OBJ_SUFFIX='obj',
         )
         if extra_substs:
@@ -694,16 +696,28 @@ class TestEmitterBasic(unittest.TestCase
         prog_paths = [o.output_path for o in objs if isinstance(o, Program)]
         self.assertEqual(prog_paths, [
             '!/dist/bin/dist-bin.prog',
             '!/dist/bin/foo/dist-subdir.prog',
             '!/final/target/final-target.prog',
             '!not-installed.prog',
         ])
 
+    def test_host_program_paths(self):
+        """The destination of a HOST_PROGRAM (almost always dist/host/bin)
+        should be accurately reflected in Program.output_path."""
+        reader = self.reader('host-program-paths')
+        objs = self.read_topsrcdir(reader)
+        prog_paths = [o.output_path for o in objs if isinstance(o, HostProgram)]
+        self.assertEqual(prog_paths, [
+            '!/dist/host/bin/final-target.hostprog',
+            '!/dist/host/bin/dist-host-bin.hostprog',
+            '!not-installed.hostprog',
+        ])
+
     def test_test_manifest_missing_manifest(self):
         """A missing manifest file should result in an error."""
         reader = self.reader('test-manifest-missing-manifest')
 
         with self.assertRaisesRegexp(BuildReaderError, 'IOError: Missing files'):
             self.read_topsrcdir(reader)
 
     def test_empty_test_manifest_rejected(self):