bug 1255485 - build PROGRAMs directly in dist/bin instead of copying them. r?build
Historically we built all our binaries in directories in the objdir, then
symlinked them into dist/bin. Some binaries needed to be copied instead
so that certain relative path lookups work properly, so we resorted to
sprinkling `NSDISTMODE=copy` around Makefiles.
This change makes it so we build PROGRAMs (not any other sort of targets)
directly in dist/bin instead. We could do the same for our other targets
with a little more work.
There were several places in the tree that were copying built binaries to
some other place and needed fixup to match the new location of binaries.
On Windows pdb files are left in the objdir where the program was
originally linked. symbolstore.py needs to locate the pdb file both to
determine whether it should dump symbols for a binary and also to copy
the pdb file into the symbol package. We fix this by simply looking for
the pdb file in the current working directory if it isn't present next
to the binary, which matches how we invoke symbolstore.py.
MozReview-Commit-ID: 8TOD1uTXD5e
--- a/browser/app/Makefile.in
+++ b/browser/app/Makefile.in
@@ -41,17 +41,17 @@ endif
PROGRAMS_DEST = $(DIST)/bin
include $(topsrcdir)/config/rules.mk
ifneq (,$(filter-out WINNT,$(OS_ARCH)))
ifdef COMPILE_ENVIRONMENT
libs::
- cp -p $(MOZ_APP_NAME)$(BIN_SUFFIX) $(DIST)/bin/$(MOZ_APP_NAME)-bin$(BIN_SUFFIX)
+ cp -p $(DIST)/bin/$(MOZ_APP_NAME)$(BIN_SUFFIX) $(DIST)/bin/$(MOZ_APP_NAME)-bin$(BIN_SUFFIX)
endif
GARBAGE += $(addprefix $(FINAL_TARGET)/defaults/pref/, firefox.js)
endif
ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
--- a/config/makefiles/target_binaries.mk
+++ b/config/makefiles/target_binaries.mk
@@ -2,18 +2,18 @@
# vim:set ts=8 sw=8 sts=8 noet:
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
ifndef NO_DIST_INSTALL
-ifneq (,$(strip $(PROGRAM)$(SIMPLE_PROGRAMS)$(RUST_PROGRAMS)))
-PROGRAMS_EXECUTABLES = $(SIMPLE_PROGRAMS) $(PROGRAM) $(RUST_PROGRAMS)
+ifneq (,$(strip $(SIMPLE_PROGRAMS)$(RUST_PROGRAMS)))
+PROGRAMS_EXECUTABLES = $(SIMPLE_PROGRAMS) $(RUST_PROGRAMS)
PROGRAMS_DEST ?= $(FINAL_TARGET)
PROGRAMS_TARGET := target
INSTALL_TARGETS += PROGRAMS
endif
ifdef SHARED_LIBRARY
SHARED_LIBRARY_FILES = $(SHARED_LIBRARY)
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -547,17 +547,17 @@ alltags:
define EXPAND_CC_OR_CXX
$(if $(PROG_IS_C_ONLY_$(1)),$(EXPAND_CC),$(EXPAND_CCC))
endef
#
# PROGRAM = Foo
# creates OBJS, links with LIBS to create Foo
#
-$(PROGRAM): $(PROGOBJS) $(STATIC_LIBS_DEPS) $(EXTRA_DEPS) $(RESFILE) $(GLOBAL_DEPS)
+$(PROGRAM): $(PROGOBJS) $(STATIC_LIBS_DEPS) $(EXTRA_DEPS) $(RESFILE) $(GLOBAL_DEPS) $(call mkdir_deps,$(FINAL_TARGET))
$(REPORT_BUILD)
@$(RM) $@.manifest
ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
$(EXPAND_LINK) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(PROGOBJS) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
ifdef MSMANIFEST_TOOL
@if test -f $@.manifest; then \
if test -f '$(srcdir)/$@.manifest'; then \
echo 'Embedding manifest from $(srcdir)/$@.manifest and $@.manifest'; \
@@ -812,17 +812,17 @@ endif
ifdef MOZ_AUTOMATION
ifeq (,$(filter 1,$(MOZ_AUTOMATION_BUILD_SYMBOLS)))
DUMP_SYMS_TARGETS :=
endif
endif
ifdef MOZ_CRASHREPORTER
-$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(file)_syms.track)))
+$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(notdir $(file))_syms.track)))
else ifneq (,$(and $(LLVM_SYMBOLIZER),$(filter WINNT,$(OS_ARCH)),$(MOZ_AUTOMATION)))
$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(addsuffix .pdb,$(basename $(file))))))
PDB_FILES = $(addsuffix .pdb,$(basename $(DUMP_SYMS_TARGETS)))
PDB_DEST ?= $(FINAL_TARGET)
PDB_TARGET = target
INSTALL_TARGETS += PDB
endif
--- a/js/src/shell/moz.build
+++ b/js/src/shell/moz.build
@@ -59,9 +59,9 @@ if CONFIG['CC_TYPE'] in ('msvc', 'clang-
# Place a GDB Python auto-load file next to the shell executable, both in
# the build directory and in the dist/bin directory.
DEFINES['topsrcdir'] = '%s/js/src' % TOPSRCDIR
FINAL_TARGET_PP_FILES += ['js-gdb.py.in']
OBJDIR_FILES.js.src.shell += ['!/dist/bin/js-gdb.py']
# People expect the js shell to wind up in the top-level JS dir.
-OBJDIR_FILES.js.src += ['!js%s' % CONFIG['BIN_SUFFIX']]
+OBJDIR_FILES.js.src += ['!/dist/bin/js%s' % CONFIG['BIN_SUFFIX']]
--- a/modules/libmar/tests/moz.build
+++ b/modules/libmar/tests/moz.build
@@ -3,10 +3,10 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
if CONFIG['OS_TARGET'] != 'Android':
TEST_HARNESS_FILES.xpcshell.modules.libmar.tests.unit += [
- '!../tool/signmar%s' % CONFIG['BIN_SUFFIX'],
+ '!/dist/bin/signmar%s' % CONFIG['BIN_SUFFIX'],
]
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1124,17 +1124,17 @@ class RecursiveMakeBackend(CommonBackend
interfaces_manifests = ' '.join(interfaces_manifests),
xpidl_rules=rules.getvalue(),
xpidl_modules=' '.join(xpt_modules),
xpt_files=' '.join(sorted(xpt_files - registered_xpt_files)),
registered_xpt_files=' '.join(sorted(registered_xpt_files)),
))
def _process_program(self, obj, backend_file):
- backend_file.write('PROGRAM = %s\n' % obj.program)
+ backend_file.write('PROGRAM = %s\n' % self._pretty_path(obj.output_path, backend_file))
if not obj.cxx_link and not self.environment.bin_suffix:
backend_file.write('PROG_IS_C_ONLY_%s := 1\n' % obj.program)
def _process_host_program(self, program, backend_file):
backend_file.write('HOST_PROGRAM = %s\n' % program)
def _process_rust_program_base(self, obj, backend_file,
target_variable,
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -12,16 +12,17 @@ All data structures of interest are chil
Logic for populating these data structures is not defined in this class.
Instead, what we have here are dumb container classes. The emitter module
contains the code for converting executed mozbuild files into these data
structures.
"""
from __future__ import absolute_import, unicode_literals
+from mozbuild.frontend.context import ObjDirPath
from mozbuild.util import StrictOrderingOnAppendList
from mozpack.chrome.manifest import ManifestEntry
import mozpack.path as mozpath
from .context import FinalTargetValue
from collections import defaultdict, OrderedDict
@@ -80,16 +81,20 @@ class ContextDerived(TreeMetadata):
self._context = context
@property
def install_target(self):
return self._context['FINAL_TARGET']
@property
+ def installed(self):
+ return self._context['DIST_INSTALL'] is not False
+
+ @property
def defines(self):
defines = self._context['DEFINES']
return Defines(self._context, defines) if defines else None
@property
def relobjdir(self):
return mozpath.relpath(self.objdir, self.topobjdir)
@@ -421,16 +426,23 @@ class BaseProgram(Linkable):
Linkable.__init__(self, context)
bin_suffix = context.config.substs.get(self.SUFFIX_VAR, '')
if not program.endswith(bin_suffix):
program += bin_suffix
self.program = program
self.is_unit_test = is_unit_test
+ @property
+ def output_path(self):
+ if self.installed:
+ return ObjDirPath(self._context, '!/' + mozpath.join(self.install_target, self.program))
+ else:
+ return ObjDirPath(self._context, '!' + self.program)
+
def __repr__(self):
return '<%s: %s/%s>' % (type(self).__name__, self.relobjdir, self.program)
class Program(BaseProgram):
"""Context derived container object for PROGRAM"""
SUFFIX_VAR = 'BIN_SUFFIX'
KIND = 'target'
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -794,17 +794,18 @@ class TreeMetadataEmitter(LoggingMixin):
'GENERATED_FILES: %s') % (symbols_file,), context)
shared_args['symbols_file'] = symbols_file.target_basename
if shared_lib:
lib = SharedLibrary(context, libname, **shared_args)
self._libs[libname].append(lib)
self._linkage.append((context, lib, 'USE_LIBS'))
linkables.append(lib)
- generated_files.add(lib.lib_name)
+ if not lib.installed:
+ generated_files.add(lib.lib_name)
if symbols_file and isinstance(symbols_file, SourcePath):
script = mozpath.join(
mozpath.dirname(mozpath.dirname(__file__)),
'action', 'generate_symbols_file.py')
defines = ()
if lib.defines:
defines = lib.defines.get_defines()
yield GeneratedFile(context, script,
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -607,31 +607,50 @@ class Dumper:
elapsed = time.time() - t_start
print('Finished processing %s in %.2fs' % (file, elapsed),
file=sys.stderr)
# Platform-specific subclasses. For the most part, these just have
# logic to determine what files to extract symbols from.
+def locate_pdb(path):
+ '''Given a path to a binary, attempt to locate the matching pdb file with simple heuristics:
+ * Look for a pdb file with the same base name next to the binary
+ * Look for a pdb file with the same base name in the cwd
+
+ Returns the path to the pdb file if it exists, or None if it could not be located.
+ '''
+ path, ext = os.path.splitext(path)
+ pdb = path + '.pdb'
+ if os.path.isfile(pdb):
+ return pdb
+ # If there's no pdb next to the file, see if there's a pdb with the same root name
+ # in the cwd. We build some binaries directly into dist/bin, but put the pdb files
+ # in the relative objdir, which is the cwd when running this script.
+ base = os.path.basename(pdb)
+ pdb = os.path.join(os.getcwd(), base)
+ if os.path.isfile(pdb):
+ return pdb
+ return None
+
class Dumper_Win32(Dumper):
fixedFilenameCaseCache = {}
def ShouldProcess(self, file):
"""This function will allow processing of exe or dll files that have pdb
files with the same base name next to them."""
if file.endswith(".exe") or file.endswith(".dll"):
- path, ext = os.path.splitext(file)
- if os.path.isfile(path + ".pdb"):
+ if locate_pdb(file) is not None:
return True
return False
def CopyDebug(self, file, debug_file, guid, code_file, code_id):
- file = "%s.pdb" % os.path.splitext(file)[0]
+ file = locate_pdb(file)
def compress(path):
compressed_file = path[:-1] + '_'
# ignore makecab's output
makecab = buildconfig.substs['MAKECAB']
success = subprocess.call([makecab, "-D",
"CompressionType=MSZIP",
path, compressed_file],
stdout=open(os.devnull, 'w'),
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -477,18 +477,17 @@ class TestFunctional(HelperMixin, unitte
else:
self.dump_syms = os.path.join(self.topsrcdir,
'toolkit',
'crashreporter',
'tools',
'win32',
'dump_syms_vc{_MSC_VER}.exe'.format(**buildconfig.substs))
self.target_bin = os.path.join(buildconfig.topobjdir,
- 'browser',
- 'app',
+ 'dist', 'bin',
'firefox.exe')
else:
self.dump_syms = os.path.join(buildconfig.topobjdir,
'dist', 'host', 'bin',
'dump_syms')
self.target_bin = os.path.join(buildconfig.topobjdir,
'dist', 'bin', 'firefox')
@@ -497,26 +496,28 @@ class TestFunctional(HelperMixin, unitte
HelperMixin.tearDown(self)
def testSymbolstore(self):
if self.skip_test:
raise unittest.SkipTest('Skipping test in non-Firefox product')
dist_include_manifest = os.path.join(buildconfig.topobjdir,
'_build_manifests/install/dist_include')
dist_include = os.path.join(buildconfig.topobjdir, 'dist/include')
+ browser_app = os.path.join(buildconfig.topobjdir, 'browser/app')
output = subprocess.check_output([sys.executable,
self.script_path,
'--vcs-info',
'-s', self.topsrcdir,
'--install-manifest=%s,%s' % (dist_include_manifest,
dist_include),
self.dump_syms,
self.test_dir,
self.target_bin],
- stderr=open(os.devnull, 'w'))
+ stderr=open(os.devnull, 'w'),
+ cwd=browser_app)
lines = filter(lambda x: x.strip(), output.splitlines())
self.assertEqual(1, len(lines),
'should have one filename in the output')
symbol_file = os.path.join(self.test_dir, lines[0])
self.assertTrue(os.path.isfile(symbol_file))
symlines = open(symbol_file, 'r').readlines()
file_lines = [l for l in symlines if l.startswith('FILE')]
def check_hg_path(lines, match):