Bug 1429875 - Implement OBJ_SUFFIX overriding for the profile generation phase on linux in mozbuild.
MozReview-Commit-ID: 8PtgxfbxuE
--- a/config/config.mk
+++ b/config/config.mk
@@ -469,25 +469,28 @@ define CHECK_BINARY
$(call LOCAL_CHECKS,$(1))
$(call CHECK_MOZGLUE_ORDER,$(1))
endef
# autoconf.mk sets OBJ_SUFFIX to an error to avoid use before including
# this file
OBJ_SUFFIX := $(_OBJ_SUFFIX)
+OBJS_VAR_SUFFIX := OBJS
+
# PGO builds with GCC build objects with instrumentation in a first pass,
# then objects optimized, without instrumentation, in a second pass. If
# we overwrite the objects from the first pass with those from the second,
# we end up not getting instrumentation data for better optimization on
# incremental builds. As a consequence, we use a different object suffix
# for the first pass.
-ifndef NO_PROFILE_GUIDED_OPTIMIZE
ifdef MOZ_PROFILE_GENERATE
ifdef GNU_CC
+OBJS_VAR_SUFFIX := PGO_OBJS
+ifndef NO_PROFILE_GUIDED_OPTIMIZE
OBJ_SUFFIX := i_o
endif
endif
endif
PLY_INCLUDE = -I$(MOZILLA_DIR)/other-licenses/ply
export CL_INCLUDES_PREFIX
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -538,17 +538,17 @@ endef
#
# PROGRAM = Foo
# creates OBJS, links with LIBS to create Foo
#
$(PROGRAM): $(PROGOBJS) $(STATIC_LIBS) $(EXTRA_DEPS) $(RESFILE) $(GLOBAL_DEPS) $(call mkdir_deps,$(FINAL_TARGET))
$(REPORT_BUILD)
@$(RM) $@.manifest
ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
- $(LINKER) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) -IMPLIB:$(basename $(@F)).lib $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $($(notdir $@)_OBJS) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
+ $(LINKER) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) -IMPLIB:$(basename $(@F)).lib $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $($(notdir $@)_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
ifdef MSMANIFEST_TOOL
@if test -f $@.manifest; then \
if test -f '$(srcdir)/$(notdir $@).manifest'; then \
echo 'Embedding manifest from $(srcdir)/$(notdir $@).manifest and $@.manifest'; \
$(MT) -NOLOGO -MANIFEST '$(win_srcdir)/$(notdir $@).manifest' $@.manifest -OUTPUTRESOURCE:$@\;1; \
else \
echo 'Embedding manifest from $@.manifest'; \
$(MT) -NOLOGO -MANIFEST $@.manifest -OUTPUTRESOURCE:$@\;1; \
@@ -559,17 +559,17 @@ ifdef MSMANIFEST_TOOL
fi
endif # MSVC with manifest tool
ifdef MOZ_PROFILE_GENERATE
# touch it a few seconds into the future to work around FAT's
# 2-second granularity
touch -t `date +%Y%m%d%H%M.%S -d 'now+5seconds'` pgo.relink
endif
else # !WINNT || GNU_CC
- $(call EXPAND_CC_OR_CXX,$@) -o $@ $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) $($(notdir $@)_OBJS) $(RESFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
+ $(call EXPAND_CC_OR_CXX,$@) -o $@ $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) $($(notdir $@)_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
$(call CHECK_BINARY,$@)
endif # WINNT && !GNU_CC
ifdef ENABLE_STRIP
$(STRIP) $(STRIP_FLAGS) $@
endif
ifdef MOZ_POST_PROGRAM_COMMAND
$(MOZ_POST_PROGRAM_COMMAND) $@
@@ -610,25 +610,25 @@ endif
# Foo.o (from either Foo.c or Foo.cpp).
#
# SIMPLE_PROGRAMS = Foo Bar
# creates Foo.o Bar.o, links with LIBS to create Foo, Bar.
#
$(SIMPLE_PROGRAMS): %$(BIN_SUFFIX): %.$(OBJ_SUFFIX) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
$(REPORT_BUILD)
ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
- $(LINKER) -nologo -out:$@ -pdb:$(LINK_PDBFILE) $($@_OBJS) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
+ $(LINKER) -nologo -out:$@ -pdb:$(LINK_PDBFILE) $($@_$(OBJS_VAR_SUFFIX)) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
ifdef MSMANIFEST_TOOL
@if test -f $@.manifest; then \
$(MT) -NOLOGO -MANIFEST $@.manifest -OUTPUTRESOURCE:$@\;1; \
rm -f $@.manifest; \
fi
endif # MSVC with manifest tool
else
- $(call EXPAND_CC_OR_CXX,$@) $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) -o $@ $($@_OBJS) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
+ $(call EXPAND_CC_OR_CXX,$@) $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) -o $@ $($@_$(OBJS_VAR_SUFFIX)) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
$(call CHECK_BINARY,$@)
endif # WINNT && !GNU_CC
ifdef ENABLE_STRIP
$(STRIP) $(STRIP_FLAGS) $@
endif
ifdef MOZ_POST_PROGRAM_COMMAND
$(MOZ_POST_PROGRAM_COMMAND) $@
@@ -647,17 +647,17 @@ endif
endif
ifndef CROSS_COMPILE
$(call CHECK_STDCXX,$@)
endif
$(LIBRARY): $(OBJS) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
$(REPORT_BUILD)
$(RM) $(REAL_LIBRARY)
- $(AR) $(AR_FLAGS) $(OBJS) $($@_OBJS)
+ $(AR) $(AR_FLAGS) $(OBJS) $($@_$(OBJS_VAR_SUFFIX))
ifeq ($(OS_ARCH),WINNT)
# Import libraries are created by the rules creating shared libraries.
# The rules to copy them to $(DIST)/lib depend on $(IMPORT_LIBRARY),
# but make will happily consider the import library before it is refreshed
# when rebuilding the corresponding shared library. Defining an empty recipe
# for import libraries forces make to wait for the shared library recipe to
# have run before considering other targets that depend on the import library.
@@ -675,17 +675,17 @@ endif
# symlinks back to the originals. The symlinks are a no-op for stabs debugging,
# so no need to conditionalize on OS version or debugging format.
$(SHARED_LIBRARY): $(OBJS) $(RESFILE) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
$(REPORT_BUILD)
ifndef INCREMENTAL_LINKER
$(RM) $@
endif
- $(MKSHLIB) $($@_OBJS) $(RESFILE) $(LDFLAGS) $(STATIC_LIBS) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(SHARED_LIBS) $(EXTRA_DSO_LDOPTS) $(MOZ_GLUE_LDFLAGS) $(OS_LIBS)
+ $(MKSHLIB) $($@_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(LDFLAGS) $(STATIC_LIBS) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(SHARED_LIBS) $(EXTRA_DSO_LDOPTS) $(MOZ_GLUE_LDFLAGS) $(OS_LIBS)
$(call CHECK_BINARY,$@)
ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
ifdef MSMANIFEST_TOOL
ifdef EMBED_MANIFEST_AT
@if test -f $@.manifest; then \
if test -f '$(srcdir)/$@.manifest'; then \
echo 'Embedding manifest from $(srcdir)/$@.manifest and $@.manifest'; \
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -203,25 +203,31 @@ class CommonBackend(BuildBackend):
}
json.dump(d, fh, sort_keys=True, indent=4)
def _expand_libs(self, input_bin):
os_libs = []
shared_libs = []
static_libs = []
objs = []
+ no_pgo_objs = []
seen_objs = set()
seen_libs = set()
def add_objs(lib):
for o in lib.objs:
if o not in seen_objs:
seen_objs.add(o)
objs.append(o)
+ # This is slightly odd, buf for consistency with the
+ # recursivemake backend we don't replace OBJ_SUFFIX if any
+ # object in a library has `no_pgo` set.
+ if lib.no_pgo_objs or lib.no_pgo:
+ no_pgo_objs.append(o)
def expand(lib, recurse_objs, system_libs):
if isinstance(lib, StaticLibrary):
if lib.no_expand_lib:
static_libs.append(lib)
recurse_objs = False
elif recurse_objs:
add_objs(lib)
@@ -253,17 +259,17 @@ class CommonBackend(BuildBackend):
seen_libs.add(lib)
shared_libs.append(lib)
for lib in input_bin.linked_system_libs:
if lib not in seen_libs:
seen_libs.add(lib)
os_libs.append(lib)
- return objs, shared_libs, os_libs, static_libs
+ return objs, no_pgo_objs, shared_libs, os_libs, static_libs
def _make_list_file(self, objdir, objs, name):
if not objs:
return None
list_style = self.environment.substs.get('EXPAND_LIBS_LIST_STYLE')
list_file_path = mozpath.join(objdir, name)
objs = [os.path.relpath(o, objdir) for o in objs]
if list_style == 'linkerscript':
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1320,43 +1320,78 @@ class RecursiveMakeBackend(CommonBackend
def pretty_relpath(lib):
return '$(DEPTH)/%s' % mozpath.relpath(lib.objdir, topobjdir)
topobjdir = mozpath.normsep(obj.topobjdir)
# This will create the node even if there aren't any linked libraries.
build_target = self._build_target_for_obj(obj)
self._compile_graph[build_target]
- objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
+ objs, no_pgo_objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
if obj.KIND == 'target':
obj_target = obj.name
if isinstance(obj, Program):
obj_target = self._pretty_path(obj.output_path, backend_file)
+ is_unit_test = isinstance(obj, BaseProgram) and obj.is_unit_test
+ profile_gen_objs = []
+
+ if (self.environment.substs.get('MOZ_PGO') and
+ self.environment.substs.get('GNU_CC')):
+ # We use a different OBJ_SUFFIX for the profile generate phase on
+ # linux. These get picked up via OBJS_VAR_SUFFIX in config.mk.
+ if not is_unit_test and not isinstance(obj, SimpleProgram):
+ profile_gen_objs = [o if o in no_pgo_objs else '%s.%s' %
+ (mozpath.splitext(o)[0], 'i_o') for o in objs]
+
+ def write_obj_deps(target, objs_ref, pgo_objs_ref):
+ if pgo_objs_ref:
+ backend_file.write('ifdef MOZ_PROFILE_GENERATE\n')
+ backend_file.write('%s: %s\n' % (target, pgo_objs_ref))
+ backend_file.write('else\n')
+ backend_file.write('%s: %s\n' % (target, objs_ref))
+ backend_file.write('endif\n')
+ else:
+ backend_file.write('%s: %s\n' % (target, objs_ref))
+
objs_ref = ' \\\n '.join(os.path.relpath(o, obj.objdir)
for o in objs)
+ pgo_objs_ref = ' \\\n '.join(os.path.relpath(o, obj.objdir)
+ for o in profile_gen_objs)
# Don't bother with a list file if we're only linking objects built
# in this directory or building a real static library. This
# accommodates clang-plugin, where we would otherwise pass an
# incorrect list file format to the host compiler as well as when
# creating an archive with AR, which doesn't understand list files.
if (objs == obj.objs and not isinstance(obj, StaticLibrary) or
- isinstance(obj, StaticLibrary) and obj.no_expand_lib):
- backend_file.write_once('%s_OBJS := %s\n' %
- (obj.name, objs_ref))
- backend_file.write_once('%s: %s\n' % (obj_target, objs_ref))
+ isinstance(obj, StaticLibrary) and obj.no_expand_lib):
+ backend_file.write_once('%s_OBJS := %s\n' % (obj.name,
+ objs_ref))
+ if profile_gen_objs:
+ backend_file.write_once('%s_PGO_OBJS := %s\n' % (obj.name,
+ pgo_objs_ref))
+ write_obj_deps(obj_target, objs_ref, pgo_objs_ref)
elif not isinstance(obj, StaticLibrary):
list_file_path = '%s.list' % obj.name.replace('.', '_')
list_file_ref = self._make_list_file(obj.objdir, objs,
list_file_path)
backend_file.write_once('%s_OBJS := %s\n' %
(obj.name, list_file_ref))
backend_file.write_once('%s: %s\n' % (obj_target, list_file_path))
- backend_file.write_once('%s: %s\n' % (obj_target, objs_ref))
+ if profile_gen_objs:
+ pgo_list_file_path = '%s_pgo.list' % obj.name.replace('.', '_')
+ pgo_list_file_ref = self._make_list_file(obj.objdir,
+ profile_gen_objs,
+ pgo_list_file_path)
+ backend_file.write_once('%s_PGO_OBJS := %s\n' %
+ (obj.name, pgo_list_file_ref))
+ backend_file.write_once('%s: %s\n' % (obj_target,
+ pgo_list_file_path))
+ write_obj_deps(obj_target, objs_ref, pgo_objs_ref)
for lib in shared_libs:
backend_file.write_once('SHARED_LIBS += %s/%s\n' %
(pretty_relpath(lib), lib.import_name))
for lib in static_libs:
backend_file.write_once('STATIC_LIBS += %s/%s\n' %
(pretty_relpath(lib), lib.import_name))
for lib in os_libs:
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -383,26 +383,30 @@ class LinkageMultipleRustLibrariesError(
class Linkable(ContextDerived):
"""Generic context derived container object for programs and libraries"""
__slots__ = (
'cxx_link',
'lib_defines',
'linked_libraries',
'linked_system_libs',
+ 'no_pgo_sources',
+ 'no_pgo',
'sources',
)
def __init__(self, context):
ContextDerived.__init__(self, context)
self.cxx_link = False
self.linked_libraries = []
self.linked_system_libs = []
self.lib_defines = Defines(context, {})
self.sources = defaultdict(list)
+ self.no_pgo_sources = []
+ self.no_pgo = False
def link_library(self, obj):
assert isinstance(obj, BaseLibrary)
if obj.KIND != self.KIND:
raise LinkageWrongKindError('%s != %s' % (obj.KIND, self.KIND))
# Linking multiple Rust libraries into an object would result in
# multiple copies of the Rust standard library, as well as linking
# errors from duplicate symbols.
@@ -432,26 +436,33 @@ class Linkable(ContextDerived):
def source_files(self):
all_sources = []
# This is ordered for reproducibility and consistently w/
# config/rules.mk
for suffix in ('.c', '.S', '.cpp', '.m', '.mm', '.s'):
all_sources += self.sources.get(suffix, [])
return all_sources
- @property
- def objs(self):
+ def _get_objs(self, sources):
obj_prefix = ''
if self.KIND == 'host':
obj_prefix = 'host_'
return [mozpath.join(self.objdir, '%s%s.%s' % (obj_prefix,
mozpath.splitext(mozpath.basename(f))[0],
self.config.substs.get('OBJ_SUFFIX', '')))
- for f in self.source_files()]
+ for f in sources]
+
+ @property
+ def no_pgo_objs(self):
+ return self._get_objs(self.no_pgo_sources)
+
+ @property
+ def objs(self):
+ return self._get_objs(self.source_files())
class BaseProgram(Linkable):
"""Context derived container object for programs, which is a unicode
string.
This class handles automatically appending a binary suffix to the program
name.
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -946,16 +946,20 @@ class TreeMetadataEmitter(LoggingMixin):
ctxt_sources[variable][canonical_suffix] += sorted(srcs)
yield obj
if ctxt_sources:
for linkable in linkables:
for target_var in ('SOURCES', 'UNIFIED_SOURCES'):
for suffix, srcs in ctxt_sources[target_var].items():
linkable.sources[suffix] += srcs
+ if no_pgo_sources:
+ linkable.no_pgo_sources = no_pgo_sources
+ elif no_pgo:
+ linkable.no_pgo = True
for host_linkable in host_linkables:
for suffix, srcs in ctxt_sources['HOST_SOURCES'].items():
host_linkable.sources[suffix] += srcs
for f, flags in all_flags.iteritems():
if flags.flags:
ext = mozpath.splitext(f)[1]
yield PerSourceFlag(context, f, flags.flags)