Bug 1315410 - Isolate perfherder metrics by build type; r?ted
Currently, artifact, static analysis, and valgrind build metrics go to
the same bucket as non-artifact build metrics. This results in
bi-modal metrics.
In this commit, we introduce a "perfherder_extra_options" config
value which is concatenated with the auto-derived values. We change
mozharness configs to add appropriate entries for various build
configurations.
I'm not keen on adding more functionality to mozharness. But this is
easier than having BuildMixin read state from the build system. I also
like the easy extensibility of this mechanism.
MozReview-Commit-ID: BBWA4ZsZ0cp
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/32_artifact.py
@@ -28,16 +28,17 @@ config = {
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': True,
'enable_talos_sendchange': False,
# allows triggering of test jobs when --artifact try syntax is detected on buildbot
'enable_unittest_sendchange': True,
+ 'perfherder_extra_options': ['artifact'],
#########################################################################
#########################################################################
###### 32 bit specific ######
'base_name': 'Linux_%(branch)s_Artifact_build',
'platform': 'linux',
'stage_platform': 'linux',
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug_artifact.py
@@ -32,16 +32,17 @@ config = {
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': True,
# debug specific
'debug_build': True,
'enable_talos_sendchange': False,
# allows triggering of test jobs when --artifact try syntax is detected on buildbot
'enable_unittest_sendchange': True,
+ 'perfherder_extra_options': ['artifact'],
#########################################################################
#########################################################################
###### 32 bit specific ######
'base_name': 'Linux_%(branch)s_Artifact_build',
'platform': 'linux',
'stage_platform': 'linux-debug',
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py
@@ -22,16 +22,17 @@ config = {
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': True,
'enable_talos_sendchange': False,
# allows triggering of test jobs when --artifact try syntax is detected on buildbot
'enable_unittest_sendchange': True,
+ 'perfherder_extra_options': ['artifact'],
#########################################################################
#########################################################################
###### 64 bit specific ######
'base_name': 'Linux_x86-64_%(branch)s_Artifact_build',
'platform': 'linux64',
'stage_platform': 'linux64',
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug_artifact.py
@@ -26,16 +26,17 @@ config = {
'enable_count_ctors': True,
# debug specific
'debug_build': True,
# decides whether we want to use moz_sign_cmd in env
'enable_signing': False,
'enable_talos_sendchange': False,
# allows triggering of test jobs when --artifact try syntax is detected on buildbot
'enable_unittest_sendchange': True,
+ 'perfherder_extra_options': ['artifact'],
#########################################################################
#########################################################################
###### 64 bit specific ######
'base_name': 'Linux_x86-64_%(branch)s_Artifact_build',
'platform': 'linux64',
'stage_platform': 'linux64-debug',
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_debug.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_debug.py
@@ -18,16 +18,17 @@ config = {
'stage_platform': 'linux64-st-an-debug',
'build_type': 'st-an-debug',
'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
clang.manifest",
'platform_supports_post_upload_to_latest': False,
'enable_signing': False,
'enable_talos_sendchange': False,
'enable_unittest_sendchange': False,
+ 'perfherder_extra_options': ['static-analysis'],
#### 64 bit build specific #####
'env': {
'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
'MOZ_AUTOMATION': '1',
'DISPLAY': ':2',
'HG_SHARE_BASE_DIR': '/builds/hg-shared',
'MOZ_OBJDIR': MOZ_OBJDIR,
'TINDERBOX_OUTPUT': '1',
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py
@@ -20,16 +20,17 @@ config = {
'enable_ccache': True,
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': True,
'enable_talos_sendchange': False,
'enable_unittest_sendchange': False,
+ 'perfherder_extra_options': ['static-analysis'],
#########################################################################
#########################################################################
###### 64 bit specific ######
'base_name': 'Linux_x86-64_%(branch)s_Static_Analysis',
'platform': 'linux64',
'stage_platform': 'linux64-st-an',
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_valgrind.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_valgrind.py
@@ -19,16 +19,17 @@ config = {
'stage_platform': 'linux64-valgrind',
'publish_nightly_en_US_routes': False,
'build_type': 'valgrind',
'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
releng.manifest",
'platform_supports_post_upload_to_latest': False,
'enable_signing': False,
'enable_talos_sendchange': False,
+ 'perfherder_extra_options': ['valgrind'],
#### 64 bit build specific #####
'env': {
'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
'MOZ_AUTOMATION': '1',
'DISPLAY': ':2',
'HG_SHARE_BASE_DIR': '/builds/hg-shared',
'MOZ_OBJDIR': 'obj-firefox',
'TINDERBOX_OUTPUT': '1',
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_artifact.py
@@ -25,16 +25,17 @@ config = {
'vcs_share_base': '/builds/hg-shared',
'objdir': 'obj-firefox',
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': False,
'enable_talos_sendchange': False,
# allows triggering of test jobs when --artifact try syntax is detected on buildbot
'enable_unittest_sendchange': True,
+ 'perfherder_extra_options': ['artifact'],
#########################################################################
#########################################################################
###### 64 bit specific ######
'base_name': 'OS X 10.7 %(branch)s_Artifact_build',
'platform': 'macosx64',
'stage_platform': 'macosx64',
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug_artifact.py
@@ -26,16 +26,17 @@ config = {
'enable_ccache': True,
'vcs_share_base': '/builds/hg-shared',
'objdir': MOZ_OBJDIR,
# debug specific
'debug_build': True,
'enable_talos_sendchange': False,
# allows triggering of test jobs when --artifact try syntax is detected on buildbot
'enable_unittest_sendchange': True,
+ 'perfherder_extra_options': ['artifact'],
#########################################################################
#########################################################################
###### 64 bit specific ######
'base_name': 'OS X 10.7 %(branch)s_Artifact_build',
'platform': 'macosx64',
'stage_platform': 'macosx64-debug',
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_stat_and_debug.py
+++ b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_stat_and_debug.py
@@ -19,16 +19,17 @@ config = {
'build_type': 'st-an-debug',
'tooltool_manifest_src': "browser/config/tooltool-manifests/macosx64/\
clang.manifest",
'platform_supports_post_upload_to_latest': False,
'enable_signing': False,
'enable_talos_sendchange': False,
'enable_unittest_sendchange': False,
'objdir': MOZ_OBJDIR,
+ 'perfherder_extra_options': ['static-analysis'],
#### 64 bit build specific #####
'env': {
'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
'MOZ_AUTOMATION': '1',
'HG_SHARE_BASE_DIR': '/builds/hg-shared',
'MOZ_OBJDIR': MOZ_OBJDIR,
'TINDERBOX_OUTPUT': '1',
'TOOLTOOL_CACHE': '/builds/tooltool_cache',
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_artifact.py
@@ -44,16 +44,17 @@ config = {
'tooltool_script': [sys.executable,
'C:/mozilla-build/tooltool.py'],
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': False,
'enable_talos_sendchange': False,
# allows triggering of test jobs when --artifact try syntax is detected on buildbot
'enable_unittest_sendchange': True,
'max_build_output_timeout': 60 * 80,
+ 'perfherder_extra_options': ['artifact'],
#########################################################################
#########################################################################
###### 32 bit specific ######
'base_name': 'WINNT_5.2_%(branch)s_Artifact_build',
'platform': 'win32',
'stage_platform': 'win32',
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug_artifact.py
+++ b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug_artifact.py
@@ -48,16 +48,17 @@ config = {
'tooltool_bootstrap': "setup.sh",
'enable_count_ctors': False,
# debug specific
'debug_build': True,
'enable_talos_sendchange': False,
# allows triggering of test jobs when --artifact try syntax is detected on buildbot
'enable_unittest_sendchange': True,
'max_build_output_timeout': 60 * 80,
+ 'perfherder_extra_options': ['artifact'],
#########################################################################
#########################################################################
###### 32 bit specific ######
'base_name': 'WINNT_5.2_%(branch)s_Artifact_build',
'platform': 'win32',
'stage_platform': 'win32-debug',
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_stat_and_debug.py
+++ b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_stat_and_debug.py
@@ -16,16 +16,17 @@ config = {
'debug_build': True,
'enable_signing': False,
'enable_talos_sendchange': False,
'enable_unittest_sendchange': False,
'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/\
clang.manifest",
'platform_supports_post_upload_to_latest': False,
'objdir': MOZ_OBJDIR,
+ 'perfherder_extra_options': ['static-analysis'],
#### 32 bit build specific #####
'env': {
'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
'MOZ_AUTOMATION': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1',
'MOZ_OBJDIR': 'obj-firefox',
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -523,16 +523,19 @@ class PerfherderResourceOptionsMixin(Scr
self.info('instance_metadata.json not found; unable to '
'determine instance type')
except Exception:
self.warning('error reading instance_metadata: %s' %
traceback.format_exc())
opts.append('buildbot-%s' % instance)
+ # Allow configs to specify their own values.
+ opts.extend(self.config.get('perfherder_extra_options', []))
+
return opts
class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
"""Provides resource monitoring capabilities to scripts.
When this class is in the inheritance chain, resource usage stats of the
executing script will be recorded.