Bug 1315041 - Record environment state in perfherder options; r?wlach, nthomas
Currently, perfherder data containing resource usage goes into the same
bucket regardless of the execution environment. This means results for
buildbot, taskcluster, and for different AWS instance types are all
mixed together, making the data noisy.
This commit adds extraOptions values to the perfherder data to
identify:
* buildbot vs taskcluster
* the TC instance type
Data will go to separate buckets and should be much more consistent.
MozReview-Commit-ID: A0zSkdVI6ZM
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -3,29 +3,31 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
'''Python usage, esp. virtualenv.
'''
import distutils.version
+import errno
import os
import subprocess
import sys
import json
import socket
import traceback
import urlparse
import mozharness
from mozharness.base.script import (
PostScriptAction,
PostScriptRun,
PreScriptAction,
+ ScriptMixin,
)
from mozharness.base.errors import VirtualenvErrorList
from mozharness.base.log import WARNING, FATAL
from mozharness.mozilla.proxxy import Proxxy
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
'external_tools',
@@ -488,17 +490,53 @@ class VirtualenvMixin(object):
def activate_virtualenv(self):
"""Import the virtualenv's packages into this Python interpreter."""
bin_dir = os.path.dirname(self.query_python_path())
activate = os.path.join(bin_dir, 'activate_this.py')
execfile(activate, dict(__file__=activate))
-class ResourceMonitoringMixin(object):
+# This is (sadly) a mixin for logging methods.
+class PerfherderResourceOptionsMixin(ScriptMixin):
+ def perfherder_resource_options(self):
+ """Obtain a list of extraOptions values to identify the env."""
+ opts = []
+
+ if 'TASKCLUSTER_INSTANCE_TYPE' in os.environ:
+ # Include the instance type so results can be grouped.
+ opts.append('taskcluster-%s' % os.environ['TASKCLUSTER_INSTANCE_TYPE'])
+ else:
+ # We assume !taskcluster => buildbot.
+ instance = 'unknown'
+
+ # Try to load EC2 instance type from metadata file. This file
+ # may not exist in many scenarios (including when inside a chroot).
+ # So treat it as optional.
+ # TODO support Windows.
+ try:
+ # This file should exist on Linux in EC2.
+ with open('/etc/instance_metadata.json', 'rb') as fh:
+ im = json.load(fh)
+ instance = im['aws_instance_type'].encode('ascii')
+ except IOError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ self.info('instance_metadata.json not found; unable to '
+ 'determine instance type')
+ except Exception:
+ self.warning('error reading instance_metadata: %s' %
+ traceback.format_exc())
+
+ opts.append('buildbot-%s' % instance)
+
+ return opts
+
+
+class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
"""Provides resource monitoring capabilities to scripts.
When this class is in the inheritance chain, resource usage stats of the
executing script will be recorded.
This class requires the VirtualenvMixin in order to install a package used
for recording resource usage.
@@ -656,17 +694,17 @@ class ResourceMonitoringMixin(object):
{'name': 'io_write_bytes', 'value': io.write_bytes},
{'name': 'io.read_bytes', 'value': io.read_bytes},
{'name': 'io_write_time', 'value': io.write_time},
{'name': 'io_read_time', 'value': io.read_time},
])
suites.append({
'name': '%s.overall' % perfherder_name,
- 'extraOptions': perfherder_options,
+ 'extraOptions': perfherder_options + self.perfherder_resource_options(),
'subtests': overall,
})
for phase in rm.phases.keys():
phase_duration = rm.phases[phase][1] - rm.phases[phase][0]
subtests = [
{
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -45,17 +45,20 @@ from mozharness.mozilla.purge import Pur
from mozharness.mozilla.mock import MockMixin
from mozharness.mozilla.secrets import SecretsMixin
from mozharness.mozilla.signing import SigningMixin
from mozharness.mozilla.mock import ERROR_MSGS as MOCK_ERROR_MSGS
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.testing.unittest import tbox_print_summary
from mozharness.mozilla.updates.balrog import BalrogMixin
from mozharness.mozilla.taskcluster_helper import Taskcluster
-from mozharness.base.python import VirtualenvMixin
+from mozharness.base.python import (
+ PerfherderResourceOptionsMixin,
+ VirtualenvMixin,
+)
AUTOMATION_EXIT_CODES = EXIT_STATUS_DICT.values()
AUTOMATION_EXIT_CODES.sort()
MISSING_CFG_KEY_MSG = "The key '%s' could not be determined \
Please add this to your config."
ERROR_MSGS = {
@@ -582,17 +585,17 @@ def generate_build_ID():
def generate_build_UID():
return uuid.uuid4().hex
class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
SigningMixin, VirtualenvMixin, MercurialScript,
- SecretsMixin):
+ SecretsMixin, PerfherderResourceOptionsMixin):
def __init__(self, **kwargs):
# objdir is referenced in _query_abs_dirs() so let's make sure we
# have that attribute before calling BaseScript.__init__
self.objdir = None
super(BuildScript, self).__init__(**kwargs)
# epoch is only here to represent the start of the buildbot build
# that this mozharn script came from. until I can grab bbot's
# status.build.gettime()[0] this will have to do as a rough estimate
@@ -1858,16 +1861,17 @@ or run without that action (ie: --no-{ac
if 'duration' not in resources:
self.info('resource usage lacks duration; ignoring')
return None
data = {
'name': 'build times',
'value': resources['duration'],
+ 'extraOptions': self.perfherder_resource_options(),
'subtests': [],
}
for phase in resources['phases']:
if 'duration' not in phase:
continue
data['subtests'].append({
'name': phase['name'],