Bug 1315041 - Record environment state in perfherder options; r?wlach, nthomas draft
authorGregory Szorc <gps@mozilla.com>
Thu, 03 Nov 2016 14:41:58 -0700
changeset 434128 586472f6f477c2aee21e21300515abb65100282e
parent 434126 9bb32d8f818d9981d7984634d963885d2cdd1d79
child 536039 3a542fe87695ead4250a0677d78353c5ba5a9cd5
push id34740
push userbmo:gps@mozilla.com
push dateFri, 04 Nov 2016 22:45:17 +0000
reviewerswlach, nthomas
bugs1315041
milestone52.0a1
Bug 1315041 - Record environment state in perfherder options; r?wlach, nthomas Currently, perfherder data containing resource usage goes into the same bucket regardless of the execution environment. This means results for buildbot, taskcluster, and for different AWS instance types are all mixed together, making the data noisy. This commit adds extraOptions values to the perfherder data to identify: * buildbot vs taskcluster * the TC instance type Data will go to separate buckets and should be much more consistent. MozReview-Commit-ID: A0zSkdVI6ZM
testing/mozharness/mozharness/base/python.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -3,29 +3,31 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 '''Python usage, esp. virtualenv.
 '''
 
 import distutils.version
+import errno
 import os
 import subprocess
 import sys
 import json
 import socket
 import traceback
 import urlparse
 
 import mozharness
 from mozharness.base.script import (
     PostScriptAction,
     PostScriptRun,
     PreScriptAction,
+    ScriptMixin,
 )
 from mozharness.base.errors import VirtualenvErrorList
 from mozharness.base.log import WARNING, FATAL
 from mozharness.mozilla.proxxy import Proxxy
 
 external_tools_path = os.path.join(
     os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
     'external_tools',
@@ -488,17 +490,53 @@ class VirtualenvMixin(object):
 
     def activate_virtualenv(self):
         """Import the virtualenv's packages into this Python interpreter."""
         bin_dir = os.path.dirname(self.query_python_path())
         activate = os.path.join(bin_dir, 'activate_this.py')
         execfile(activate, dict(__file__=activate))
 
 
-class ResourceMonitoringMixin(object):
+# This is (sadly) a mixin for logging methods.
+class PerfherderResourceOptionsMixin(ScriptMixin):
+    def perfherder_resource_options(self):
+        """Obtain a list of extraOptions values to identify the env."""
+        opts = []
+
+        if 'TASKCLUSTER_INSTANCE_TYPE' in os.environ:
+            # Include the instance type so results can be grouped.
+            opts.append('taskcluster-%s' % os.environ['TASKCLUSTER_INSTANCE_TYPE'])
+        else:
+            # We assume !taskcluster => buildbot.
+            instance = 'unknown'
+
+            # Try to load EC2 instance type from metadata file. This file
+            # may not exist in many scenarios (including when inside a chroot).
+            # So treat it as optional.
+            # TODO support Windows.
+            try:
+                # This file should exist on Linux in EC2.
+                with open('/etc/instance_metadata.json', 'rb') as fh:
+                    im = json.load(fh)
+                    instance = im['aws_instance_type'].encode('ascii')
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                self.info('instance_metadata.json not found; unable to '
+                          'determine instance type')
+            except Exception:
+                self.warning('error reading instance_metadata: %s' %
+                             traceback.format_exc())
+
+            opts.append('buildbot-%s' % instance)
+
+        return opts
+
+
+class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
     """Provides resource monitoring capabilities to scripts.
 
     When this class is in the inheritance chain, resource usage stats of the
     executing script will be recorded.
 
     This class requires the VirtualenvMixin in order to install a package used
     for recording resource usage.
 
@@ -656,17 +694,17 @@ class ResourceMonitoringMixin(object):
                 {'name': 'io_write_bytes', 'value': io.write_bytes},
                 {'name': 'io.read_bytes', 'value': io.read_bytes},
                 {'name': 'io_write_time', 'value': io.write_time},
                 {'name': 'io_read_time', 'value': io.read_time},
             ])
 
             suites.append({
                 'name': '%s.overall' % perfherder_name,
-                'extraOptions': perfherder_options,
+                'extraOptions': perfherder_options + self.perfherder_resource_options(),
                 'subtests': overall,
 
             })
 
             for phase in rm.phases.keys():
                 phase_duration = rm.phases[phase][1] - rm.phases[phase][0]
                 subtests = [
                     {
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -45,17 +45,20 @@ from mozharness.mozilla.purge import Pur
 from mozharness.mozilla.mock import MockMixin
 from mozharness.mozilla.secrets import SecretsMixin
 from mozharness.mozilla.signing import SigningMixin
 from mozharness.mozilla.mock import ERROR_MSGS as MOCK_ERROR_MSGS
 from mozharness.mozilla.testing.errors import TinderBoxPrintRe
 from mozharness.mozilla.testing.unittest import tbox_print_summary
 from mozharness.mozilla.updates.balrog import BalrogMixin
 from mozharness.mozilla.taskcluster_helper import Taskcluster
-from mozharness.base.python import VirtualenvMixin
+from mozharness.base.python import (
+    PerfherderResourceOptionsMixin,
+    VirtualenvMixin,
+)
 
 AUTOMATION_EXIT_CODES = EXIT_STATUS_DICT.values()
 AUTOMATION_EXIT_CODES.sort()
 
 MISSING_CFG_KEY_MSG = "The key '%s' could not be determined \
 Please add this to your config."
 
 ERROR_MSGS = {
@@ -582,17 +585,17 @@ def generate_build_ID():
 
 
 def generate_build_UID():
     return uuid.uuid4().hex
 
 
 class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
                   SigningMixin, VirtualenvMixin, MercurialScript,
-                  SecretsMixin):
+                  SecretsMixin, PerfherderResourceOptionsMixin):
     def __init__(self, **kwargs):
         # objdir is referenced in _query_abs_dirs() so let's make sure we
         # have that attribute before calling BaseScript.__init__
         self.objdir = None
         super(BuildScript, self).__init__(**kwargs)
         # epoch is only here to represent the start of the buildbot build
         # that this mozharn script came from. until I can grab bbot's
         # status.build.gettime()[0] this will have to do as a rough estimate
@@ -1858,16 +1861,17 @@ or run without that action (ie: --no-{ac
 
         if 'duration' not in resources:
             self.info('resource usage lacks duration; ignoring')
             return None
 
         data = {
             'name': 'build times',
             'value': resources['duration'],
+            'extraOptions': self.perfherder_resource_options(),
             'subtests': [],
         }
 
         for phase in resources['phases']:
             if 'duration' not in phase:
                 continue
             data['subtests'].append({
                 'name': phase['name'],