--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -6,28 +6,26 @@
# ***** END LICENSE BLOCK *****
'''Python usage, esp. virtualenv.
'''
import distutils.version
import os
import subprocess
import sys
-import time
import json
import socket
import traceback
import urlparse
import mozharness
from mozharness.base.script import (
PostScriptAction,
PostScriptRun,
PreScriptAction,
- PreScriptRun,
)
from mozharness.base.errors import VirtualenvErrorList
from mozharness.base.log import WARNING, FATAL
from mozharness.mozilla.proxxy import Proxxy
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
'external_tools',
@@ -752,228 +750,14 @@ class ResourceMonitoringMixin(object):
start_time, end_time = rm.phases[phase]
cpu_percent, cpu_times, io, swap = resources(phase)
log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
def _tinderbox_print(self, message):
self.info('TinderboxPrint: %s' % message)
-class InfluxRecordingMixin(object):
- """Provides InfluxDB stat recording to scripts.
-
- This class records stats to an InfluxDB server, if enabled. Stat recording
- is enabled in a script by inheriting from this class, and adding an
- influxdb_credentials line to the influx_credentials_file (usually oauth.txt
- in automation). This line should look something like:
-
- influxdb_credentials = 'http://goldiewilson-onepointtwentyone-1.c.influxdb.com:8086/db/DBNAME/series?u=DBUSERNAME&p=DBPASSWORD'
-
- Where DBNAME, DBUSERNAME, and DBPASSWORD correspond to the database name,
- and user/pw credentials for recording to the database. The stats from
- mozharness are recorded in the 'mozharness' table.
- """
-
- @PreScriptRun
- def influxdb_recording_init(self):
- self.recording = False
- self.post = None
- self.posturl = None
- self.build_metrics_summary = None
- self.res_props = self.config.get('build_resources_path') % self.query_abs_dirs()
- self.info("build_resources.json path: %s" % self.res_props)
- if self.res_props:
- self.rmtree(self.res_props)
-
- try:
- site_packages_path = self.query_python_site_packages_path()
- if site_packages_path not in sys.path:
- sys.path.append(site_packages_path)
-
- self.post = get_tlsv1_post()
-
- auth = os.path.join(os.getcwd(), self.config['influx_credentials_file'])
- if not os.path.exists(auth):
- self.warning("Unable to start influxdb recording: %s not found" % (auth,))
- return
- credentials = {}
- execfile(auth, credentials)
- if 'influxdb_credentials' in credentials:
- self.posturl = credentials['influxdb_credentials']
- self.recording = True
- else:
- self.warning("Unable to start influxdb recording: no credentials")
- return
-
- except Exception:
- # The exact reason for failing to start stats doesn't really matter.
- # If anything fails, we just won't record stats for this job.
- self.warning("Unable to start influxdb recording: %s" %
- traceback.format_exc())
- return
-
- @PreScriptAction
- def influxdb_recording_pre_action(self, action):
- if not self.recording:
- return
-
- self.start_time = time.time()
-
- @PostScriptAction
- def influxdb_recording_post_action(self, action, success=None):
- if not self.recording:
- return
-
- elapsed_time = time.time() - self.start_time
-
- c = {}
- p = {}
- if self.buildbot_config:
- c = self.buildbot_config.get('properties', {})
- if self.buildbot_properties:
- p = self.buildbot_properties
- self.record_influx_stat([{
- "points": [[
- action,
- elapsed_time,
- c.get('buildername'),
- c.get('product'),
- c.get('platform'),
- c.get('branch'),
- c.get('slavename'),
- c.get('revision'),
- p.get('gaia_revision'),
- c.get('buildid'),
- ]],
- "name": "mozharness",
- "columns": [
- "action",
- "runtime",
- "buildername",
- "product",
- "platform",
- "branch",
- "slavename",
- "gecko_revision",
- "gaia_revision",
- "buildid",
- ],
- }])
-
- def _get_resource_usage(self, res, name, iolen, cpulen):
- c = {}
- p = {}
- if self.buildbot_config:
- c = self.buildbot_config.get('properties', {})
- if self.buildbot_properties:
- p = self.buildbot_properties
-
- data = [
- # Build properties
- c.get('buildername'),
- c.get('product'),
- c.get('platform'),
- c.get('branch'),
- c.get('slavename'),
- c.get('revision'),
- p.get('gaia_revision'),
- c.get('buildid'),
-
- # Mach step properties
- name,
- res.get('start'),
- res.get('end'),
- res.get('duration'),
- res.get('cpu_percent'),
- ]
- # The io and cpu_times fields are arrays, though they aren't always
- # present if a step completes before resource utilization is measured.
- # We add the arrays if they exist, otherwise we just do an array of None
- # to fill up the stat point.
- data.extend(res.get('io', [None] * iolen))
- data.extend(res.get('cpu_times', [None] * cpulen))
- return data
-
- @PostScriptAction('build')
- def record_mach_stats(self, action, success=None):
- if not os.path.exists(self.res_props):
- self.info('No build_resources.json found, not logging stats')
- return
- with open(self.res_props) as fh:
- resources = json.load(fh)
- data = {
- "points": [
- ],
- "name": "mach",
- "columns": [
- # Build properties
- "buildername",
- "product",
- "platform",
- "branch",
- "slavename",
- "gecko_revision",
- "gaia_revision",
- "buildid",
-
- # Mach step properties
- "name",
- "start",
- "end",
- "duration",
- "cpu_percent",
- ],
- }
-
- # The io and cpu_times fields aren't static - they may vary based
- # on the specific platform being measured. Mach records the field
- # names, which we use as the column names here.
- data['columns'].extend(resources['io_fields'])
- data['columns'].extend(resources['cpu_times_fields'])
- iolen = len(resources['io_fields'])
- cpulen = len(resources['cpu_times_fields'])
-
- if 'duration' in resources:
- self.build_metrics_summary = {
- 'name': 'build times',
- 'value': resources['duration'],
- 'subtests': [],
- }
-
- # The top-level data has the overall resource usage, which we record
- # under the name 'TOTAL' to separate it from the individual phases.
- data['points'].append(self._get_resource_usage(resources, 'TOTAL', iolen, cpulen))
-
- # Each phases also has the same resource stats as the top-level.
- for phase in resources['phases']:
- data['points'].append(self._get_resource_usage(phase, phase['name'], iolen, cpulen))
- if 'duration' not in phase:
- self.build_metrics_summary = None
- elif self.build_metrics_summary:
- self.build_metrics_summary['subtests'].append({
- 'name': phase['name'],
- 'value': phase['duration'],
- })
-
- self.record_influx_stat([data])
-
- def record_influx_stat(self, json_data):
- if not self.recording:
- return
- try:
- r = self.post(self.posturl, data=json.dumps(json_data), timeout=5)
- if r.status_code != 200:
- self.warning("Failed to log stats. Return code = %i, stats = %s" % (r.status_code, json_data))
-
- # Disable recording for the rest of this job. Even if it's just
- # intermittent, we don't want to keep the build from progressing.
- self.recording = False
- except Exception, e:
- self.warning('Failed to log stats. Exception = %s' % str(e))
- self.recording = False
-
-
# __main__ {{{1
if __name__ == '__main__':
'''TODO: unit tests.
'''
pass
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ b/testing/mozharness/scripts/fx_desktop_build.py
@@ -71,17 +71,16 @@ class FxDesktopBuild(BuildScript, TryToo
"%(objdir)s/dist/fennec*",
"%(objdir)s/dist/seamonkey*",
"%(objdir)s/dist/thunderbird*",
"%(objdir)s/dist/install/sea/*.exe"
],
'stage_product': 'firefox',
'platform_supports_post_upload_to_latest': True,
'latest_mar_dir': '/pub/mozilla.org/firefox/nightly/latest-%(branch)s',
- 'influx_credentials_file': 'oauth.txt',
'build_resources_path': '%(abs_src_dir)s/obj-firefox/.mozbuild/build_resources.json',
'nightly_promotion_branches': ['mozilla-central', 'mozilla-aurora'],
# try will overwrite these
'clone_with_purge': False,
'clone_by_revision': False,
'tinderbox_build_dir': None,
'to_tinderbox_dated': True,