Bug 1252582 - Remove graph server output from talos, only post perfherder_data. r?wlach
MozReview-Commit-ID: Gg0eyVi6am7
--- a/testing/talos/talos/output.py
+++ b/testing/talos/talos/output.py
@@ -2,21 +2,18 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""output formats for Talos"""
import filter
import json
-import post_file
-import time
import utils
-from StringIO import StringIO
from mozlog import get_proxy_logger
# NOTE: we have a circular dependecy with output.py when we import results
import results as TalosResults
LOG = get_proxy_logger()
@@ -116,286 +113,16 @@ class Output(object):
@classmethod
def CanvasMark_Metric(cls, val_list):
"""CanvasMark benchmark score (NOTE: this is identical to JS_Metric)"""
results = [i for i, j in val_list]
LOG.info("CanvasMark benchmark")
return sum(results)
-class GraphserverOutput(Output):
-
- retries = 5 # number of times to attempt to contact graphserver
- info_format = ['title', 'testname', 'branch_name', 'sourcestamp',
- 'buildid', 'date']
-
- @classmethod
- def check(cls, urls):
- # ensure results_url link exists
- post_file.test_links(*urls)
-
- def __call__(self):
- """
- results to send to graphserver:
- construct all the strings of data, one string per test and one string
- per counter
- """
-
- result_strings = []
-
- info_dict = dict(
- title=self.results.title,
- date=self.results.date,
- branch_name=self.results.browser_config['branch_name'],
- sourcestamp=self.results.browser_config['sourcestamp'],
- buildid=self.results.browser_config['buildid'],
- browser_name=self.results.browser_config['browser_name'],
- browser_version=self.results.browser_config['browser_version']
- )
-
- for test in self.results.results:
- LOG.debug("Working with test: %s" % test.name())
-
- # get full name of test
- testname = test.name()
- if test.format == 'tpformat':
- # for some reason, we append the test extension to tp results
- # but not ts
- # http://hg.mozilla.org/build/talos/file/170c100911b6/talos
- # /run_tests.py#l176
- testname += test.extension()
-
- LOG.debug("Generating results file: %s" % test.name())
-
- # HACK: when running xperf, we upload xperf counters to the graph
- # server but we do not want to
- # upload the test results as they will confuse the graph server
- if not (test.format == 'tpformat' and test.using_xperf):
- vals = []
- for result in test.results:
- filtered_val = result.values(testname,
- test.test_config['filters'])
- vals.extend([[i['value'], j] for i, j in filtered_val])
- result_strings.append(self.construct_results(vals,
- testname=testname,
- **info_dict))
-
- # counter results
- for cd in test.all_counter_results:
- for counter_type, values in cd.items():
- # get the counter name
- counterName = '%s_%s' % (test.name(),
- self.shortName(counter_type))
- if not values:
- # failed to collect any data for this counter
- LOG.error(
- "No results collected for: " + counterName
- )
-# NOTE: we are not going to enforce this warning for now as this happens too
-# frequently: bugs 803413, 802475, 805925
-# raise utils.TalosError("Unable to proceed with missing
-# counter '%s'" % counterName)
-# (jhammel: we probably should do this in e.g. results.py vs in
-# graphserver-specific code anyway)
-
- # exclude counters whose values are tuples (bad for
- # graphserver)
- if len(values) > 0 and isinstance(values[0], list):
- print "Not uploading counter data for %s" % counterName
- print values
- continue
-
- if test.mainthread() and 'mainthreadio' in counterName:
- print ("Not uploading Mainthread IO data for %s"
- % counterName)
- print values
- continue
-
- # counter values
- vals = [[x, 'NULL'] for x in values]
-
- # append test name extension but only for tpformat tests
- if test.format == 'tpformat':
- counterName += test.extension()
-
- info = info_dict.copy()
- info['testname'] = counterName
-
- # append the counter string
- LOG.info(
- "Generating results file: %s" % counterName)
- result_strings.append(self.construct_results(vals, **info))
-
- return result_strings
-
- def responsiveness_test(self, testname):
- """returns if the test is a responsiveness test"""
- # XXX currently this just looks for the string
- # 'responsiveness' in the test name.
- # It would be nice to be more declarative about this
- return 'responsiveness' in testname
-
- def construct_results(self, vals, testname, **info):
- """
- return results string appropriate to graphserver
- - vals: list of 2-tuples: [(val, page)
- - kwargs: info necessary for self.info_format interpolation
- see https://wiki.mozilla.org/Buildbot/Talos/DataFormat
- """
-
- info['testname'] = testname
- info_format = self.info_format
- responsiveness = self.responsiveness_test(testname)
- _type = 'VALUES'
- average = None
- if responsiveness:
- _type = 'AVERAGE'
- average = self.responsiveness_Metric([val for (val, page) in vals])
- elif testname.startswith('v8_7'):
- _type = 'AVERAGE'
- average = self.v8_Metric(vals)
- elif testname.startswith('kraken'):
- _type = 'AVERAGE'
- average = self.JS_Metric(vals)
- elif testname.startswith('tcanvasmark'):
- _type = 'AVERAGE'
- average = self.CanvasMark_Metric(vals)
-
- # ensure that we have all of the info data available
- missing = [i for i in info_format if i not in info]
- if missing:
- raise utils.TalosError("Missing keys: %s" % missing)
- info = ','.join([str(info[key]) for key in info_format])
-
- # write the data
- buffer = StringIO()
- buffer.write("START\n")
- buffer.write("%s\n" % _type)
- buffer.write('%s\n' % info)
- if average is not None:
- # write some kind of average
- buffer.write("%s\n" % average)
- else:
- for i, (val, page) in enumerate(vals):
- try:
- buffer.write("%d,%.2f,%s\n" % (i, float(val), page))
- except ValueError:
- LOG.info(
- "We expected a numeric value and recieved '%s' instead"
- % val
- )
- pass
-
- buffer.write("END")
- return buffer.getvalue()
-
- def process_Request(self, post):
- """get links from the graphserver response"""
- links = ""
- for line in post.splitlines():
- if line.find("RETURN\t") > -1:
- line = line.replace("RETURN\t", "")
- links += line + '\n'
- LOG.debug("process_Request line: %s" % line)
- if not links:
- raise utils.TalosError("send failed, graph server says:\n%s"
- % post)
- return links
-
- def post(self, results, server, path, scheme, tbpl_output):
- """post results to the graphserver"""
-
- links = []
- wait_time = 5 # number of seconds between each attempt
-
- for index, data_string in enumerate(results):
- times = 0
- msg = ""
- while times < self.retries:
- LOG.info(
- "Posting result %d of %d to %s://%s%s, attempt %d" % (
- index, len(results), scheme, server, path, times)
- )
- try:
- links.append(self.process_Request(
- post_file.post_multipart(server, path,
- files=[("filename",
- "data_string",
- data_string)])))
- break
- except utils.TalosError, e:
- msg = str(e)
- except Exception, e:
- msg = str(e)
- times += 1
- time.sleep(wait_time)
- wait_time *= 2
- else:
- raise utils.TalosError(
- "Graph server unreachable (%d attempts)\n%s"
- % (self.retries, msg)
- )
-
- # add TBPL output
- self.add_tbpl_output(links, tbpl_output, server, scheme)
-
- def add_tbpl_output(self, links, tbpl_output, server, scheme):
- """
- add graphserver links such that TBPL can parse them.
- graphserver returns a response like:
-
- 'tsvgr\tgraph.html#tests=[[224,113,14]]\ntsvgr\t2965.75\tgraph.html
- #tests=[[224,113,14]]\n'
-
- for each ts posted (tsvgr, in this case)
- """
-
- url_format = "%s://%s/%s"
-
- # XXX this will not work for multiple URLs :(
- tbpl_output.setdefault('graphserver', {})
-
- # XXX link_format to be deprecated; see
- # https://bugzilla.mozilla.org/show_bug.cgi?id=816634
- link_format = '<a href=\'%s\'>%s</a>'
-
- for response in links:
-
- # parse the response:
- # graphserver returns one of two responses. For 'AVERAGE' payloads
- # graphserver returns a line
- # 'RETURN\t<test name>\t<value>\t<path segment>' :
- # http://hg.mozilla.org/graphs/file/8884ef9418bf/server/pyfomatic
- # /collect.py#l277
- # For 'VALUES' payloads, graphserver prepends an additional line
- # 'RETURN\t<test name>\t<path segment>' :
- # http://hg.mozilla.org/graphs/file/8884ef9418bf/server/pyfomatic
- # /collect.py#l274
- # see https://bugzilla.mozilla.org/show_bug.cgi?id=816634#c56 for
- # a more verbose explanation
- lines = [line.strip() for line in response.strip().splitlines()]
- assert len(lines) in (1, 2), """\
-Should have one line for 'AVERAGE' payloads,
-two lines for 'VALUES' payloads. You received:
-%s""" % lines
- testname, result, path = lines[-1].split()
- if self.isMemoryMetric(testname):
- result = filesizeformat(result)
-
- # add it to the output
- url = url_format % (scheme, server, path)
- tbpl_output['graphserver'][testname] = {'url': url,
- 'result': result}
-
- # output to legacy TBPL; to be deprecated, see
- # https://bugzilla.mozilla.org/show_bug.cgi?id=816634
- linkName = '%s: %s' % (testname, result)
- print 'RETURN: %s' % link_format % (url, linkName)
-
-
class PerfherderOutput(Output):
def __init__(self, results):
Output.__init__(self, results)
def output(self, results, results_url, tbpl_output):
"""output to the a file if results_url starts with file://
- results : json instance
- results_url : file:// URL
@@ -540,10 +267,9 @@ class PerfherderOutput(Output):
varray = [float(v) for v in vals]
subtest['value'] = filter.mean(varray)
if counter_subtests:
suites.append({'name': test.name(),
'subtests': counter_subtests})
return test_results
# available output formats
-formats = {'datazilla_urls': PerfherderOutput,
- 'results_urls': GraphserverOutput}
+formats = {'datazilla_urls': PerfherderOutput}
deleted file mode 100644
--- a/testing/talos/talos/post_file.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
-# Submitter: Wade Leftwich
-# Licensing:
-# according to http://aspn.activestate.com/ASPN/Cookbook/Python
-# "Except where otherwise noted, recipes in the Python Cookbook
-# are published under the Python license ."
-# This recipe is covered under the Python license:
-# http://www.python.org/license
-
-import httplib
-import mimetypes
-import urlparse
-import socket
-from socket import error, herror, gaierror, timeout
-socket.setdefaulttimeout(None)
-
-
-def link_exists(host, selector, scheme='http'):
- url = "%s://%s%s" % (scheme, host, selector)
- host, path = urlparse.urlsplit(url)[1:3]
- found = 0
- msg = "ping"
- try:
- h = httplib.HTTP(host) # Make HTTPConnection Object
- h.putrequest('HEAD', selector)
- h.putheader('content-type', "text/plain")
- h.putheader('content-length', str(len(msg)))
- h.putheader('Host', host)
- h.endheaders()
- h.send(msg)
-
- errcode, errmsg, headers = h.getreply()
- if errcode == 200:
- found = 1
- else:
- print ("WARNING: graph server Status %d %s : %s"
- % (errcode, errmsg, url))
- except Exception, e:
- print "WARNING: graph server ", e.__class__, e, url
- return found
-
-
-def test_links(*urls):
- """ensure urls exist"""
- # should this function return True/False, etc?
-
- for url in urls:
- url_split = urlparse.urlsplit(url)
- scheme, server, path, _, _ = url_split
- if (scheme in ('http', 'https') and
- not link_exists(server, path, scheme)): # noqa
-
- print 'WARNING: graph server link does not exist: %s' % url
-
-
-def post_multipart(host, selector, fields=(), files=()):
- """
- Post fields and files to an http host as multipart/form-data.
- fields is a sequence of (name, value) elements for regular form fields.
- files is a sequence of (name, filename, value) elements for data to be
- uploaded as files
- Return the server's response page.
- """
- try:
- host = host.replace('http://', '')
- index = host.find('/')
- if index > 0:
- selector = '/'.join([host[index:], selector.lstrip('/')])
- host = host[0:index]
-
- # Summarized results to the official graph server
- content_type, body = encode_multipart_formdata(fields, files)
-
- headers = {
- "Content-Type": content_type,
- "Content-length": str(len(body)),
- "Host": host,
- "Accept": "text/plain"
- }
- conn = httplib.HTTPConnection(host)
- conn.request("POST", selector, body, headers)
- response = conn.getresponse()
- return response.read()
- except (httplib.HTTPException, error, herror, gaierror, timeout), e:
- print "WARNING: graph server unreachable"
- print "WARNING: " + str(e)
- raise
- except:
- print "WARNING: graph server unreachable"
- raise
-
-
-def encode_multipart_formdata(fields, files):
- """
- fields is a sequence of (name, value) elements for regular form fields.
- files is a sequence of (name, filename, value) elements for data to be
- uploaded as files
- Return (content_type, body) ready for httplib.HTTP instance
- """
- BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
- CRLF = '\r\n'
- L = []
- for (key, value) in fields:
- L.append('--' + BOUNDARY)
- L.append('Content-Disposition: form-data; name="%s"' % key)
- L.append('')
- L.append(value)
- for (key, filename, value) in files:
- L.append('--' + BOUNDARY)
- L.append('Content-Disposition: form-data; name="%s"; filename="%s"'
- % (key, filename))
- L.append('Content-Type: %s' % get_content_type(filename))
- L.append('')
- L.append(value)
- L.append('--' + BOUNDARY + '--')
- L.append('')
- body = CRLF.join(L)
- content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
- return content_type, body
-
-
-def get_content_type(filename):
- return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
--- a/testing/talos/talos/results.py
+++ b/testing/talos/talos/results.py
@@ -14,24 +14,19 @@ import os
import re
import csv
from talos import output, utils, filter
class TalosResults(object):
"""Container class for Talos results"""
- def __init__(self, title, date, browser_config):
+ def __init__(self):
self.results = []
- # info needed for graphserver
- self.title = title
- self.date = date
- self.browser_config = browser_config
-
def add(self, test_results):
self.results.append(test_results)
def check_output_formats(self, output_formats):
"""check output formats"""
# ensure formats are available
formats = output_formats.keys()
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -154,35 +154,28 @@ def run_tests(config, browser_config):
date = int(time.mktime(time.strptime(testdate,
'%a, %d %b %Y %H:%M:%S GMT')))
else:
date = int(time.time())
LOG.debug("using testdate: %d" % date)
LOG.debug("actual date: %d" % int(time.time()))
# results container
- talos_results = TalosResults(title=title,
- date=date,
- browser_config=browser_config)
+ talos_results = TalosResults()
# results links
if not browser_config['develop']:
results_urls = dict(
- # hardcoded, this will be removed soon anyway.
- results_urls=['http://graphs.mozilla.org/server/collect.cgi'],
# another hack; datazilla stands for Perfherder
# and do not require url, but a non empty dict is required...
datazilla_urls=['local.json'],
)
else:
# local mode, output to files
- results_urls = dict(
- results_urls=[os.path.abspath('local.out')],
- datazilla_urls=[os.path.abspath('local.json')]
- )
+ results_urls = dict(datazilla_urls=[os.path.abspath('local.json')])
talos_results.check_output_formats(results_urls)
httpd = setup_webserver(browser_config['webserver'])
httpd.start()
testname = None
# run the tests
timer = utils.Timer()
@@ -219,18 +212,17 @@ def run_tests(config, browser_config):
LOG.info("Completed test suite (%s)" % timer.elapsed())
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop']:
print
print ("Thanks for running Talos locally. Results are in"
- " %s and %s" % (results_urls['results_urls'],
- results_urls['datazilla_urls']))
+ " %s and %s" % (results_urls['datazilla_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0
def main(args=sys.argv[1:]):
try: