Bug 1252582 - Remove graph server output from talos, only post perfherder_data. r?wlach draft
authorJoel Maher <jmaher@mozilla.com>
Tue, 01 Mar 2016 13:58:49 -0800
changeset 336084 9ddfd7a22a9b7cbc3eb227afdf255913255a1fa4
parent 335957 1846e981637e0547c9b7935b2329580e03780259
child 515299 a33c8d803ed7e3b66edae7acd61dbd003235a61b
push id11967
push userjmaher@mozilla.com
push dateWed, 02 Mar 2016 10:46:01 +0000
reviewerswlach
bugs1252582
milestone47.0a1
Bug 1252582 - Remove graph server output from talos, only post perfherder_data. r?wlach MozReview-Commit-ID: Gg0eyVi6am7
testing/talos/talos/output.py
testing/talos/talos/post_file.py
testing/talos/talos/results.py
testing/talos/talos/run_tests.py
--- a/testing/talos/talos/output.py
+++ b/testing/talos/talos/output.py
@@ -2,21 +2,18 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """output formats for Talos"""
 
 import filter
 import json
-import post_file
-import time
 import utils
 
-from StringIO import StringIO
 from mozlog import get_proxy_logger
 
 # NOTE: we have a circular dependecy with output.py when we import results
 import results as TalosResults
 
 LOG = get_proxy_logger()
 
 
@@ -116,286 +113,16 @@ class Output(object):
     @classmethod
     def CanvasMark_Metric(cls, val_list):
         """CanvasMark benchmark score (NOTE: this is identical to JS_Metric)"""
         results = [i for i, j in val_list]
         LOG.info("CanvasMark benchmark")
         return sum(results)
 
 
-class GraphserverOutput(Output):
-
-    retries = 5  # number of times to attempt to contact graphserver
-    info_format = ['title', 'testname', 'branch_name', 'sourcestamp',
-                   'buildid', 'date']
-
-    @classmethod
-    def check(cls, urls):
-        # ensure results_url link exists
-        post_file.test_links(*urls)
-
-    def __call__(self):
-        """
-        results to send to graphserver:
-        construct all the strings of data, one string per test and one string
-        per counter
-        """
-
-        result_strings = []
-
-        info_dict = dict(
-            title=self.results.title,
-            date=self.results.date,
-            branch_name=self.results.browser_config['branch_name'],
-            sourcestamp=self.results.browser_config['sourcestamp'],
-            buildid=self.results.browser_config['buildid'],
-            browser_name=self.results.browser_config['browser_name'],
-            browser_version=self.results.browser_config['browser_version']
-        )
-
-        for test in self.results.results:
-            LOG.debug("Working with test: %s" % test.name())
-
-            # get full name of test
-            testname = test.name()
-            if test.format == 'tpformat':
-                # for some reason, we append the test extension to tp results
-                # but not ts
-                # http://hg.mozilla.org/build/talos/file/170c100911b6/talos
-                # /run_tests.py#l176
-                testname += test.extension()
-
-            LOG.debug("Generating results file: %s" % test.name())
-
-            # HACK: when running xperf, we upload xperf counters to the graph
-            # server but we do not want to
-            # upload the test results as they will confuse the graph server
-            if not (test.format == 'tpformat' and test.using_xperf):
-                vals = []
-                for result in test.results:
-                    filtered_val = result.values(testname,
-                                                 test.test_config['filters'])
-                    vals.extend([[i['value'], j] for i, j in filtered_val])
-                result_strings.append(self.construct_results(vals,
-                                                             testname=testname,
-                                                             **info_dict))
-
-            # counter results
-            for cd in test.all_counter_results:
-                for counter_type, values in cd.items():
-                    # get the counter name
-                    counterName = '%s_%s' % (test.name(),
-                                             self.shortName(counter_type))
-                    if not values:
-                        # failed to collect any data for this counter
-                        LOG.error(
-                            "No results collected for: " + counterName
-                        )
-# NOTE: we are not going to enforce this warning for now as this happens too
-# frequently: bugs 803413, 802475, 805925
-#                        raise utils.TalosError("Unable to proceed with missing
-# counter '%s'" % counterName)
-# (jhammel: we probably should do this in e.g. results.py vs in
-# graphserver-specific code anyway)
-
-                    # exclude counters whose values are tuples (bad for
-                    # graphserver)
-                    if len(values) > 0 and isinstance(values[0], list):
-                        print "Not uploading counter data for %s" % counterName
-                        print values
-                        continue
-
-                    if test.mainthread() and 'mainthreadio' in counterName:
-                        print ("Not uploading Mainthread IO data for %s"
-                               % counterName)
-                        print values
-                        continue
-
-                    # counter values
-                    vals = [[x, 'NULL'] for x in values]
-
-                    # append test name extension but only for tpformat tests
-                    if test.format == 'tpformat':
-                        counterName += test.extension()
-
-                    info = info_dict.copy()
-                    info['testname'] = counterName
-
-                    # append the counter string
-                    LOG.info(
-                        "Generating results file: %s" % counterName)
-                    result_strings.append(self.construct_results(vals, **info))
-
-        return result_strings
-
-    def responsiveness_test(self, testname):
-        """returns if the test is a responsiveness test"""
-        # XXX currently this just looks for the string
-        # 'responsiveness' in the test name.
-        # It would be nice to be more declarative about this
-        return 'responsiveness' in testname
-
-    def construct_results(self, vals, testname, **info):
-        """
-        return results string appropriate to graphserver
-        - vals: list of 2-tuples: [(val, page)
-        - kwargs: info necessary for self.info_format interpolation
-        see https://wiki.mozilla.org/Buildbot/Talos/DataFormat
-        """
-
-        info['testname'] = testname
-        info_format = self.info_format
-        responsiveness = self.responsiveness_test(testname)
-        _type = 'VALUES'
-        average = None
-        if responsiveness:
-            _type = 'AVERAGE'
-            average = self.responsiveness_Metric([val for (val, page) in vals])
-        elif testname.startswith('v8_7'):
-            _type = 'AVERAGE'
-            average = self.v8_Metric(vals)
-        elif testname.startswith('kraken'):
-            _type = 'AVERAGE'
-            average = self.JS_Metric(vals)
-        elif testname.startswith('tcanvasmark'):
-            _type = 'AVERAGE'
-            average = self.CanvasMark_Metric(vals)
-
-        # ensure that we have all of the info data available
-        missing = [i for i in info_format if i not in info]
-        if missing:
-            raise utils.TalosError("Missing keys: %s" % missing)
-        info = ','.join([str(info[key]) for key in info_format])
-
-        # write the data
-        buffer = StringIO()
-        buffer.write("START\n")
-        buffer.write("%s\n" % _type)
-        buffer.write('%s\n' % info)
-        if average is not None:
-            # write some kind of average
-            buffer.write("%s\n" % average)
-        else:
-            for i, (val, page) in enumerate(vals):
-                try:
-                    buffer.write("%d,%.2f,%s\n" % (i, float(val), page))
-                except ValueError:
-                    LOG.info(
-                        "We expected a numeric value and recieved '%s' instead"
-                        % val
-                    )
-                    pass
-
-        buffer.write("END")
-        return buffer.getvalue()
-
-    def process_Request(self, post):
-        """get links from the graphserver response"""
-        links = ""
-        for line in post.splitlines():
-            if line.find("RETURN\t") > -1:
-                line = line.replace("RETURN\t", "")
-                links += line + '\n'
-            LOG.debug("process_Request line: %s" % line)
-        if not links:
-            raise utils.TalosError("send failed, graph server says:\n%s"
-                                   % post)
-        return links
-
-    def post(self, results, server, path, scheme, tbpl_output):
-        """post results to the graphserver"""
-
-        links = []
-        wait_time = 5  # number of seconds between each attempt
-
-        for index, data_string in enumerate(results):
-            times = 0
-            msg = ""
-            while times < self.retries:
-                LOG.info(
-                    "Posting result %d of %d to %s://%s%s, attempt %d" % (
-                        index, len(results), scheme, server, path, times)
-                )
-                try:
-                    links.append(self.process_Request(
-                        post_file.post_multipart(server, path,
-                                                 files=[("filename",
-                                                         "data_string",
-                                                         data_string)])))
-                    break
-                except utils.TalosError, e:
-                    msg = str(e)
-                except Exception, e:
-                    msg = str(e)
-                times += 1
-                time.sleep(wait_time)
-                wait_time *= 2
-            else:
-                raise utils.TalosError(
-                    "Graph server unreachable (%d attempts)\n%s"
-                    % (self.retries, msg)
-                )
-
-        # add TBPL output
-        self.add_tbpl_output(links, tbpl_output, server, scheme)
-
-    def add_tbpl_output(self, links, tbpl_output, server, scheme):
-        """
-        add graphserver links such that TBPL can parse them.
-        graphserver returns a response like:
-
-          'tsvgr\tgraph.html#tests=[[224,113,14]]\ntsvgr\t2965.75\tgraph.html
-          #tests=[[224,113,14]]\n'
-
-        for each ts posted (tsvgr, in this case)
-        """
-
-        url_format = "%s://%s/%s"
-
-        # XXX this will not work for multiple URLs :(
-        tbpl_output.setdefault('graphserver', {})
-
-        # XXX link_format to be deprecated; see
-        # https://bugzilla.mozilla.org/show_bug.cgi?id=816634
-        link_format = '<a href=\'%s\'>%s</a>'
-
-        for response in links:
-
-            # parse the response:
-            # graphserver returns one of two responses.  For 'AVERAGE' payloads
-            # graphserver returns a line
-            # 'RETURN\t<test name>\t<value>\t<path segment>' :
-            # http://hg.mozilla.org/graphs/file/8884ef9418bf/server/pyfomatic
-            # /collect.py#l277
-            # For 'VALUES' payloads, graphserver prepends an additional line
-            # 'RETURN\t<test name>\t<path segment>' :
-            # http://hg.mozilla.org/graphs/file/8884ef9418bf/server/pyfomatic
-            # /collect.py#l274
-            # see https://bugzilla.mozilla.org/show_bug.cgi?id=816634#c56 for
-            # a more verbose explanation
-            lines = [line.strip() for line in response.strip().splitlines()]
-            assert len(lines) in (1, 2), """\
-Should have one line for 'AVERAGE' payloads,
-two lines for 'VALUES' payloads. You received:
-%s""" % lines
-            testname, result, path = lines[-1].split()
-            if self.isMemoryMetric(testname):
-                result = filesizeformat(result)
-
-            # add it to the output
-            url = url_format % (scheme, server, path)
-            tbpl_output['graphserver'][testname] = {'url': url,
-                                                    'result': result}
-
-            # output to legacy TBPL; to be deprecated, see
-            # https://bugzilla.mozilla.org/show_bug.cgi?id=816634
-            linkName = '%s: %s' % (testname, result)
-            print 'RETURN: %s' % link_format % (url, linkName)
-
-
 class PerfherderOutput(Output):
     def __init__(self, results):
         Output.__init__(self, results)
 
     def output(self, results, results_url, tbpl_output):
         """output to the a file if results_url starts with file://
         - results : json instance
         - results_url : file:// URL
@@ -540,10 +267,9 @@ class PerfherderOutput(Output):
                             varray = [float(v) for v in vals]
                             subtest['value'] = filter.mean(varray)
             if counter_subtests:
                 suites.append({'name': test.name(),
                                'subtests': counter_subtests})
         return test_results
 
 # available output formats
-formats = {'datazilla_urls': PerfherderOutput,
-           'results_urls': GraphserverOutput}
+formats = {'datazilla_urls': PerfherderOutput}
deleted file mode 100644
--- a/testing/talos/talos/post_file.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
-#   Submitter: Wade Leftwich
-# Licensing:
-#   according to http://aspn.activestate.com/ASPN/Cookbook/Python
-#   "Except where otherwise noted, recipes in the Python Cookbook
-#    are published under the Python license ."
-#   This recipe is covered under the Python license:
-#   http://www.python.org/license
-
-import httplib
-import mimetypes
-import urlparse
-import socket
-from socket import error, herror, gaierror, timeout
-socket.setdefaulttimeout(None)
-
-
-def link_exists(host, selector, scheme='http'):
-    url = "%s://%s%s" % (scheme, host, selector)
-    host, path = urlparse.urlsplit(url)[1:3]
-    found = 0
-    msg = "ping"
-    try:
-        h = httplib.HTTP(host)  # Make HTTPConnection Object
-        h.putrequest('HEAD', selector)
-        h.putheader('content-type', "text/plain")
-        h.putheader('content-length', str(len(msg)))
-        h.putheader('Host', host)
-        h.endheaders()
-        h.send(msg)
-
-        errcode, errmsg, headers = h.getreply()
-        if errcode == 200:
-            found = 1
-        else:
-            print ("WARNING: graph server Status %d %s : %s"
-                   % (errcode, errmsg, url))
-    except Exception, e:
-        print "WARNING: graph server ", e.__class__,  e, url
-    return found
-
-
-def test_links(*urls):
-    """ensure urls exist"""
-    # should this function return True/False, etc?
-
-    for url in urls:
-        url_split = urlparse.urlsplit(url)
-        scheme, server, path, _, _ = url_split
-        if (scheme in ('http', 'https') and
-            not link_exists(server, path, scheme)):  # noqa
-
-            print 'WARNING: graph server link does not exist: %s' % url
-
-
-def post_multipart(host, selector, fields=(), files=()):
-    """
-    Post fields and files to an http host as multipart/form-data.
-    fields is a sequence of (name, value) elements for regular form fields.
-    files is a sequence of (name, filename, value) elements for data to be
-    uploaded as files
-    Return the server's response page.
-    """
-    try:
-        host = host.replace('http://', '')
-        index = host.find('/')
-        if index > 0:
-            selector = '/'.join([host[index:], selector.lstrip('/')])
-            host = host[0:index]
-
-        # Summarized results to the official graph server
-        content_type, body = encode_multipart_formdata(fields, files)
-
-        headers = {
-            "Content-Type": content_type,
-            "Content-length": str(len(body)),
-            "Host": host,
-            "Accept": "text/plain"
-        }
-        conn = httplib.HTTPConnection(host)
-        conn.request("POST", selector, body, headers)
-        response = conn.getresponse()
-        return response.read()
-    except (httplib.HTTPException, error, herror, gaierror, timeout), e:
-        print "WARNING: graph server unreachable"
-        print "WARNING: " + str(e)
-        raise
-    except:
-        print "WARNING: graph server unreachable"
-        raise
-
-
-def encode_multipart_formdata(fields, files):
-    """
-    fields is a sequence of (name, value) elements for regular form fields.
-    files is a sequence of (name, filename, value) elements for data to be
-    uploaded as files
-    Return (content_type, body) ready for httplib.HTTP instance
-    """
-    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
-    CRLF = '\r\n'
-    L = []
-    for (key, value) in fields:
-        L.append('--' + BOUNDARY)
-        L.append('Content-Disposition: form-data; name="%s"' % key)
-        L.append('')
-        L.append(value)
-    for (key, filename, value) in files:
-        L.append('--' + BOUNDARY)
-        L.append('Content-Disposition: form-data; name="%s"; filename="%s"'
-                 % (key, filename))
-        L.append('Content-Type: %s' % get_content_type(filename))
-        L.append('')
-        L.append(value)
-    L.append('--' + BOUNDARY + '--')
-    L.append('')
-    body = CRLF.join(L)
-    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
-    return content_type, body
-
-
-def get_content_type(filename):
-    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
--- a/testing/talos/talos/results.py
+++ b/testing/talos/talos/results.py
@@ -14,24 +14,19 @@ import os
 import re
 import csv
 from talos import output, utils, filter
 
 
 class TalosResults(object):
     """Container class for Talos results"""
 
-    def __init__(self, title, date, browser_config):
+    def __init__(self):
         self.results = []
 
-        # info needed for graphserver
-        self.title = title
-        self.date = date
-        self.browser_config = browser_config
-
     def add(self, test_results):
         self.results.append(test_results)
 
     def check_output_formats(self, output_formats):
         """check output formats"""
 
         # ensure formats are available
         formats = output_formats.keys()
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -154,35 +154,28 @@ def run_tests(config, browser_config):
         date = int(time.mktime(time.strptime(testdate,
                                              '%a, %d %b %Y %H:%M:%S GMT')))
     else:
         date = int(time.time())
     LOG.debug("using testdate: %d" % date)
     LOG.debug("actual date: %d" % int(time.time()))
 
     # results container
-    talos_results = TalosResults(title=title,
-                                 date=date,
-                                 browser_config=browser_config)
+    talos_results = TalosResults()
 
     # results links
     if not browser_config['develop']:
         results_urls = dict(
-            # hardcoded, this will be removed soon anyway.
-            results_urls=['http://graphs.mozilla.org/server/collect.cgi'],
             # another hack; datazilla stands for Perfherder
             # and do not require url, but a non empty dict is required...
             datazilla_urls=['local.json'],
         )
     else:
         # local mode, output to files
-        results_urls = dict(
-            results_urls=[os.path.abspath('local.out')],
-            datazilla_urls=[os.path.abspath('local.json')]
-        )
+        results_urls = dict(datazilla_urls=[os.path.abspath('local.json')])
     talos_results.check_output_formats(results_urls)
 
     httpd = setup_webserver(browser_config['webserver'])
     httpd.start()
 
     testname = None
     # run the tests
     timer = utils.Timer()
@@ -219,18 +212,17 @@ def run_tests(config, browser_config):
     LOG.info("Completed test suite (%s)" % timer.elapsed())
 
     # output results
     if results_urls:
         talos_results.output(results_urls)
         if browser_config['develop']:
             print
             print ("Thanks for running Talos locally. Results are in"
-                   " %s and %s" % (results_urls['results_urls'],
-                                   results_urls['datazilla_urls']))
+                   " %s and %s" % (results_urls['datazilla_urls']))
 
     # we will stop running tests on a failed test, or we will return 0 for
     # green
     return 0
 
 
 def main(args=sys.argv[1:]):
     try: