Bug 1449195 - Process raptor test results and format for output (WIP)
MozReview-Commit-ID: AO8rPbnFGY5
--- a/testing/raptor/raptor/control_server.py
+++ b/testing/raptor/raptor/control_server.py
@@ -14,82 +14,93 @@ import threading
from mozlog import get_proxy_logger
LOG = get_proxy_logger(component='control_server')
here = os.path.abspath(os.path.dirname(__file__))
-class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+def MakeCustomHandlerClass(results_handler):
+
+ class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler, object):
+
+ def __init__(self, *args, **kwargs):
+ self.results_handler = results_handler
+ super(MyHandler, self).__init__(*args, **kwargs)
- def do_GET(self):
- # get handler, received request for test settings from web ext runner
- self.send_response(200)
- validFiles = ['raptor-firefox-tp6.json']
- head, tail = os.path.split(self.path)
- if tail in validFiles:
- LOG.info('reading test settings from ' + tail)
- try:
- with open(tail) as json_settings:
- self.send_header('Access-Control-Allow-Origin', '*')
- self.send_header('Content-type', 'application/json')
- self.end_headers()
- self.wfile.write(json.dumps(json.load(json_settings)))
- self.wfile.close()
- LOG.info('sent test settings to web ext runner')
- except Exception as ex:
- LOG.info('control server exception')
- LOG.info(ex)
- else:
- LOG.info('received request for unknown file: ' + self.path)
+ def do_GET(self):
+ # get handler, received request for test settings from web ext runner
+ self.send_response(200)
+ validFiles = ['raptor-firefox-tp6.json']
+ head, tail = os.path.split(self.path)
+ if tail in validFiles:
+ LOG.info('reading test settings from ' + tail)
+ try:
+ with open(tail) as json_settings:
+ self.send_header('Access-Control-Allow-Origin', '*')
+ self.send_header('Content-type', 'application/json')
+ self.end_headers()
+ self.wfile.write(json.dumps(json.load(json_settings)))
+ self.wfile.close()
+ LOG.info('sent test settings to web ext runner')
+ except Exception as ex:
+ LOG.info('control server exception')
+ LOG.info(ex)
+ else:
+ LOG.info('received request for unknown file: ' + self.path)
- def do_POST(self):
- # post handler, received something from webext
- self.send_response(200)
- self.send_header('Access-Control-Allow-Origin', '*')
- self.send_header('Content-type', 'text/html')
- self.end_headers()
- content_len = int(self.headers.getheader('content-length'))
- post_body = self.rfile.read(content_len)
- # could have received a status update or test results
- data = json.loads(post_body)
- LOG.info("received " + data['type'] + ": " + str(data['data']))
+ def do_POST(self):
+ # post handler, received something from webext
+ self.send_response(200)
+ self.send_header('Access-Control-Allow-Origin', '*')
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
+ content_len = int(self.headers.getheader('content-length'))
+ post_body = self.rfile.read(content_len)
+ # could have received a status update or test results
+ data = json.loads(post_body)
+ LOG.info("received " + data['type'] + ": " + str(data['data']))
+ if data['type'] == 'webext_results':
+ self.results_handler.add(data)
- def do_OPTIONS(self):
- self.send_response(200, "ok")
- self.send_header('Access-Control-Allow-Origin', '*')
- self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
- self.send_header("Access-Control-Allow-Headers", "X-Requested-With")
- self.send_header("Access-Control-Allow-Headers", "Content-Type")
- self.end_headers()
+ def do_OPTIONS(self):
+ self.send_response(200, "ok")
+ self.send_header('Access-Control-Allow-Origin', '*')
+ self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
+ self.send_header("Access-Control-Allow-Headers", "X-Requested-With")
+ self.send_header("Access-Control-Allow-Headers", "Content-Type")
+ self.end_headers()
+
+ return MyHandler
class RaptorControlServer():
"""Container class for Raptor Control Server"""
- def __init__(self):
+ def __init__(self, results_handler):
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.server = None
self._server_thread = None
self.port = None
+ self.results_handler = results_handler
def start(self):
config_dir = os.path.join(here, 'tests')
os.chdir(config_dir)
# pick a free port
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
self.port = sock.getsockname()[1]
sock.close()
server_address = ('', self.port)
server_class = BaseHTTPServer.HTTPServer
- handler_class = MyHandler
+ handler_class = MakeCustomHandlerClass(self.results_handler)
httpd = server_class(server_address, handler_class)
self._server_thread = threading.Thread(target=httpd.serve_forever)
self._server_thread.setDaemon(True) # don't hang on exit
self._server_thread.start()
LOG.info("raptor control server running on port %d..." % self.port)
self.server = httpd
--- a/testing/raptor/raptor/gen_test_config.py
+++ b/testing/raptor/raptor/gen_test_config.py
@@ -13,20 +13,22 @@ webext_dir = os.path.join(os.path.dirnam
LOG = get_proxy_logger(component="gen_test_url")
def gen_test_config(browser, test, cs_port):
LOG.info("writing test settings url background js, so webext can get it")
data = """// this file is auto-generated by raptor, do not edit directly
function getTestConfig() {
- return {"browser": "%s", "test_settings_url": "http://localhost:%d/%s.json"};
+ return {"browser": "%s",
+ "cs_port": "%d",
+ "test_settings_url": "http://localhost:%d/%s.json"};
}
-""" % (browser, cs_port, test)
+""" % (browser, cs_port, cs_port, test)
webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
file = open(webext_background_script, "w")
file.write(data)
file.close()
LOG.info("finished writing test config into webext")
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -20,18 +20,19 @@ from mozrunner import runners
here = os.path.abspath(os.path.dirname(__file__))
webext_dir = os.path.join(os.path.dirname(here), 'webext')
sys.path.insert(0, here)
from cmdline import parse_args
from control_server import RaptorControlServer
from gen_test_config import gen_test_config
from outputhandler import OutputHandler
+from manifest import get_raptor_test_list
from playback import get_playback
-from manifest import get_raptor_test_list
+from results import RaptorResultsHandler
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary):
self.config = {}
self.config['app'] = app
@@ -50,27 +51,30 @@ class Raptor(object):
with open(pref_file, 'r') as fh:
prefs = json.load(fh)
try:
self.profile = create_profile(self.config['app'], preferences=prefs)
except NotImplementedError:
self.profile = None
+ # create results holder
+ self.results_handler = RaptorResultsHandler()
+
# Create the runner
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(
binary, profile=self.profile, process_args=process_args)
def start_control_server(self):
- self.control_server = RaptorControlServer()
+ self.control_server = RaptorControlServer(self.results_handler)
self.control_server.start()
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
@@ -115,18 +119,18 @@ class Raptor(object):
proc.output.append(
"__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp"
% first_time)
proc.output.append(
"__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp"
% (int(time.time()) * 1000))
def process_results(self):
- self.log.info('todo: process results and dump in PERFHERDER_JSON blob')
- self.log.info('- or - do we want the control server to do that?')
+ self.results_handler.summarize()
+ self.results_handler.format_and_output()
def clean_up(self):
self.control_server.stop()
self.runner.stop()
self.log.info("raptor finished")
def main(args=sys.argv[1:]):
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/results.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# class to process, format, and report raptor test results
+# received from the raptor control server
+from __future__ import absolute_import
+
+from mozlog import get_proxy_logger
+
+LOG = get_proxy_logger(component='results-handler')
+
+
+class RaptorResultsHandler():
+ """Handle Raptor test results"""
+
+ def __init__(self):
+ self.results = []
+
+ def add(self, new_results):
+ # add to results
+ LOG.info("received results")
+ self.results.append(new_results)
+
+ def summarize(self):
+ # summarize results data - calculate mean etc.
+ LOG.info("self.results:")
+ LOG.info(str(self.results))
+
+ def format_and_output(self):
+ # format into PERFHERDER_DATA json, ensure valid json, and then output
+ LOG.info("TODO: output results in PERFERDER_DATA format")
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -12,16 +12,17 @@
// repo) or 'webkit/PerformanceTests' dir (for benchmarks) first run:
// 'python -m SimpleHTTPServer 8081'
// to serve out the pages that we want to prototype with. Also
// update the manifest content 'matches' accordingly
var browserName;
var ext;
var settingsURL = null;
+var cs_port = null;
var testType;
var pageCycles = 0;
var pageCycle = 0;
var pageCycleDelay = 1000;
var testURL;
var testTabID = 0;
var results = {"page": "", "measurements": {}};
var getHero = false;
@@ -281,17 +282,17 @@ function verifyResults() {
+ x + " but only have " + count);
}
}
postToControlServer("results", results);
}
function postToControlServer(msgType, msgData) {
// requires 'control server' running at port 8000 to receive results
- var url = "http://127.0.0.1:8000/";
+ var url = "http://127.0.0.1:" + cs_port + "/";
var client = new XMLHttpRequest();
client.onreadystatechange = function() {
if (client.readyState == XMLHttpRequest.DONE && client.status == 200) {
console.log("post success");
}
};
client.open("POST", url, true);
@@ -327,16 +328,17 @@ function cleanUp() {
if (browserName === "firefox")
window.dump("\n__raptor_shutdownBrowser\n");
}
function runner() {
let config = getTestConfig();
settingsURL = config.test_settings_url;
+ cs_port = config.cs_port;
browserName = config.browser;
getBrowserInfo().then(function() {
getTestSettings().then(function() {
if (testType == "benchmark") {
// webkit benchmark type of test
console.log("benchmark test start");
} else if (testType == "pageload") {
// standard pageload test