Bug 1449195 - Process raptor test results and format for output (WIP) draft
authorRob Wood <rwood@mozilla.com>
Wed, 02 May 2018 15:17:18 -0400
changeset 791127 66a88aba573c5bdfe01c6b9cea2cd56f07dce07c
parent 791094 bc311a04727333f0dab9aaa650ece30181ddf0f3
push id108707
push userrwood@mozilla.com
push dateThu, 03 May 2018 15:40:16 +0000
bugs1449195
milestone61.0a1
Bug 1449195 - Process raptor test results and format for output (WIP) MozReview-Commit-ID: AO8rPbnFGY5
testing/raptor/raptor/control_server.py
testing/raptor/raptor/gen_test_config.py
testing/raptor/raptor/raptor.py
testing/raptor/raptor/results.py
testing/raptor/webext/raptor/runner.js
--- a/testing/raptor/raptor/control_server.py
+++ b/testing/raptor/raptor/control_server.py
@@ -14,82 +14,93 @@ import threading
 
 from mozlog import get_proxy_logger
 
 LOG = get_proxy_logger(component='control_server')
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
-class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+def MakeCustomHandlerClass(results_handler):
+
+    class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler, object):
+
+        def __init__(self, *args, **kwargs):
+            self.results_handler = results_handler
+            super(MyHandler, self).__init__(*args, **kwargs)
 
-    def do_GET(self):
-        # get handler, received request for test settings from web ext runner
-        self.send_response(200)
-        validFiles = ['raptor-firefox-tp6.json']
-        head, tail = os.path.split(self.path)
-        if tail in validFiles:
-            LOG.info('reading test settings from ' + tail)
-            try:
-                with open(tail) as json_settings:
-                    self.send_header('Access-Control-Allow-Origin', '*')
-                    self.send_header('Content-type', 'application/json')
-                    self.end_headers()
-                    self.wfile.write(json.dumps(json.load(json_settings)))
-                    self.wfile.close()
-                    LOG.info('sent test settings to web ext runner')
-            except Exception as ex:
-                LOG.info('control server exception')
-                LOG.info(ex)
-        else:
-            LOG.info('received request for unknown file: ' + self.path)
+        def do_GET(self):
+            # get handler, received request for test settings from web ext runner
+            self.send_response(200)
+            validFiles = ['raptor-firefox-tp6.json']
+            head, tail = os.path.split(self.path)
+            if tail in validFiles:
+                LOG.info('reading test settings from ' + tail)
+                try:
+                    with open(tail) as json_settings:
+                        self.send_header('Access-Control-Allow-Origin', '*')
+                        self.send_header('Content-type', 'application/json')
+                        self.end_headers()
+                        self.wfile.write(json.dumps(json.load(json_settings)))
+                        self.wfile.close()
+                        LOG.info('sent test settings to web ext runner')
+                except Exception as ex:
+                    LOG.info('control server exception')
+                    LOG.info(ex)
+            else:
+                LOG.info('received request for unknown file: ' + self.path)
 
-    def do_POST(self):
-        # post handler, received something from webext
-        self.send_response(200)
-        self.send_header('Access-Control-Allow-Origin', '*')
-        self.send_header('Content-type', 'text/html')
-        self.end_headers()
-        content_len = int(self.headers.getheader('content-length'))
-        post_body = self.rfile.read(content_len)
-        # could have received a status update or test results
-        data = json.loads(post_body)
-        LOG.info("received " + data['type'] + ": " + str(data['data']))
+        def do_POST(self):
+            # post handler, received something from webext
+            self.send_response(200)
+            self.send_header('Access-Control-Allow-Origin', '*')
+            self.send_header('Content-type', 'text/html')
+            self.end_headers()
+            content_len = int(self.headers.getheader('content-length'))
+            post_body = self.rfile.read(content_len)
+            # could have received a status update or test results
+            data = json.loads(post_body)
+            LOG.info("received " + data['type'] + ": " + str(data['data']))
+            if data['type'] == 'webext_results':
+                self.results_handler.add(data)
 
-    def do_OPTIONS(self):
-        self.send_response(200, "ok")
-        self.send_header('Access-Control-Allow-Origin', '*')
-        self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
-        self.send_header("Access-Control-Allow-Headers", "X-Requested-With")
-        self.send_header("Access-Control-Allow-Headers", "Content-Type")
-        self.end_headers()
+        def do_OPTIONS(self):
+            self.send_response(200, "ok")
+            self.send_header('Access-Control-Allow-Origin', '*')
+            self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
+            self.send_header("Access-Control-Allow-Headers", "X-Requested-With")
+            self.send_header("Access-Control-Allow-Headers", "Content-Type")
+            self.end_headers()
+
+    return MyHandler
 
 
 class RaptorControlServer():
     """Container class for Raptor Control Server"""
 
-    def __init__(self):
+    def __init__(self, results_handler):
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.server = None
         self._server_thread = None
         self.port = None
+        self.results_handler = results_handler
 
     def start(self):
         config_dir = os.path.join(here, 'tests')
         os.chdir(config_dir)
 
         # pick a free port
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.bind(('', 0))
         self.port = sock.getsockname()[1]
         sock.close()
         server_address = ('', self.port)
 
         server_class = BaseHTTPServer.HTTPServer
-        handler_class = MyHandler
+        handler_class = MakeCustomHandlerClass(self.results_handler)
 
         httpd = server_class(server_address, handler_class)
 
         self._server_thread = threading.Thread(target=httpd.serve_forever)
         self._server_thread.setDaemon(True)  # don't hang on exit
         self._server_thread.start()
         LOG.info("raptor control server running on port %d..." % self.port)
         self.server = httpd
--- a/testing/raptor/raptor/gen_test_config.py
+++ b/testing/raptor/raptor/gen_test_config.py
@@ -13,20 +13,22 @@ webext_dir = os.path.join(os.path.dirnam
 LOG = get_proxy_logger(component="gen_test_url")
 
 
 def gen_test_config(browser, test, cs_port):
     LOG.info("writing test settings url background js, so webext can get it")
 
     data = """// this file is auto-generated by raptor, do not edit directly
 function getTestConfig() {
-    return {"browser": "%s", "test_settings_url": "http://localhost:%d/%s.json"};
+    return {"browser": "%s",
+            "cs_port": "%d",
+            "test_settings_url": "http://localhost:%d/%s.json"};
 }
 
-""" % (browser, cs_port, test)
+""" % (browser, cs_port, cs_port, test)
 
     webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
 
     file = open(webext_background_script, "w")
     file.write(data)
     file.close()
 
     LOG.info("finished writing test config into webext")
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -20,18 +20,19 @@ from mozrunner import runners
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext')
 sys.path.insert(0, here)
 
 from cmdline import parse_args
 from control_server import RaptorControlServer
 from gen_test_config import gen_test_config
 from outputhandler import OutputHandler
+from manifest import get_raptor_test_list
 from playback import get_playback
-from manifest import get_raptor_test_list
+from results import RaptorResultsHandler
 
 
 class Raptor(object):
     """Container class for Raptor"""
 
     def __init__(self, app, binary):
         self.config = {}
         self.config['app'] = app
@@ -50,27 +51,30 @@ class Raptor(object):
             with open(pref_file, 'r') as fh:
                 prefs = json.load(fh)
 
         try:
             self.profile = create_profile(self.config['app'], preferences=prefs)
         except NotImplementedError:
             self.profile = None
 
+        # create results holder
+        self.results_handler = RaptorResultsHandler()
+
         # Create the runner
         self.output_handler = OutputHandler()
         process_args = {
             'processOutputLine': [self.output_handler],
         }
         runner_cls = runners[app]
         self.runner = runner_cls(
             binary, profile=self.profile, process_args=process_args)
 
     def start_control_server(self):
-        self.control_server = RaptorControlServer()
+        self.control_server = RaptorControlServer(self.results_handler)
         self.control_server.start()
 
     def get_playback_config(self, test):
         self.config['playback_tool'] = test.get('playback')
         self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
         self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
         _key = 'playback_binary_zip_%s' % self.config['platform']
         self.config['playback_binary_zip'] = test.get(_key, None)
@@ -115,18 +119,18 @@ class Raptor(object):
         proc.output.append(
             "__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp"
             % first_time)
         proc.output.append(
             "__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp"
             % (int(time.time()) * 1000))
 
     def process_results(self):
-        self.log.info('todo: process results and dump in PERFHERDER_JSON blob')
-        self.log.info('- or - do we want the control server to do that?')
+        self.results_handler.summarize()
+        self.results_handler.format_and_output()
 
     def clean_up(self):
         self.control_server.stop()
         self.runner.stop()
         self.log.info("raptor finished")
 
 
 def main(args=sys.argv[1:]):
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/results.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# class to process, format, and report raptor test results
+# received from the raptor control server
+from __future__ import absolute_import
+
+from mozlog import get_proxy_logger
+
+LOG = get_proxy_logger(component='results-handler')
+
+
+class RaptorResultsHandler():
+    """Handle Raptor test results"""
+
+    def __init__(self):
+        self.results = []
+
+    def add(self, new_results):
+        # add to results
+        LOG.info("received results")
+        self.results.append(new_results)
+
+    def summarize(self):
+        # summarize results data - calculate mean etc.
+        LOG.info("self.results:")
+        LOG.info(str(self.results))
+
+    def format_and_output(self):
+        # format into PERFHERDER_DATA json, ensure valid json, and then output
+        LOG.info("TODO: output results in PERFERDER_DATA format")
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -12,16 +12,17 @@
 // repo) or 'webkit/PerformanceTests' dir (for benchmarks) first run:
 // 'python -m SimpleHTTPServer 8081'
 // to serve out the pages that we want to prototype with. Also
 // update the manifest content 'matches' accordingly
 
 var browserName;
 var ext;
 var settingsURL = null;
+var cs_port = null;
 var testType;
 var pageCycles = 0;
 var pageCycle = 0;
 var pageCycleDelay = 1000;
 var testURL;
 var testTabID = 0;
 var results = {"page": "", "measurements": {}};
 var getHero = false;
@@ -281,17 +282,17 @@ function verifyResults() {
                   + x + " but only have " + count);
     }
   }
   postToControlServer("results", results);
 }
 
 function postToControlServer(msgType, msgData) {
   // requires 'control server' running at port 8000 to receive results
-  var url = "http://127.0.0.1:8000/";
+  var url = "http://127.0.0.1:" + cs_port + "/";
   var client = new XMLHttpRequest();
   client.onreadystatechange = function() {
     if (client.readyState == XMLHttpRequest.DONE && client.status == 200) {
       console.log("post success");
     }
   };
 
   client.open("POST", url, true);
@@ -327,16 +328,17 @@ function cleanUp() {
   if (browserName === "firefox")
     window.dump("\n__raptor_shutdownBrowser\n");
 
 }
 
 function runner() {
   let config = getTestConfig();
   settingsURL = config.test_settings_url;
+  cs_port = config.cs_port;
   browserName = config.browser;
   getBrowserInfo().then(function() {
     getTestSettings().then(function() {
       if (testType == "benchmark") {
         // webkit benchmark type of test
         console.log("benchmark test start");
       } else if (testType == "pageload") {
         // standard pageload test