--- a/testing/raptor/raptor/control_server.py
+++ b/testing/raptor/raptor/control_server.py
@@ -18,19 +18,17 @@ LOG = get_proxy_logger(component='contro
here = os.path.abspath(os.path.dirname(__file__))
class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_GET(self):
# get handler, received request for test settings from web ext runner
self.send_response(200)
- validFiles = ['raptor-firefox-tp7.json',
- 'raptor-chrome-tp7.json',
- 'raptor-speedometer.json']
+ validFiles = ['raptor-firefox-tp6.json']
head, tail = os.path.split(self.path)
if tail in validFiles:
LOG.info('reading test settings from ' + tail)
try:
with open(tail) as json_settings:
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Content-type', 'application/json')
self.end_headers()
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/alternate-server-replay.py
@@ -0,0 +1,186 @@
+# This file was copied from mitmproxy/addons/serverplayback.py release tag 2.0.2 and modified by
+# Benjamin Smedberg
+
+# Altered features:
+# * --kill returns 404 rather than dropping the whole HTTP/2 connection on the floor
+# * best-match response handling is used to improve success rates
+from __future__ import absolute_import, print_function
+
+import hashlib
+import sys
+import urllib
+from collections import defaultdict
+
+from mitmproxy import ctx
+from mitmproxy import exceptions
+from mitmproxy import http
+from mitmproxy import io
+from typing import Any # noqa
+from typing import List # noqa
+
+
+class ServerPlayback:
+ def __init__(self, replayfiles):
+ self.options = None
+ self.replayfiles = replayfiles
+ self.flowmap = {}
+
+ def load(self, flows):
+ for i in flows:
+ if i.response:
+ l = self.flowmap.setdefault(self._hash(i.request), [])
+ l.append(i)
+
+ def clear(self):
+ self.flowmap = {}
+
+ def _parse(self, r):
+ """
+ Return (path, queries, formdata, content) for a request.
+ """
+ _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
+ queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
+ queries = defaultdict(list)
+ for k, v in queriesArray:
+ queries[k].append(v)
+
+ content = None
+ formdata = None
+ if r.raw_content != b'':
+ if r.multipart_form:
+ formdata = r.multipart_form
+ elif r.urlencoded_form:
+ formdata = r.urlencoded_form
+ else:
+ content = r.content
+ return (path, queries, formdata, content)
+
+ def _hash(self, r):
+ """
+ Calculates a loose hash of the flow request.
+ """
+ path, queries, _, _ = self._parse(r)
+
+ key = [str(r.port), str(r.scheme), str(r.method), str(path)] # type: List[Any]
+ if not self.options.server_replay_ignore_host:
+ key.append(r.host)
+
+ if len(queries):
+ key.append("?")
+
+ return hashlib.sha256(
+ repr(key).encode("utf8", "surrogateescape")
+ ).digest()
+
+ def _match(self, request_a, request_b):
+ """
+ Calculate a match score between two requests.
+ Match algorithm:
+ * identical query keys: 3 points
+ * matching query param present: 1 point
+ * matching query param value: 3 points
+ * identical form keys: 3 points
+ * matching form param present: 1 point
+ * matching form param value: 3 points
+ * matching body (no multipart or encoded form): 4 points
+ """
+ match = 0
+
+ path_a, queries_a, form_a, content_a = self._parse(request_a)
+ path_b, queries_b, form_b, content_b = self._parse(request_b)
+
+ keys_a = set(queries_a.keys())
+ keys_b = set(queries_b.keys())
+ if keys_a == keys_b:
+ match += 3
+
+ for key in keys_a:
+ values_a = set(queries_a[key])
+ values_b = set(queries_b[key])
+ if len(values_a) == len(values_b):
+ match += 1
+ if values_a == values_b:
+ match += 3
+
+ if form_a and form_b:
+ keys_a = set(form_a.keys())
+ keys_b = set(form_b.keys())
+ if keys_a == keys_b:
+ match += 3
+
+ for key in keys_a:
+ values_a = set(form_a.get_all(key))
+ values_b = set(form_b.get_all(key))
+ if len(values_a) == len(values_b):
+ match += 1
+ if values_a == values_b:
+ match += 3
+
+ elif content_a and (content_a == content_b):
+ match += 4
+
+ return match
+
+ def next_flow(self, request):
+ """
+ Returns the next flow object, or None if no matching flow was
+ found.
+ """
+ hsh = self._hash(request)
+ flows = self.flowmap.get(hsh, None)
+ if flows is None:
+ return None
+
+ # if it's an exact match, great!
+ if len(flows) == 1:
+ candidate = flows[0]
+ if (candidate.request.url == request.url and
+ candidate.request.raw_content == request.raw_content):
+ ctx.log.info("For request {} found exact replay match".format(request.url))
+ return candidate
+
+ # find the best match between the request and the available flow candidates
+ match = -1
+ flow = None
+ ctx.log.debug("Candiate flows for request: {}".format(request.url))
+ for candidate_flow in flows:
+ candidate_match = self._match(candidate_flow.request, request)
+ ctx.log.debug(" score={} url={}".format(candidate_match, candidate_flow.request.url))
+ if candidate_match > match:
+ match = candidate_match
+ flow = candidate_flow
+ ctx.log.info("For request {} best match {} with score=={}".format(request.url,
+ flow.request.url, match))
+ return candidate_flow
+
+ def configure(self, options, updated):
+ self.options = options
+ self.clear()
+ try:
+ flows = io.read_flows_from_paths(self.replayfiles)
+ except exceptions.FlowReadException as e:
+ raise exceptions.OptionsError(str(e))
+ self.load(flows)
+
+ def request(self, f):
+ if self.flowmap:
+ rflow = self.next_flow(f.request)
+ if rflow:
+ response = rflow.response.copy()
+ response.is_replay = True
+ if self.options.refresh_server_playback:
+ response.refresh()
+ f.response = response
+ elif self.options.replay_kill_extra:
+ ctx.log.warn(
+ "server_playback: killed non-replay request {}".format(
+ f.request.url
+ )
+ )
+ f.response = http.HTTPResponse.make(404, b'', {'content-type': 'text/plain'})
+
+
+def start():
+ files = sys.argv[1:]
+ print("Replaying from files: {}".format(files))
+ return ServerPlayback(files)
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-playback-set.manifest
@@ -0,0 +1,9 @@
+[
+ {
+ "filename": "mitmproxy-recording-set-win10.zip",
+ "size": 9189938,
+ "digest": "e904917ed6bf1cef7201284385dc603a283e8e22f992876f17edcf0f1f20db95b609f0d8c7f593b4a0a6c20957dcb6a4d502c562ed74fb6cf4bc255c2f691f32",
+ "algorithm": "sha512",
+ "unpack": true
+ }
+]
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-rel-bin-osx.manifest
@@ -0,0 +1,9 @@
+[
+ {
+ "filename": "mitmproxy-2.0.2-osx.tar.gz",
+ "size": 32324573,
+ "digest": "06423c76e7e99fd9705eae3dc6e2423b1ffb8c42caa98fd010d59dc6ed1f0827376e238c48108106da558444b826e085a58aeb30cf9c79e9d0122a2cb17ae8e6",
+ "algorithm": "sha512",
+ "unpack": true
+ }
+]
--- a/testing/raptor/raptor/playback/mitmproxy.py
+++ b/testing/raptor/raptor/playback/mitmproxy.py
@@ -1,40 +1,272 @@
'''This helps loading mitmproxy's cert and change proxy settings for Firefox.'''
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import os
+import signal
+import subprocess
+import sys
+
+import time
+
+import mozinfo
from mozlog import get_proxy_logger
+from mozprocess import ProcessHandler
from .base import Playback
here = os.path.dirname(os.path.realpath(__file__))
-tooltool_cache = os.path.join(here, 'tooltoolcache')
+LOG = get_proxy_logger(component='mitmproxy')
+
+mozharness_dir = os.path.join(here, '../../../mozharness')
+sys.path.insert(0, mozharness_dir)
+
+TOOLTOOL_PATH = os.path.join(mozharness_dir, 'external_tools', 'tooltool.py')
+
+# path for mitmproxy certificate, generated auto after mitmdump is started
+# on local machine it is 'HOME', however it is different on production machines
+try:
+ DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
+ '.mitmproxy', 'mitmproxy-ca-cert.cer')
+except Exception:
+ DEFAULT_CERT_PATH = os.path.join(os.getenv('HOMEDRIVE'), os.getenv('HOMEPATH'),
+ '.mitmproxy', 'mitmproxy-ca-cert.cer')
-LOG = get_proxy_logger(component='mitmproxy')
+MITMPROXY_SETTINGS = '''// Start with a comment
+// Load up mitmproxy cert
+var certdb = Cc["@mozilla.org/security/x509certdb;1"].getService(Ci.nsIX509CertDB);
+var certdb2 = certdb;
+
+try {
+certdb2 = Cc["@mozilla.org/security/x509certdb;1"].getService(Ci.nsIX509CertDB2);
+} catch (e) {}
+
+cert = "%(cert)s";
+certdb2.addCertFromBase64(cert, "C,C,C", "");
+
+// Use mitmdump as the proxy
+// Manual proxy configuration
+pref("network.proxy.type", 1);
+pref("network.proxy.http", "127.0.0.1");
+pref("network.proxy.http_port", 8080);
+pref("network.proxy.ssl", "127.0.0.1");
+pref("network.proxy.ssl_port", 8080);
+'''
class Mitmproxy(Playback):
def __init__(self, config):
self.config = config
+ self.mitmproxy_proc = None
+ self.recordings = config.get('playback_recordings', None)
+ self.browser_path = config.get('binary', None)
+
+ # bindir is where we will download all mitmproxy required files
+ # if invoved via mach we will have received this in config; otherwise
+ # not running via mach (invoved direcdtly in testing/raptor) so figure it out
+ if self.config.get("obj_path", None) is not None:
+ self.bindir = self.config.get("obj_path")
+ else:
+ # bit of a pain to get object dir when not running via mach - need to go from
+ # the binary folder i.e.
+ # /mozilla-unified/obj-x86_64-apple-darwin17.4.0/dist/Nightly.app/Contents/MacOS/
+ # back to:
+ # mozilla-unified/obj-x86_64-apple-darwin17.4.0/
+ # note, this may need to be updated per platform
+ self.bindir = os.path.normpath(os.path.join(self.config['binary'],
+ '..', '..', '..', '..',
+ '..', 'testing', 'raptor'))
+
+ self.recordings_path = self.bindir
+ LOG.info("bindir to be used for mitmproxy downloads and exe files: %s" % self.bindir)
+
+ # go ahead and download and setup mitmproxy
self.download()
+ # mitmproxy must be started before setup, so that the CA cert is available
+ self.start()
self.setup()
+ def _tooltool_fetch(self, manifest):
+ def outputHandler(line):
+ LOG.info(line)
+ command = [sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest]
+
+ proc = ProcessHandler(
+ command, processOutputLine=outputHandler, storeOutput=False,
+ cwd=self.bindir)
+
+ proc.run()
+
+ try:
+ proc.wait()
+ except Exception:
+ if proc.poll() is None:
+ proc.kill(signal.SIGTERM)
+
def download(self):
- LOG.info("todo: download mitmproxy release binary")
+ # download mitmproxy binary and pageset using tooltool
+ # note: tooltool automatically unpacks the files as well
+ if not os.path.exists(self.bindir):
+ os.makedirs(self.bindir)
+ LOG.info("downloading mitmproxy binary")
+ _manifest = os.path.join(here, self.config['playback_binary_manifest'])
+ self._tooltool_fetch(_manifest)
+ LOG.info("downloading mitmproxy pageset")
+ _manifest = os.path.join(here, self.config['playback_pageset_manifest'])
+ self._tooltool_fetch(_manifest)
return
def setup(self):
- LOG.info("todo: setup mitmproxy")
+ # install the generated CA certificate into Firefox
+ # mitmproxy cert setup needs path to mozharness install; mozharness has set this
+ # value in the SCRIPTSPATH env var for us in mozharness/mozilla/testing/talos.py
+ scripts_path = os.environ.get('SCRIPTSPATH')
+ LOG.info('scripts_path: %s' % str(scripts_path))
+ self.install_mitmproxy_cert(self.mitmproxy_proc,
+ self.browser_path,
+ str(scripts_path))
return
def start(self):
- LOG.info("todo: start mitmproxy playback")
+ mitmdump_path = os.path.join(self.bindir, 'mitmdump')
+ recordings_list = self.recordings.split()
+ self.mitmproxy_proc = self.start_mitmproxy_playback(mitmdump_path,
+ self.recordings_path,
+ recordings_list,
+ self.browser_path)
return
def stop(self):
- LOG.info("todo: stop mitmproxy playback")
+ self.stop_mitmproxy_playback()
return
+
+ def configure_mitmproxy(self,
+ fx_install_dir,
+ scripts_path,
+ certificate_path=DEFAULT_CERT_PATH):
+ # scripts_path is path to mozharness on test machine; needed so can import
+ if scripts_path is not False:
+ sys.path.insert(1, scripts_path)
+ sys.path.insert(1, os.path.join(scripts_path, 'mozharness'))
+ from mozharness.mozilla.firefox.autoconfig import write_autoconfig_files
+ certificate = self._read_certificate(certificate_path)
+ write_autoconfig_files(fx_install_dir=fx_install_dir,
+ cfg_contents=MITMPROXY_SETTINGS % {
+ 'cert': certificate})
+
+ def _read_certificate(self, certificate_path):
+ ''' Return the certificate's hash from the certificate file.'''
+ # NOTE: mitmproxy's certificates do not exist until one of its binaries
+ # has been executed once on the host
+ with open(certificate_path, 'r') as fd:
+ contents = fd.read()
+ return ''.join(contents.splitlines()[1:-1])
+
+ def is_mitmproxy_cert_installed(self, browser_install):
+ """Verify mitmxproy CA cert was added to Firefox"""
+ from mozharness.mozilla.firefox.autoconfig import read_autoconfig_file
+ try:
+ # read autoconfig file, confirm mitmproxy cert is in there
+ certificate = self._read_certificate(DEFAULT_CERT_PATH)
+ contents = read_autoconfig_file(browser_install)
+ if (MITMPROXY_SETTINGS % {'cert': certificate}) in contents:
+ LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
+ else:
+ LOG.info("Firefox autoconfig file contents:")
+ LOG.info(contents)
+ return False
+ except Exception:
+ LOG.info("Failed to read Firefox autoconfig file, when verifying CA cert install")
+ return False
+ return True
+
+ def install_mitmproxy_cert(self, mitmproxy_proc, browser_path, scripts_path):
+ """Install the CA certificate generated by mitmproxy, into Firefox"""
+ LOG.info("Installing mitmxproxy CA certficate into Firefox")
+ # browser_path is exe, we want install dir
+ browser_install = os.path.dirname(browser_path)
+ # on macosx we need to remove the last folders 'Content/MacOS'
+ if mozinfo.os == 'mac':
+ browser_install = browser_install[:-14]
+
+ LOG.info('Calling configure_mitmproxy with browser folder: %s' % browser_install)
+ self.configure_mitmproxy(browser_install, scripts_path)
+ # cannot continue if failed to add CA cert to Firefox, need to check
+ if not self.is_mitmproxy_cert_installed(browser_install):
+ LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox')
+ self.stop_mitmproxy_playback(mitmproxy_proc)
+ sys.exit()
+
+ def start_mitmproxy_playback(self,
+ mitmdump_path,
+ mitmproxy_recording_path,
+ mitmproxy_recordings_list,
+ browser_path):
+ """Startup mitmproxy and replay the specified flow file"""
+
+ LOG.info("mitmdump path: %s" % mitmdump_path)
+ LOG.info("recording path: %s" % mitmproxy_recording_path)
+ LOG.info("recordings list: %s" % mitmproxy_recordings_list)
+ LOG.info("browser path: %s" % browser_path)
+
+ mitmproxy_recordings = []
+ # recording names can be provided in comma-separated list; build py list including path
+ for recording in mitmproxy_recordings_list:
+ mitmproxy_recordings.append(os.path.join(mitmproxy_recording_path, recording))
+
+ # cmd line to start mitmproxy playback using custom playback script is as follows:
+ # <path>/mitmdump -s "<path>mitmdump-alternate-server-replay/alternate-server-replay.py
+ # <path>recording-1.mp <path>recording-2.mp..."
+ param = os.path.join(here, 'alternate-server-replay.py')
+ env = os.environ.copy()
+
+ # this part is platform-specific
+ if mozinfo.os == 'win':
+ param2 = '""' + param.replace('\\', '\\\\\\') + ' ' + \
+ ' '.join(mitmproxy_recordings).replace('\\', '\\\\\\') + '""'
+ sys.path.insert(1, mitmdump_path)
+ else:
+ # mac and linux
+ param2 = param + ' ' + ' '.join(mitmproxy_recordings)
+
+ # mitmproxy needs some DLL's that are a part of Firefox itself, so add to path
+ env["PATH"] = os.path.dirname(browser_path) + ";" + env["PATH"]
+
+ command = [mitmdump_path, '-k', '-s', param2]
+
+ LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
+ LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command))
+ # to turn off mitmproxy log output, use these params for Popen:
+ # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+ mitmproxy_proc = subprocess.Popen(command, env=env)
+ time.sleep(10)
+ data = mitmproxy_proc.poll()
+ if data is None: # None value indicates process hasn't terminated
+ LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid)
+ return mitmproxy_proc
+ # cannot continue as we won't be able to playback the pages
+ LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data)
+ sys.exit()
+
+ def stop_mitmproxy_playback(self):
+ """Stop the mitproxy server playback"""
+ mitmproxy_proc = self.mitmproxy_proc
+ LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid)
+ if mozinfo.os == 'win':
+ mitmproxy_proc.kill()
+ else:
+ mitmproxy_proc.terminate()
+ time.sleep(10)
+ status = mitmproxy_proc.poll()
+ if status is None: # None value indicates process hasn't terminated
+ # I *think* we can still continue, as process will be automatically
+ # killed anyway when mozharness is done (?) if not, we won't be able
+ # to startup mitmxproy next time if it is already running
+ LOG.error("Failed to kill the mitmproxy playback process")
+ LOG.info(str(status))
+ else:
+ LOG.info("Successfully killed the mitmproxy playback process")
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -1,4 +1,2 @@
# raptor tests
-[include:tests/raptor-firefox-tp7.ini]
-[include:tests/raptor-chrome-tp7.ini]
-[include:tests/raptor-speedometer.ini]
+[include:tests/raptor-firefox-tp6.ini]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -61,28 +61,38 @@ class Raptor(object):
runner_cls = runners[app]
self.runner = runner_cls(
binary, profile=self.profile, process_args=process_args)
def start_control_server(self):
self.control_server = RaptorControlServer()
self.control_server.start()
+ def get_playback_config(self, test):
+ self.config['playback_tool'] = test.get('playback')
+ self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
+ self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
+ _key = 'playback_binary_zip_%s' % self.config['platform']
+ self.config['playback_binary_zip'] = test.get(_key, None)
+ self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
+ _key = 'playback_pageset_zip_%s' % self.config['platform']
+ self.config['playback_pageset_zip'] = test.get(_key, None)
+ self.config['playback_recordings'] = test.get('playback_recordings', None)
+
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
gen_test_config(self.config['app'], test['name'])
self.profile.addons.install(os.path.join(webext_dir, 'raptor'))
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
- self.config['playback_tool'] = test.get('playback')
- self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
+ self.get_playback_config(test)
+ # startup the playback tool
self.playback = get_playback(self.config)
- self.playback.start()
self.runner.start()
first_time = int(time.time()) * 1000
proc = self.runner.process_handler
self.output_handler.proc = proc
try:
rename from testing/raptor/raptor/tests/raptor-firefox-tp7.ini
rename to testing/raptor/raptor/tests/raptor-firefox-tp6.ini
--- a/testing/raptor/raptor/tests/raptor-firefox-tp7.ini
+++ b/testing/raptor/raptor/tests/raptor-firefox-tp6.ini
@@ -1,22 +1,20 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-# raptor tp7 firefox
+# raptor tp6 firefox
[DEFAULT]
apps = firefox
type = pageload
playback = mitmproxy
-release_bin_mac = mitmproxy-2.0.2-osx.tar.gz
+playback_binary_manifest = mitmproxy-rel-bin-osx.manifest
+playback_binary_zip_mac = mitmproxy-2.0.2-osx.tar.gz
+playback_pageset_manifest = mitmproxy-playback-set.manifest
+playback_pageset_zip_mac = mitmproxy-recording-set-win10.zip
page_cycles = 25
-[raptor-firefox-tp7]
-test_url = http://localhost:8081/heroes
-measure =
- fnbpaint
- hero
-hero =
- mugshot
- title
- anime
+[raptor-firefox-tp6]
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = mitmproxy-recording-amazon.mp
+measure = fnbpaint
--- a/testing/raptor/test/test_playback.py
+++ b/testing/raptor/test/test_playback.py
@@ -7,20 +7,27 @@ from mozlog.structuredlog import set_def
set_default_logger(StructuredLogger('test_playback'))
from raptor.playback import get_playback, Mitmproxy
config = {}
-def test_get_playback():
+def test_get_playback(get_binary):
config['playback_tool'] = 'mitmproxy'
+ config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
+ config['playback_binary_zip_mac'] = 'mitmproxy-2.0.2-osx.tar.gz'
+ config['playback_pageset_manifest'] = 'mitmproxy-playback-set.manifest'
+ config['playback_pageset_zip_mac'] = 'mitmproxy-recording-set-win10.zip'
+ config['playback_recordings'] = 'mitmproxy-recording-amazon.mp'
+ config['binary'] = get_binary('firefox')
playback = get_playback(config)
assert isinstance(playback, Mitmproxy)
+ playback.stop()
def test_get_unsupported_playback():
config['playback_tool'] = 'unsupported'
playback = get_playback(config)
assert playback is None
--- a/testing/raptor/test/test_raptor.py
+++ b/testing/raptor/test/test_raptor.py
@@ -53,17 +53,17 @@ def test_start_and_stop_server(raptor):
def test_start_browser(get_binary, app):
binary = get_binary(app)
assert binary
raptor = Raptor(app, binary)
raptor.start_control_server()
test = {}
- test['name'] = 'raptor-{}-tp7'.format(app)
+ test['name'] = 'raptor-{}-tp6'.format(app)
thread = threading.Thread(target=raptor.run_test, args=(test,))
thread.start()
timeout = time.time() + 5 # seconds
while time.time() < timeout:
try:
is_running = raptor.runner.is_running()
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -8,17 +8,17 @@
"name": "Raptor",
"version": "0.1",
"description": "Performance measurement framework prototype",
"background": {
"scripts": ["auto_gen_test_config.js", "runner.js"]
},
"content_scripts": [
{
- "matches": ["http://*/tp6/tp6-*.html", "http://*/heroes/*"],
+ "matches": ["<all_urls>"],
"js": ["measure.js"]
},
{
"matches": ["http://*/Speedometer/index.html*"],
"js": ["benchmark-relay.js"]
}
],
"permissions": [
--- a/testing/raptor/webext/raptor/measure.js
+++ b/testing/raptor/webext/raptor/measure.js
@@ -1,13 +1,13 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// content script for use with tp7 pageload tests
+// content script for use with pageload tests
var perfData = window.performance;
var gRetryCounter = 0;
// measure hero element; must exist inside test page;
// default only; this is set via control server settings json
var getHero = false;
var heroesToCapture = [];
@@ -35,33 +35,41 @@ function contentHandler() {
// chrome, no promise so use callback
chrome.storage.local.get("settings", function(item) {
setup(item.settings);
});
}
}
function setup(settings) {
- getFNBPaint = settings.measure.fnbpaint;
- getFCP = settings.measure.fcp;
- if (settings.measure.hero.length !== 0) {
- getHero = true;
- heroesToCapture = settings.measure.hero;
- }
- if (getHero) {
- console.log("hero elements to measure: " + heroesToCapture);
- measureHero();
- }
- if (getFNBPaint) {
- console.log("will be measuring fnbpaint");
- measureFNBPaint();
- }
- if (getFCP) {
- console.log("will be measuring first-contentful-paint");
- measureFirstContentfulPaint();
+ if (settings.measure !== undefined) {
+ if (settings.measure.fnbpaint !== undefined) {
+ getFNBPaint = settings.measure.fnbpaint;
+ if (getFNBPaint) {
+ console.log("will be measuring fnbpaint");
+ measureFNBPaint();
+ }
+ }
+ if (settings.measure.fcp !== undefined) {
+ getFCP = settings.measure.fcp;
+ if (getFCP) {
+ console.log("will be measuring first-contentful-paint");
+ measureFirstContentfulPaint();
+ }
+ }
+ if (settings.measure.hero !== undefined) {
+ if (settings.measure.hero.length !== 0) {
+ getHero = true;
+ heroesToCapture = settings.measure.hero;
+ console.log("hero elements to measure: " + heroesToCapture);
+ measureHero();
+ }
+ }
+ } else {
+ console.log("abort: 'measure' key not found in test settings");
}
}
function measureHero() {
var obs = null;
var heroElementsFound = window.document.querySelectorAll("[elementtiming]");
console.log("found " + heroElementsFound.length + " hero elements in the page");
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -52,20 +52,31 @@ function getTestSettings() {
results.type = testType;
if (settings.page_timeout !== undefined) {
pageTimeout = settings.page_timeout;
}
console.log("using page timeout (ms): " + pageTimeout);
if (testType == "pageload") {
- getFNBPaint = settings.measure.fnbpaint;
- getFCP = settings.measure.fcp;
- if (settings.measure.hero.length !== 0) {
- getHero = true;
+ if (settings.measure !== undefined) {
+ if (settings.measure.fnbpaint !== undefined) {
+ getFNBPaint = settings.measure.fnbpaint;
+ }
+ if (settings.measure.fcp !== undefined) {
+ getFCP = settings.measure.fcp;
+ }
+ if (settings.measure.hero !== undefined) {
+ if (settings.measure.hero.length !== 0) {
+ getHero = true;
+ }
+ }
+ } else {
+ console.log("abort: 'measure' key not found in test settings");
+ cleanUp();
}
}
// write options to storage that our content script needs to know
if (browserName === "firefox") {
ext.storage.local.clear().then(function() {
ext.storage.local.set({settings}).then(function() {
console.log("wrote settings to ext local storage");
@@ -159,19 +170,20 @@ function nextCycle() {
var text = "begin pagecycle " + pageCycle;
console.log("\n" + text);
postToControlServer("status", text);
// set page timeout alarm
setTimeoutAlarm("raptor-page-timeout", pageTimeout);
if (testType == "pageload") {
- if (getHero)
+ if (getHero) {
isHeroPending = true;
pendingHeroes = Array.from(settings.measure.hero);
+ }
if (getFNBPaint)
isFNBPaintPending = true;
if (getFCP)
isFCPPending = true;
} else if (testType == "benchmark") {
isBenchmarkPending = true;
}
// reload the test page
@@ -222,17 +234,17 @@ function resultListener(request, sender,
if (request.type && request.value) {
console.log("result: " + request.type + " " + request.value);
sendResponse({text: "confirmed " + request.type});
if (!(request.type in results.measurements))
results.measurements[request.type] = [];
if (testType == "pageload") {
- // a single tp7 pageload measurement was received
+ // a single pageload measurement was received
if (request.type.indexOf("hero") > -1) {
results.measurements[request.type].push(request.value);
var _found = request.type.split("hero:")[1];
var index = pendingHeroes.indexOf(_found);
if (index > -1) {
pendingHeroes.splice(index, 1);
if (pendingHeroes.length == 0) {
console.log("measured all expected hero elements");
@@ -322,17 +334,17 @@ function runner() {
settingsURL = config.test_settings_url;
browserName = config.browser;
getBrowserInfo().then(function() {
getTestSettings().then(function() {
if (testType == "benchmark") {
// webkit benchmark type of test
console.log("benchmark test start");
} else if (testType == "pageload") {
- // standard 'tp7' pageload test
+ // standard pageload test
console.log("pageloader test start");
}
// results listener
ext.runtime.onMessage.addListener(resultListener);
// tab creation listener
ext.tabs.onCreated.addListener(testTabCreated);
// timeout alarm listener
ext.alarms.onAlarm.addListener(timeoutAlarmListener);