--- a/testing/mozharness/configs/talos/linux_config.py
+++ b/testing/mozharness/configs/talos/linux_config.py
@@ -26,16 +26,17 @@ config = {
'tooltool.py': "/tools/tooltool.py",
},
"title": os.uname()[1].lower().split('.')[0],
"default_actions": [
"clobber",
"read-buildbot-config",
"download-and-extract",
"populate-webroot",
+ "setup-mitmproxy",
"create-virtualenv",
"install",
"run-tests",
],
"default_blob_upload_servers": [
"https://blobupload.elasticbeanstalk.com",
],
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
--- a/testing/mozharness/configs/talos/mac_config.py
+++ b/testing/mozharness/configs/talos/mac_config.py
@@ -29,16 +29,17 @@ config = {
'tooltool.py': "/tools/tooltool.py",
},
"title": os.uname()[1].lower().split('.')[0],
"default_actions": [
"clobber",
"read-buildbot-config",
"download-and-extract",
"populate-webroot",
+ "setup-mitmproxy",
"create-virtualenv",
"install",
"run-tests",
],
"run_cmd_checks_enabled": True,
"preflight_run_cmd_suites": [
SCREEN_RESOLUTION_CHECK,
],
--- a/testing/mozharness/configs/talos/windows_config.py
+++ b/testing/mozharness/configs/talos/windows_config.py
@@ -28,16 +28,17 @@ config = {
'tooltool.py': [PYTHON, 'C:/mozilla-build/tooltool.py'],
},
"title": socket.gethostname().split('.')[0],
"default_actions": [
"clobber",
"read-buildbot-config",
"download-and-extract",
"populate-webroot",
+ "setup-mitmproxy",
"create-virtualenv",
"install",
"run-tests",
],
"default_blob_upload_servers": [
"https://blobupload.elasticbeanstalk.com",
],
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -139,23 +139,25 @@ class Talos(TestingMixin, MercurialScrip
] + testing_config_options + copy.deepcopy(blobupload_config_options)
def __init__(self, **kwargs):
kwargs.setdefault('config_options', self.config_options)
kwargs.setdefault('all_actions', ['clobber',
'read-buildbot-config',
'download-and-extract',
'populate-webroot',
+ 'setup-mitmproxy',
'create-virtualenv',
'install',
'run-tests',
])
kwargs.setdefault('default_actions', ['clobber',
'download-and-extract',
'populate-webroot',
+ 'setup-mitmproxy',
'create-virtualenv',
'install',
'run-tests',
])
kwargs.setdefault('config', {})
super(Talos, self).__init__(**kwargs)
self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
@@ -164,16 +166,19 @@ class Talos(TestingMixin, MercurialScrip
self.installer_url = self.config.get("installer_url")
self.talos_json_url = self.config.get("talos_json_url")
self.talos_json = self.config.get("talos_json")
self.talos_json_config = self.config.get("talos_json_config")
self.tests = None
self.gecko_profile = self.config.get('gecko_profile')
self.gecko_profile_interval = self.config.get('gecko_profile_interval')
self.pagesets_name = None
+ self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings
+ self.mitmdump_alt_playback_repo = None # script used for mitdump playback, found in github
+ self.mitmdump = None # path to mitdump tool itself, in py3 venv
# We accept some configuration options from the try commit message in the format mozharness: <options>
# Example try commit message:
# mozharness: --geckoProfile try: <stuff>
def query_gecko_profile_options(self):
gecko_results = []
if self.buildbot_config:
# this is inside automation
@@ -279,16 +284,20 @@ class Talos(TestingMixin, MercurialScrip
if 'suite' in self.config:
kw_options['suite'] = self.config['suite']
if self.config.get('title'):
kw_options['title'] = self.config['title']
if self.config.get('branch'):
kw_options['branchName'] = self.config['branch']
if self.symbols_path:
kw_options['symbolsPath'] = self.symbols_path
+ # if using mitmproxy, we've already created a py3 venv just
+ # for it; need to add the path to that env/mitdump tool
+ if self.mitmdump:
+ kw_options['mitmdumpPath'] = self.mitmdump
kw_options.update(kw)
# talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
tests = kw_options.get('activeTests')
if tests and not isinstance(tests, basestring):
tests = ':'.join(tests) # Talos expects this format
kw_options['activeTests'] = tests
for key, value in kw_options.items():
options.extend(['--%s' % key, value])
@@ -341,16 +350,105 @@ class Talos(TestingMixin, MercurialScrip
)
archive = os.path.join(src_talos_pageset, self.pagesets_name)
unzip = self.query_exe('unzip')
unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
self.run_command(unzip_cmd, halt_on_failure=True)
else:
self.info("Not downloading pageset because the no-download option was specified")
+ def setup_mitmproxy(self):
+ """Some talos tests require the use of mitmproxy to playback the pages; set it up here"""
+ if not self.query_mitmproxy_recording_set():
+ self.info("Skipping: mitmproxy is not required")
+ return
+
+ # setup python 3.x virtualenv
+ self.setup_py3_virtualenv()
+
+ # install mitmproxy
+ self.install_mitmproxy()
+
+ # get the modified replay script from github
+ self.get_mitmproxy_replay_script()
+
+ # download the recording set; will be overridden by the --no-download
+ if '--no-download' not in self.config['talos_extra_options']:
+ self.download_mitmproxy_recording_set()
+ else:
+ self.info("Not downloading mitmproxy recording set because no-download was specified")
+
+ def setup_py3_virtualenv(self):
+ """Mitmproxy needs Python 3.x; set up a separate py 3.x env here"""
+ self.info("Setting up python 3.x virtualenv, required for mitmproxy")
+ # first download the py3 package
+ self.py3_path = self.fetch_python3()
+ # now create the py3 venv
+ self.py3_venv_configuration(python_path=self.py3_path, venv_path='py3venv')
+ self.py3_create_venv()
+ requirements = [os.path.join(self.talos_path, 'mitmproxy_requirements.txt')]
+ self.py3_install_requirement_files(requirements)
+
+ def install_mitmproxy(self):
+ """Install the mitmproxy tool into the Python 3.x env"""
+ self.info("Installing mitmproxy")
+ self.py3_install_modules(modules=['mitmproxy'])
+ self.mitmdump = os.path.join(self.py3_path_to_executables(), 'mitmdump')
+ self.run_command([self.mitmdump, '--version'], env=self.query_env())
+
+ def query_mitmproxy_recording_set(self):
+ """Mitmproxy requires external playback archives to be downloaded and extracted"""
+ if self.mitmproxy_recording_set:
+ return self.mitmproxy_recording_set
+ if self.query_talos_json_config() and self.suite is not None:
+ self.mitmproxy_recording_set = self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set')
+ return self.mitmproxy_recording_set
+
+ def query_mitmdump_alt_playback_repo(self):
+ """Retrieve mitmdump alternative playback repo name"""
+ if self.mitmdump_alt_playback_repo:
+ return self.mitmdump_alt_playback_repo
+ if self.query_talos_json_config() and self.suite is not None:
+ self.mitmdump_alt_playback_repo = self.talos_json_config['suites'][self.suite].get('mitmdump_alt_playback_repo')
+ return self.mitmdump_alt_playback_repo
+
+ def get_mitmproxy_replay_script(self):
+ """We use a custom playback script for mitmproxy; retrieve it from Git here"""
+ # We use a special mitmproxy replay script dev by :bsmedberg
+ # clone it to (testing/talos/talos)
+ if self.query_mitmdump_alt_playback_repo() and self.suite is not None:
+ # if the script already exists locally don't re-clone
+ dest = os.path.join(self.talos_path, 'talos', 'mitmdump-alternate-server-replay')
+ alt_replay_script = os.path.join(dest, 'alternate-server-replay.py')
+ if os.path.exists(alt_replay_script):
+ self.info("Replay script already exists locally: %s" % alt_replay_script)
+ return
+ self.info("Getting alternate mitmproxy playback script from %s" % str(self.mitmdump_alt_playback_repo))
+ try:
+ self.run_command(['git', 'clone', self.mitmdump_alt_playback_repo, dest], env=self.query_env())
+ except:
+ self.fatal("Aborting: Unable to retrieve mitmdump alternate playback script")
+ else:
+ self.fatal("Aborting: mitmproxy_alt_playback_repo not specified in talos.json")
+
+ def download_mitmproxy_recording_set(self):
+ """Download the set of mitmproxy recording files that will be played back"""
+ self.info("Downloading the mitmproxy recording set using tooltool")
+ dest = os.path.join(self.talos_path, 'talos')
+ manifest_file = os.path.join(self.talos_path, 'mitmproxy-playback-set.manifest')
+ self.tooltool_fetch(
+ manifest_file,
+ output_dir=dest,
+ cache=self.config.get('tooltool_cache')
+ )
+ archive = os.path.join(dest, self.mitmproxy_recording_set)
+ unzip = self.query_exe('unzip')
+ unzip_cmd = [unzip, '-q', '-o', archive, '-d', dest]
+ self.run_command(unzip_cmd, halt_on_failure=True)
+
# Action methods. {{{1
# clobber defined in BaseScript
# read_buildbot_config defined in BuildbotMixin
def download_and_extract(self, extract_dirs=None, suite_categories=None):
return super(Talos, self).download_and_extract(
suite_categories=['common', 'talos']
)
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -58,16 +58,17 @@ class TalosRunner(MozbuildObject):
'base_work_dir': self.mozharness_dir,
'exes': {
'python': self.python_interp,
'virtualenv': [self.python_interp, self.virtualenv_script]
},
'title': socket.gethostname(),
'default_actions': [
'populate-webroot',
+ 'setup-mitmproxy',
'create-virtualenv',
'run-tests',
],
'download_tooltool': True,
'talos_extra_options': ['--develop'] + self.talos_args,
}
def make_args(self):
new file mode 100644
--- /dev/null
+++ b/testing/talos/mitmproxy-playback-set.manifest
@@ -0,0 +1,9 @@
+[
+ {
+ "filename": "mitmproxy-recording-set.zip",
+ "size": 679582,
+ "digest": "035e0729905ea1bb61efa4e720cdbc2a436dfebf364eb0d71a7a9ad04658635ede69aa5ef068e99bd515b0b68e2217a2ffe5961b3e8c7f8ead2c49ddcb9fe879",
+ "algorithm": "sha512",
+ "unpack": false
+ }
+]
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -2,22 +2,25 @@
"talos.zip": {
"url": "http://talos-bundles.pvt.build.mozilla.org/zips/talos.a6052c33d420.zip",
"path": ""
},
"extra_options": {
"android": [ "--apkPath=%(apk_path)s" ]
},
"suites": {
- "chromez": {
- "tests": ["tresize", "tcanvasmark"],
- "talos_options": ["--disable-e10s"]
- },
"chromez-e10s": {
- "tests": ["tresize", "tcanvasmark"]
+ "tests": ["Quantum_1"],
+ "mitmproxy_recording_set": "mitmproxy-recording-set.zip",
+ "mitmdump_alt_playback_repo": "https://github.com/bsmedberg/mitmdump-alternate-server-replay",
+ "talos_options": [
+ "--mitmproxy",
+ "mitmproxy-recording-1.mp",
+ "--firstNonBlankPaint"
+ ]
},
"dromaeojs": {
"tests": ["dromaeo_css", "kraken"],
"talos_options": ["--disable-e10s"]
},
"dromaeojs-e10s": {
"tests": ["dromaeo_css", "kraken"]
},
@@ -107,16 +110,26 @@
},
"xperf-e10s": {
"tests": ["tp5n"],
"pagesets_name": "tp5n.zip",
"talos_options": [
"--xperf_path",
"\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
]
+ },
+ "quantum-pageload": {
+ "tests": ["Quantum_1"],
+ "mitmproxy_recording_set": "mitmproxy-recording-set.zip",
+ "mitmdump_alt_playback_repo": "https://github.com/bsmedberg/mitmdump-alternate-server-replay",
+ "talos_options": [
+ "--mitmproxy",
+ "mitmproxy-recording-1.mp",
+ "--firstNonBlankPaint"
+ ]
}
},
"mobile-suites": {
"remote-tsvgx": {
"tests": ["tsvgm"],
"talos_options": [
"--noChrome",
"--tppagecycles", "7"
--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -111,16 +111,23 @@ def create_parser(mach_interface=False):
default=os.path.abspath('browser_failures.txt'),
help="Filename to store the errors found during the test."
" Currently used for xperf only.")
add_arg('--noShutdown', dest='shutdown', action='store_true',
help="Record time browser takes to shutdown after testing")
add_arg('--setpref', action='append', default=[], dest="extraPrefs",
metavar="PREF=VALUE",
help="defines an extra user preference")
+ add_arg('--mitmproxy',
+ help='Test uses mitmproxy to serve the pages, specify the '
+ 'path and name of the mitmdump file to playback')
+ add_arg('--mitmdumpPath',
+ help="Path to mitmproxy's mitmdump playback tool")
+ add_arg("--firstNonBlankPaint", action='store_true', dest="first_non_blank_paint",
+ help="Wait for firstNonBlankPaint event before recording the time")
add_arg('--webServer', dest='webserver',
help="DEPRECATED")
if not mach_interface:
add_arg('--develop', action='store_true', default=False,
help="useful for running tests on a developer machine."
" Doesn't upload to the graph servers.")
add_arg("--cycles", type=int,
help="number of browser cycles to run")
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -2,16 +2,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import mozversion
import os
import sys
+import subprocess
import time
import traceback
import urllib
import utils
import mozhttpd
from mozlog import get_proxy_logger
@@ -73,21 +74,100 @@ def buildCommandLine(test):
# the url as a command line flag to pass to firefox all over the place
# will just make a string for now
return ' '.join(url)
def setup_webserver(webserver):
"""use mozhttpd to setup a webserver"""
LOG.info("starting webserver on %r" % webserver)
-
host, port = webserver.split(':')
return mozhttpd.MozHttpd(host=host, port=int(port), docroot=here)
+def start_mitmproxy_playback(mitmdump_path,
+ mitmproxy_recording_path,
+ mitmproxy_recordings_list):
+ """ Startup mitmproxy and replay the specified flow file """
+ mitmproxy_recordings = []
+ # recording names can be provided in comma-separated list; build py list including path
+ for recording in mitmproxy_recordings_list:
+ mitmproxy_recordings.append(os.path.join(mitmproxy_recording_path, recording))
+
+ # cmd line to start mitmproxy playback using custom playback script is as follows:
+ # <path>/mitmdump -s "<path>mitmdump-alternate-server-replay/alternate-server-replay.py
+ # <path>recording-1.mp <path>recording-2.mp..."
+ param = os.path.join(here, 'mitmdump-alternate-server-replay', 'alternate-server-replay.py')
+ param2 = '""' + param.replace('\\', '\\\\\\') + ' ' + \
+ ' '.join(mitmproxy_recordings).replace('\\', '\\\\\\') + '""'
+ command = [mitmdump_path,
+ '-s',
+ param2,
+ '-k']
+
+ env = os.environ.copy()
+ env["PATH"] = "C:\\slave\\test\\build\\application\\firefox;" + env["PATH"]
+
+ LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
+ LOG.info("Starting mitmproxy playback using command: %s" % ' '.join(command))
+ mitmproxy_proc = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env)
+ time.sleep(5)
+ data = mitmproxy_proc.poll()
+ if data is None:
+ LOG.info("Mitmproxy playback successfully started as pid %d" % mitmproxy_proc.pid)
+ return mitmproxy_proc
+ # cannot continue as we won't be able to playback the pages
+ LOG.error('Aborting: mitmproxy playback process failed to start, poll returned: %s' % data)
+ sys.exit()
+
+
+def stop_mitmproxy_playback(mitmproxy_proc):
+ """ Stop the mitproxy server playback """
+ LOG.info("Stopping mitmproxy playback, klling process %d" % mitmproxy_proc.pid)
+ mitmproxy_proc.kill()
+ time.sleep(5)
+ exit_code = mitmproxy_proc.poll()
+ if exit_code:
+ LOG.info("Successfully killed the mitmproxy playback process")
+ else:
+ # I *think* we can still continue, as process will be automatically
+ # killed anyway when mozharness is done (?) if not, we won't be able
+ # to startup mitmxproy next time if it is already running
+ LOG.error("Failed to kill the mitmproxy playback process")
+
+
+def is_mitmproxy_cert_installed():
+ """ Verify mitmxproy CA cert was added to Firefox """
+ # TODO
+ return True
+
+
+def install_mitmproxy_cert(mitmproxy_proc, browser_path, config):
+ # install the CA certificate generated by mitmproxy, into Firefox
+ LOG.info("Installing mitmxproxy CA certficate into Firefox")
+ # need mozharness mitmproxy module
+ LOG.info("sys.path = %s" % sys.path[1])
+ LOG.info("inserting sys.path = %s" % os.path.join(os.path.dirname(sys.path[1]), 'mozharness'))
+ sys.path.insert(1, os.path.join(os.path.dirname(sys.path[1]), 'mozharness'))
+ sys.path.insert(1, "c:\\slave\\test\\scripts\\")
+ # browser_path is exe, we want install dir
+ browser_install = browser_path.split('Contents')[0]
+ LOG.info('Calling configure_mitmproxy with browser folder: %s' % browser_install)
+ from mozharness.mozilla.mitmproxy import configure_mitmproxy
+ configure_mitmproxy(browser_install)
+ # cannot continue if failed to add CA cert to Firefox, need to check
+ if not is_mitmproxy_cert_installed():
+ LOG.error('Aborting: failed to install mitmproxy CA cert into Firefox')
+ stop_mitmproxy_playback(mitmproxy_proc)
+ sys.exit()
+
+
def run_tests(config, browser_config):
"""Runs the talos tests on the given configuration and generates a report.
"""
# get the test data
tests = config['tests']
tests = useBaseTestDefaults(config.get('basetest', {}), tests)
paths = ['profile_path', 'tpmanifest', 'extensions', 'setup', 'cleanup']
@@ -184,16 +264,40 @@ def run_tests(config, browser_config):
# if e10s add as extra results option
if config['e10s']:
talos_results.add_extra_option('e10s')
if config['gecko_profile']:
talos_results.add_extra_option('geckoProfile')
+ # some tests use mitmproxy to playback pages
+ mitmproxy_recordings_list = config.get('mitmproxy', False)
+ if mitmproxy_recordings_list is not False:
+ # needed so can tell talos ttest to allow external connections
+ browser_config['mitmproxy'] = True
+
+ # start mitmproxy playback; this also generates the CA certificate
+ mitmdump_path = config.get('mitmdumpPath', False)
+ if mitmdump_path is False:
+ # cannot continue, need path for mitmdump playback tool
+ LOG.error('Aborting: mitmdumpPath was not provided on cmd line but is required')
+ sys.exit()
+
+ mitmproxy_recording_path = here
+ mitmproxy_proc = start_mitmproxy_playback(mitmdump_path,
+ mitmproxy_recording_path,
+ mitmproxy_recordings_list.split())
+
+ # install the generated CA certificate into Firefox
+ install_mitmproxy_cert(mitmproxy_proc, browser_config['browser_path'], config)
+
+ if config.get('first_non_blank_paint', False):
+ browser_config['firstNonBlankPaint'] = True
+
testname = None
# run the tests
timer = utils.Timer()
LOG.suite_start(tests=[test['name'] for test in tests])
try:
for test in tests:
testname = test['name']
LOG.test_start(testname)
@@ -219,16 +323,20 @@ def run_tests(config, browser_config):
# indicate a failure to buildbot, turn the job red
return 2
finally:
LOG.suite_end()
httpd.stop()
LOG.info("Completed test suite (%s)" % timer.elapsed())
+ # if mitmproxy was used for page playback, stop it
+ if mitmproxy_recordings_list is not False:
+ stop_mitmproxy_playback(mitmproxy_proc)
+
# output results
if results_urls:
talos_results.output(results_urls)
if browser_config['develop'] or config['gecko_profile']:
print("Thanks for running Talos locally. Results are in %s"
% (results_urls['output_urls']))
# we will stop running tests on a failed test, or we will return 0 for
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -774,8 +774,23 @@ class bloom_basic_ref(PageloaderTest):
tpcycles = 1
tppagecycles = 25
gecko_profile_interval = 1
gecko_profile_entries = 2000000
filters = filter.ignore_first.prepare(5) + filter.median.prepare()
unit = 'ms'
lower_is_better = True
alert_threshold = 5.0
+
+
+@register_test()
+class Quantum_1(PageloaderTest):
+ """
+ Quantum Pageload Test 1
+ """
+ tpmanifest = '${talos}/tests/quantum_pageload/quantum_1.manifest'
+ tpcycles = 1
+ tppagecycles = 25
+ gecko_profile_interval = 1
+ gecko_profile_entries = 2000000
+ filters = filter.ignore_first.prepare(5) + filter.median.prepare()
+ unit = 'ms'
+ lower_is_better = True
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/quantum_pageload/quantum_1.manifest
@@ -0,0 +1,1 @@
+https://www.google.com
--- a/testing/talos/talos/ttest.py
+++ b/testing/talos/talos/ttest.py
@@ -99,16 +99,21 @@ class TTest(object):
if test_config.get('responsiveness') and \
platform.system() != "Darwin":
# ignore osx for now as per bug 1245793
setup.env['MOZ_INSTRUMENT_EVENT_LOOP'] = '1'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_THRESHOLD'] = '20'
setup.env['MOZ_INSTRUMENT_EVENT_LOOP_INTERVAL'] = '10'
global_counters['responsiveness'] = []
+ # if using mitmproxy we must allow access to 'external' sites
+ if browser_config.get('mitmproxy', False):
+ LOG.info("Using mitmproxy so setting MOZ_DISABLE_NONLOCAL_CONNECTIONS to 0")
+ setup.env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '0'
+
# instantiate an object to hold test results
test_results = results.TestResults(
test_config,
global_counters,
browser_config.get('framework')
)
for i in range(test_config['cycles']):