--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -24,73 +24,57 @@ job-defaults:
- raptor/linux64_config_taskcluster.py
default:
- raptor/linux_config.py
raptor-firefox-tp6:
description: "Raptor Firefox tp6"
try-name: raptor-firefox-tp6
treeherder-symbol: Rap(tp6)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- linux64.*: ['try', 'mozilla-central']
- default: ['try']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1200
mozharness:
extra-options:
- --test=raptor-tp6
raptor-firefox-speedometer:
description: "Raptor Firefox speedometer"
try-name: raptor-firefox-speedometer
treeherder-symbol: Rap(sp)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- default: ['try', 'mozilla-central']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1500
mozharness:
extra-options:
- --test=raptor-speedometer
raptor-firefox-stylebench:
description: "Raptor Firefox StyleBench"
try-name: raptor-firefox-stylebench
treeherder-symbol: Rap(sb)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- default: ['try', 'mozilla-central']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-stylebench
raptor-firefox-motionmark-htmlsuite:
description: "Raptor Firefox MotionMark HtmlSuite"
try-name: raptor-firefox-motionmark-htmlsuite
treeherder-symbol: Rap(mm-h)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- default: ['try', 'mozilla-central']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-motionmark-htmlsuite
raptor-firefox-motionmark-animometer:
description: "Raptor Firefox MotionMark Animometer"
try-name: raptor-firefox-motionmark-animometer
treeherder-symbol: Rap(mm-a)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- default: ['try', 'mozilla-central']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-motionmark-animometer
raptor-chrome-tp6:
description: "Raptor Chrome tp6"
try-name: raptor-chrome-tp6
@@ -104,59 +88,47 @@ raptor-chrome-tp6:
extra-options:
- --test=raptor-tp6
- --app=chrome
raptor-chrome-speedometer:
description: "Raptor Chrome speedometer"
try-name: raptor-chrome-speedometer
treeherder-symbol: Rap-C(sp)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- default: ['try']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1500
mozharness:
extra-options:
- --test=raptor-speedometer
- --app=chrome
raptor-chrome-stylebench:
description: "Raptor Chrome StyleBench"
try-name: raptor-chrome-stylebench
treeherder-symbol: Rap-C(sb)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- default: ['try']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-stylebench
- --app=chrome
raptor-chrome-motionmark-htmlsuite:
description: "Raptor Chrome MotionMark HtmlSuite"
try-name: raptor-chrome-motionmark-htmlsuite
treeherder-symbol: Rap-C(mm-h)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- default: ['try']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-motionmark-htmlsuite
- --app=chrome
raptor-chrome-motionmark-animometer:
description: "Raptor Chrome MotionMark Animometer"
try-name: raptor-chrome-motionmark-animometer
treeherder-symbol: Rap-C(mm-a)
- run-on-projects:
- by-test-platform:
- macosx.*: ['try', 'mozilla-central']
- default: ['try']
+ run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-motionmark-animometer
- --app=chrome
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@@ -130,25 +130,27 @@ windows7-32/debug:
windows7-32/opt:
build-platform: win32/opt
test-sets:
- awsy
- desktop-screenshot-capture
- windows-reftest-gpu
- windows-talos
- windows-tests
+ - raptor
windows7-32-pgo/opt:
build-platform: win32-pgo/opt
test-sets:
- awsy
- desktop-screenshot-capture
- windows-reftest-gpu
- windows-talos
- windows-tests
+ - raptor
windows7-32-nightly/opt:
build-platform: win32-nightly/opt
test-sets:
- awsy
- desktop-screenshot-capture
- windows-reftest-gpu
- windows-tests
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -13,17 +13,16 @@ import subprocess
import time
from shutil import copyfile
import mozharness
from mozharness.base.errors import PythonErrorList
from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO
-from mozharness.base.python import Python3Virtualenv
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.testing.codecoverage import (
CodeCoverageMixin,
code_coverage_config_options
)
scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
@@ -39,17 +38,17 @@ RaptorErrorList = PythonErrorList + [
{'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
{'substr': r"""No such file or directory: 'browser_output.txt'""",
'level': CRITICAL,
'explanation': "Most likely the browser failed to launch, or the test was otherwise "
"unsuccessful in even starting."},
]
-class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin):
+class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin):
"""
install and run raptor tests
"""
config_options = [
[["--test"],
{"action": "store",
"dest": "test",
"help": "Raptor test to run"
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -15,17 +15,17 @@ raptor_ini = os.path.join(here, 'raptor.
tests_dir = os.path.join(here, 'tests')
LOG = get_proxy_logger(component="raptor-manifest")
required_settings = ['apps', 'type', 'page_cycles', 'test_url', 'measure',
'unit', 'lower_is_better', 'alert_threshold']
playback_settings = ['playback_binary_manifest', 'playback_binary_zip_mac',
'playback_pageset_manifest', 'playback_pageset_zip_mac',
- 'playback_recordings']
+ 'playback_recordings', 'python3_win_manifest']
def filter_app(tests, values):
for test in tests:
if values["app"] in test['apps']:
yield test
--- a/testing/raptor/raptor/playback/mitmproxy.py
+++ b/testing/raptor/raptor/playback/mitmproxy.py
@@ -17,18 +17,29 @@ from mozlog import get_proxy_logger
from mozprocess import ProcessHandler
from .base import Playback
here = os.path.dirname(os.path.realpath(__file__))
LOG = get_proxy_logger(component='raptor-mitmproxy')
# needed so unit tests can find their imports
-mozharness_dir = os.path.join(here, '../../../mozharness')
+if os.environ.get('SCRIPTSPATH', None) is not None:
+ # in production it is env SCRIPTS_PATH
+ mozharness_dir = os.environ['SCRIPTSPATH']
+else:
+ # locally it's in source tree
+ mozharness_dir = os.path.join(here, '../../../mozharness')
sys.path.insert(0, mozharness_dir)
+
+# required for using a python3 virtualenv on win for mitmproxy
+from mozharness.base.python import Python3Virtualenv
+from mozharness.mozilla.testing.testbase import TestingMixin
+from mozharness.base.vcs.vcsbase import MercurialScript
+
raptor_dir = os.path.join(here, '..')
sys.path.insert(0, raptor_dir)
from utils import transform_platform
external_tools_path = os.environ.get('EXTERNALTOOLSPATH', None)
if external_tools_path is not None:
@@ -64,21 +75,22 @@ certdb2.addCertFromBase64(cert, "C,C,C",
pref("network.proxy.type", 1);
pref("network.proxy.http", "127.0.0.1");
pref("network.proxy.http_port", 8080);
pref("network.proxy.ssl", "127.0.0.1");
pref("network.proxy.ssl_port", 8080);
'''
-class Mitmproxy(Playback):
+class Mitmproxy(Playback, Python3Virtualenv, TestingMixin, MercurialScript):
def __init__(self, config):
self.config = config
self.mitmproxy_proc = None
+ self.mitmdump_path = None
self.recordings = config.get('playback_recordings', None)
self.browser_path = config.get('binary', None)
# raptor_dir is where we will download all mitmproxy required files
# when running locally it comes from obj_path via mozharness/mach
if self.config.get("obj_path", None) is not None:
self.raptor_dir = self.config.get("obj_path")
else:
@@ -89,16 +101,21 @@ class Mitmproxy(Playback):
# add raptor to raptor_dir
self.raptor_dir = os.path.join(self.raptor_dir, "testing", "raptor")
self.recordings_path = self.raptor_dir
LOG.info("raptor_dir used for mitmproxy downloads and exe files: %s" % self.raptor_dir)
# go ahead and download and setup mitmproxy
self.download()
+
+ # on windows we must use a python3 virtualen for mitmproxy
+ if 'win' in self.config['platform']:
+ self.setup_py3_virtualenv()
+
# mitmproxy must be started before setup, so that the CA cert is available
self.start()
self.setup()
def _tooltool_fetch(self, manifest):
def outputHandler(line):
LOG.info(line)
command = [sys.executable, TOOLTOOL_PATH, 'fetch', '-o', '-m', manifest]
@@ -115,41 +132,86 @@ class Mitmproxy(Playback):
if proc.poll() is None:
proc.kill(signal.SIGTERM)
def download(self):
# download mitmproxy binary and pageset using tooltool
# note: tooltool automatically unpacks the files as well
if not os.path.exists(self.raptor_dir):
os.makedirs(self.raptor_dir)
- LOG.info("downloading mitmproxy binary")
- _manifest = os.path.join(here, self.config['playback_binary_manifest'])
- transformed_manifest = transform_platform(_manifest, self.config['platform'])
- self._tooltool_fetch(transformed_manifest)
+
+ if 'win' in self.config['platform']:
+ # on windows we need a python3 environment and use our own package from tooltool
+ self.py3_path = self.fetch_python3()
+ LOG.info("python3 path is: %s" % self.py3_path)
+ else:
+ # on osx and linux we use pre-built binaries
+ LOG.info("downloading mitmproxy binary")
+ _manifest = os.path.join(here, self.config['playback_binary_manifest'])
+ transformed_manifest = transform_platform(_manifest, self.config['platform'])
+ self._tooltool_fetch(transformed_manifest)
+
+ # we use one pageset for all platforms (pageset was recorded on win10)
LOG.info("downloading mitmproxy pageset")
_manifest = os.path.join(here, self.config['playback_pageset_manifest'])
transformed_manifest = transform_platform(_manifest, self.config['platform'])
self._tooltool_fetch(transformed_manifest)
return
+ def fetch_python3(self):
+ """Mitmproxy on windows needs Python 3.x"""
+ python3_path = os.path.join(self.raptor_dir, 'python3.6', 'python')
+ if not os.path.exists(os.path.dirname(python3_path)):
+ _manifest = os.path.join(here, self.config['python3_win_manifest'])
+ transformed_manifest = transform_platform(_manifest, self.config['platform'],
+ self.config['processor'])
+ LOG.info("downloading py3 package for mitmproxy windows: %s" % transformed_manifest)
+ self._tooltool_fetch(transformed_manifest)
+ cmd = [python3_path, '--version']
+ # just want python3 ver printed in production log
+ subprocess.Popen(cmd, env=os.environ.copy())
+ return python3_path
+
+ def setup_py3_virtualenv(self):
+ """Mitmproxy on windows needs Python 3.x; set up a separate py 3.x env here"""
+ LOG.info("Setting up python 3.x virtualenv, required for mitmproxy on windows")
+ # these next two are required for py3_venv_configuration
+ self.abs_dirs = {'base_work_dir': mozharness_dir}
+ self.log_obj = None
+ # now create the py3 venv
+ venv_path = os.path.join(self.raptor_dir, 'py3venv')
+ self.py3_venv_configuration(python_path=self.py3_path, venv_path=venv_path)
+ self.py3_create_venv()
+ self.py3_install_modules(["cffi==1.10.0"])
+ requirements = [os.path.join(here, "mitmproxy_requirements.txt")]
+ self.py3_install_requirement_files(requirements)
+ # add py3 executables path to system path
+ sys.path.insert(1, self.py3_path_to_executables())
+ # install mitmproxy itself
+ self.py3_install_modules(modules=['mitmproxy'])
+ self.mitmdump_path = os.path.join(self.py3_path_to_executables(), 'mitmdump')
+
def setup(self):
# install the generated CA certificate into Firefox
# mitmproxy cert setup needs path to mozharness install; mozharness has set this
# value in the SCRIPTSPATH env var for us in mozharness/mozilla/testing/talos.py
scripts_path = os.environ.get('SCRIPTSPATH')
LOG.info('scripts_path: %s' % str(scripts_path))
self.install_mitmproxy_cert(self.mitmproxy_proc,
self.browser_path,
str(scripts_path))
return
def start(self):
- mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump')
+ # if on windows, the mitmdump_path was already set when creating py3 env
+ if self.mitmdump_path is None:
+ self.mitmdump_path = os.path.join(self.raptor_dir, 'mitmdump')
+
recordings_list = self.recordings.split()
- self.mitmproxy_proc = self.start_mitmproxy_playback(mitmdump_path,
+ self.mitmproxy_proc = self.start_mitmproxy_playback(self.mitmdump_path,
self.recordings_path,
recordings_list,
self.browser_path)
return
def stop(self):
self.stop_mitmproxy_playback()
return
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy_requirements.txt
@@ -0,0 +1,35 @@
+argh==0.26.2
+asn1crypto==0.22.0
+blinker==1.4
+pycparser==2.17
+cffi==1.10.0
+brotlipy==0.6.0
+certifi==2017.4.17
+click==6.7
+construct==2.8.12
+cryptography==1.8.2
+cssutils==1.0.2
+EditorConfig==0.12.1
+h2==2.6.2
+hpack==3.0.0
+html2text==2016.9.19
+hyperframe==4.0.2
+idna==2.5
+jsbeautifier==1.6.12
+kaitaistruct==0.6
+mitmproxy==2.0.2
+packaging==16.8
+passlib==1.7.1
+pathtools==0.1.2
+pyasn1==0.2.3
+pyOpenSSL==16.2.0
+pyparsing==2.2.0
+pyperclip==1.5.27
+PyYAML==3.12
+requests==2.13.0
+ruamel.yaml==0.13.14
+six==1.10.0
+sortedcontainers==1.5.7
+tornado==4.4.3
+urwid==1.3.1
+watchdog==0.8.3
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/python3.manifest
@@ -0,0 +1,10 @@
+[
+ {
+ "size": 15380470,
+ "visibility": "public",
+ "digest": "cd78b88d95b69bef99d7192b71dd34118700f44db0a0069a13bfd4943b131e8d7fdac83859f8ac15d873d4b329eef69d8d75d0a6746d06fdcfc5d06da0c9784c",
+ "algorithm": "sha512",
+ "unpack": true,
+ "filename": "python3.6.zip"
+ }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/python3_x64.manifest
@@ -0,0 +1,10 @@
+[
+ {
+ "size": 16026760,
+ "visibility": "public",
+ "digest": "379428e3955671213a245ccd9ccf6f9d17d368db68c02da8baed7be629f2691127cd3e3f86807b25e2098d9840083fdc07946ab1bed0c14db4a5b628a47ed9ef",
+ "algorithm": "sha512",
+ "unpack": true,
+ "filename": "python3.6.amd64.zip"
+ }
+]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -39,16 +39,17 @@ from results import RaptorResultsHandler
class Raptor(object):
"""Container class for Raptor"""
def __init__(self, app, binary, run_local=False, obj_path=None):
self.config = {}
self.config['app'] = app
self.config['binary'] = binary
self.config['platform'] = mozinfo.os
+ self.config['processor'] = mozinfo.processor
self.config['run_local'] = run_local
self.config['obj_path'] = obj_path
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor-main')
self.control_server = None
self.playback = None
self.benchmark = None
@@ -95,16 +96,17 @@ class Raptor(object):
self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
_key = 'playback_binary_zip_%s' % self.config['platform']
self.config['playback_binary_zip'] = test.get(_key, None)
self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
_key = 'playback_pageset_zip_%s' % self.config['platform']
self.config['playback_pageset_zip'] = test.get(_key, None)
self.config['playback_recordings'] = test.get('playback_recordings', None)
+ self.config['python3_win_manifest'] = test.get('python3_win_manifest', None)
def run_test(self, test, timeout=None):
self.log.info("starting raptor test: %s" % test['name'])
self.log.info("test settings: %s" % str(test))
self.log.info("raptor config: %s" % str(self.config))
# benchmark-type tests require the benchmark test to be served out
if test.get('type') == "benchmark":
@@ -135,16 +137,17 @@ class Raptor(object):
self.runner.start()
proc = self.runner.process_handler
self.output_handler.proc = proc
self.control_server.browser_proc = proc
try:
self.runner.wait(timeout)
+
finally:
try:
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
if self.playback is not None:
self.playback.stop()
--- a/testing/raptor/raptor/tests/raptor-tp6.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6.ini
@@ -4,16 +4,17 @@
# raptor tp6
[DEFAULT]
type = pageload
playback = mitmproxy
playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
playback_binary_zip_mac = mitmproxy-2.0.2-{platform}.tar.gz
+python3_win_manifest = python3{x64}.manifest
playback_pageset_manifest = mitmproxy-playback-set.manifest
playback_pageset_zip_mac = mitmproxy-recording-set-win10.zip
page_cycles = 25
unit = ms
lower_is_better = true
alert_threshold = 2.0
[raptor-firefox-tp6-amazon]
--- a/testing/raptor/raptor/utils.py
+++ b/testing/raptor/raptor/utils.py
@@ -5,20 +5,32 @@
from __future__ import absolute_import
from mozlog import get_proxy_logger
LOG = get_proxy_logger(component="raptor-utils")
-def transform_platform(str_to_transform, config_platform):
+def transform_platform(str_to_transform, config_platform, config_processor=None):
# transform platform name i.e. 'mitmproxy-rel-bin-{platform}.manifest'
# transforms to 'mitmproxy-rel-bin-osx.manifest'
- if '{platform}' not in str_to_transform:
+ # also will transform '{x64}' if needed for 64 bit / win 10
+ if '{platform}' not in str_to_transform and '{x64}' not in str_to_transform:
return str_to_transform
+
if 'win' in config_platform:
platform_id = 'win'
elif config_platform == 'mac':
platform_id = 'osx'
else:
platform_id = 'linux64'
- return str_to_transform.replace('{platform}', platform_id)
+
+ if '{platform}' in str_to_transform:
+ str_to_transform = str_to_transform.replace('{platform}', platform_id)
+
+ if '{x64}' in str_to_transform and config_processor is not None:
+ if 'x86_64' in config_processor:
+ str_to_transform = str_to_transform.replace('{x64}', '_x64')
+ else:
+ str_to_transform = str_to_transform.replace('{x64}', '')
+
+ return str_to_transform