--- a/testing/mozharness/configs/raptor/mac_config.py
+++ b/testing/mozharness/configs/raptor/mac_config.py
@@ -21,16 +21,17 @@ config = {
"http://pypi.pub.build.mozilla.org/pub",
],
"pip_index": False,
"title": os.uname()[1].lower().split('.')[0],
"default_actions": [
"clobber",
"download-and-extract",
"populate-webroot",
+ "install-chrome",
"create-virtualenv",
"install",
"run-tests",
],
"run_cmd_checks_enabled": True,
"preflight_run_cmd_suites": [
SCREEN_RESOLUTION_CHECK,
],
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -5,16 +5,17 @@
from __future__ import absolute_import, print_function, unicode_literals
import copy
import json
import os
import re
import sys
import subprocess
+import time
from shutil import copyfile
import mozharness
from mozharness.base.errors import PythonErrorList
from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO
from mozharness.base.python import Python3Virtualenv
@@ -22,16 +23,17 @@ from mozharness.mozilla.testing.testbase
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.testing.codecoverage import (
CodeCoverageMixin,
code_coverage_config_options
)
scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
external_tools_path = os.path.join(scripts_path, 'external_tools')
+here = os.path.abspath(os.path.dirname(__file__))
RaptorErrorList = PythonErrorList + [
{'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
{'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
{'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
{'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
{'substr': r'''raptorError''', 'level': CRITICAL},
{'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
@@ -47,73 +49,90 @@ class Raptor(TestingMixin, MercurialScri
install and run raptor tests
"""
config_options = [
[["--test"],
{"action": "store",
"dest": "test",
"help": "Raptor test to run"
}],
+ [["--app"],
+ {"default": "firefox",
+ "choices": ["firefox", "chrome"],
+ "dest": "app",
+ "help": "name of the application we are testing (default: firefox)"
+ }],
[["--branch-name"],
{"action": "store",
"dest": "branch",
"help": "branch running against"
}],
[["--add-option"],
{"action": "extend",
- "dest": "raptor_extra_options",
+ "dest": "raptor_cmd_line_args",
"default": None,
"help": "extra options to raptor"
}],
] + testing_config_options + copy.deepcopy(code_coverage_config_options)
def __init__(self, **kwargs):
kwargs.setdefault('config_options', self.config_options)
kwargs.setdefault('all_actions', ['clobber',
'download-and-extract',
'populate-webroot',
+ 'install-chrome',
'create-virtualenv',
'install',
'run-tests',
])
kwargs.setdefault('default_actions', ['clobber',
'download-and-extract',
'populate-webroot',
+ 'install-chrome',
'create-virtualenv',
'install',
'run-tests',
])
kwargs.setdefault('config', {})
super(Raptor, self).__init__(**kwargs)
self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
self.run_local = self.config.get('run_local')
+
+ # app (browser testing on) defaults to firefox
+ self.app = "firefox"
+
+ if self.run_local:
+ # raptor initiated locally, get app from command line args
+ # which are passed in from mach inside 'raptor_cmd_line_args'
+ self.app = "firefox"
+ if 'raptor_cmd_line_args' in self.config:
+ for next_arg in self.config['raptor_cmd_line_args']:
+ if "chrome" in next_arg:
+ self.app = "chrome"
+ break
+ else:
+ # raptor initiated in production via mozharness
+ self.test = self.config['test']
+ self.app = self.config.get("app", "firefox")
+
self.installer_url = self.config.get("installer_url")
self.raptor_json_url = self.config.get("raptor_json_url")
self.raptor_json = self.config.get("raptor_json")
self.raptor_json_config = self.config.get("raptor_json_config")
self.repo_path = self.config.get("repo_path")
self.obj_path = self.config.get("obj_path")
- self.tests = None
+ self.test = None
self.gecko_profile = self.config.get('gecko_profile')
self.gecko_profile_interval = self.config.get('gecko_profile_interval')
- # some platforms download a mitmproxy release binary
- self.mitmproxy_rel_bin = None
- # zip file found on tooltool that contains all of the mitmproxy recordings
- self.mitmproxy_pageset = None
- # files inside the recording set
- self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None)
- # path to mitmdump tool itself, in py3 venv
- self.mitmdump = None
# We accept some configuration options from the try commit message in the
- # format mozharness: <options>
- # Example try commit message:
- # mozharness: --geckoProfile try: <stuff>
+ # format mozharness: <options>. Example try commit message: mozharness:
+ # --geckoProfile try: <stuff>
def query_gecko_profile_options(self):
gecko_results = []
# if gecko_profile is set, we add that to the raptor options
if self.gecko_profile:
gecko_results.append('--geckoProfile')
if self.gecko_profile_interval:
gecko_results.extend(
['--geckoProfileInterval', str(self.gecko_profile_interval)]
@@ -125,29 +144,87 @@ class Raptor(TestingMixin, MercurialScri
return self.abs_dirs
abs_dirs = super(Raptor, self).query_abs_dirs()
abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'],
'blobber_upload_dir')
abs_dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
self.abs_dirs = abs_dirs
return self.abs_dirs
+ def install_chrome(self):
+ # temporary hack to install google chrome in production; until chrome is in our CI
+ if self.app != "chrome":
+ self.info("Google Chrome is not required")
+ return
+
+ if self.config.get("run_local"):
+ self.info("expecting Google Chrome to be pre-installed locally")
+ return
+
+ chrome_url = "https://dl.google.com/chrome/mac/stable/GGRO/googlechrome.dmg"
+ # in production we can put the chrome build in mozharness/mozilla/testing/chrome
+ self.chrome_dest = os.path.join(here, 'chrome')
+ chrome_dmg = os.path.join(self.chrome_dest, 'googlechrome.dmg')
+
+ self.info("installing google chrome - temporary install hack")
+ self.info("chrome_dest is: %s" % self.chrome_dest)
+
+ self.chrome_path = os.path.join(self.chrome_dest, 'Google Chrome.app',
+ 'Contents', 'MacOS', 'Google Chrome')
+
+ if os.path.exists(self.chrome_path):
+ self.info("google chrome binary already exists at: %s" % self.chrome_path)
+ return
+
+ if not os.path.exists(chrome_dmg):
+ # download the chrome dmg
+ self.download_file(chrome_url, parent_dir=self.chrome_dest)
+
+ command = ["open", "googlechrome.dmg"]
+ return_code = self.run_command(command, cwd=self.chrome_dest)
+ if return_code not in [0]:
+ self.info("abort: failed to open %s/googlechrome.dmg" % self.chrome_dest)
+ return
+ # give 30 sec for open cmd to finish
+ time.sleep(30)
+
+ # now that the googlechrome dmg is mounted, extract/copy app from mnt to our folder
+ command = ["cp", "-r", "/Volumes/Google Chrome/Google Chrome.app", "."]
+ return_code = self.run_command(command, cwd=self.chrome_dest)
+ if return_code not in [0]:
+ self.info("abort: failed to open %s/googlechrome.dmg" % self.chrome_dest)
+ return
+
+ # now ensure chrome binary exists
+ if os.path.exists(self.chrome_path):
+ self.info("successfully installed Google Chrome to: %s" % self.chrome_path)
+ else:
+ self.info("abort: failed to install Google Chrome")
+
def raptor_options(self, args=None, **kw):
"""return options to raptor"""
- # binary path
- binary_path = self.binary_path or self.config.get('binary_path')
- if not binary_path:
- msg = """Raptor requires a path to the binary. You can specify binary_path or add
- download-and-extract to your action list."""
- self.fatal(msg)
- # raptor options
- if binary_path.endswith('.exe'):
- binary_path = binary_path[:-4]
options = []
- kw_options = {'binary': binary_path}
+ kw_options = {}
+
+ # binary path; if testing on firefox the binary path already came from mozharness/pro;
+ # otherwise the binary path is forwarded from cmd line arg (raptor_cmd_line_args)
+ kw_options['app'] = self.app
+ if self.app == "firefox":
+ binary_path = self.binary_path or self.config.get('binary_path')
+ if not binary_path:
+ self.fatal("Raptor requires a path to the binary.")
+ if binary_path.endswith('.exe'):
+ binary_path = binary_path[:-4]
+ kw_options['binary'] = binary_path
+ else:
+ if not self.run_local:
+ # in production we aready installed google chrome, so set the binary path for arg
+ # when running locally a --binary arg as passed in, already in raptor_cmd_line_args
+ kw_options['binary'] = self.chrome_path
+
# options overwritten from **kw
if 'test' in self.config:
kw_options['test'] = self.config['test']
if self.config.get('branch'):
kw_options['branchName'] = self.config['branch']
if self.symbols_path:
kw_options['symbolsPath'] = self.symbols_path
if self.config.get('obj_path', None) is not None:
@@ -155,44 +232,32 @@ class Raptor(TestingMixin, MercurialScri
kw_options.update(kw)
# configure profiling options
options.extend(self.query_gecko_profile_options())
# extra arguments
if args is not None:
options += args
if self.config.get('run_local', False):
options.extend(['--run-local'])
- if 'raptor_extra_options' in self.config:
- options += self.config['raptor_extra_options']
+ if 'raptor_cmd_line_args' in self.config:
+ options += self.config['raptor_cmd_line_args']
if self.config.get('code_coverage', False):
options.extend(['--code-coverage'])
for key, value in kw_options.items():
options.extend(['--%s' % key, value])
+
return options
def populate_webroot(self):
"""Populate the production test slaves' webroots"""
self.raptor_path = os.path.join(
self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
)
-
if self.config.get('run_local'):
- # raptor initiated locally, get and verify test from cmd line
self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
- if 'raptor_extra_options' in self.config:
- if '--test' in self.config['raptor_extra_options']:
- # --test specified, get test from cmd line and ensure is valid
- test_name_index = self.config['raptor_extra_options'].index('--test') + 1
- if test_name_index < len(self.config['raptor_extra_options']):
- self.test = self.config['raptor_extra_options'][test_name_index]
- else:
- self.fatal("Test name not provided")
- else:
- # raptor initiated in production via mozharness
- self.test = self.config['test']
# Action methods. {{{1
# clobber defined in BaseScript
def download_and_extract(self, extract_dirs=None, suite_categories=None):
return super(Raptor, self).download_and_extract(
suite_categories=['common', 'raptor']
)
--- a/testing/raptor/mach_commands.py
+++ b/testing/raptor/mach_commands.py
@@ -44,34 +44,34 @@ class RaptorRunner(MozbuildObject):
self.virtualenv_script = os.path.join(self.topsrcdir, 'third_party', 'python',
'virtualenv', 'virtualenv.py')
self.virtualenv_path = os.path.join(self._topobjdir, 'testing',
'raptor-venv')
self.python_interp = sys.executable
self.raptor_args = raptor_args
def make_config(self):
- default_actions = ['populate-webroot', 'create-virtualenv', 'run-tests']
+ default_actions = ['populate-webroot', 'install-chrome', 'create-virtualenv', 'run-tests']
self.config = {
'run_local': True,
'binary_path': self.binary_path,
'repo_path': self.topsrcdir,
'raptor_path': self.raptor_dir,
'obj_path': self.topobjdir,
'log_name': 'raptor',
'virtualenv_path': self.virtualenv_path,
'pypi_url': 'http://pypi.python.org/simple',
'base_work_dir': self.mozharness_dir,
'exes': {
'python': self.python_interp,
'virtualenv': [self.python_interp, self.virtualenv_script],
},
'title': socket.gethostname(),
'default_actions': default_actions,
- 'raptor_extra_options': self.raptor_args,
+ 'raptor_cmd_line_args': self.raptor_args,
'python3_manifest': {
'win32': 'python3.manifest',
'win64': 'python3_x64.manifest',
}
}
def make_args(self):
self.args = {
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -10,37 +10,39 @@ from mozlog.commandline import add_loggi
def create_parser(mach_interface=False):
parser = argparse.ArgumentParser()
add_arg = parser.add_argument
add_arg('-t', '--test', required=True, dest='test',
help="name of raptor test to run")
+ add_arg('--app', default='firefox', dest='app',
+ help="name of the application we are testing (default: firefox)",
+ choices=['firefox', 'chrome'])
+ add_arg('-b', '--binary', dest='binary',
+ help="path to the browser executable that we are testing")
if not mach_interface:
- add_arg('--app', default='firefox', dest='app',
- help="name of the application we are testing (default: firefox)",
- choices=['firefox', 'chrome'])
- add_arg('-b', '--binary', required=True, dest='binary',
- help="path to the browser executable that we are testing")
- add_arg('--branchName', dest="branch_name", default=None,
+ add_arg('--branchName', dest="branch_name", default='',
help="Name of the branch we are testing on")
add_arg('--symbolsPath', dest='symbols_path',
help="Path to the symbols for the build we are testing")
add_arg('--run-local', dest="run_local", default=False, action="store_true",
help="Flag that indicates if raptor is running locally or in production")
add_arg('--obj-path', dest="obj_path", default=None,
help="Browser build obj_path (received when running in production)")
add_logging_group(parser)
return parser
def verify_options(parser, args):
ctx = vars(args)
+ if args.binary is None:
+ parser.error("--binary is required!")
if not os.path.isfile(args.binary):
parser.error("{binary} does not exist!".format(**ctx))
def parse_args(argv=None):
parser = create_parser()
args = parser.parse_args(argv)
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -1,3 +1,3 @@
# raptor tests
-[include:tests/raptor-firefox-tp6.ini]
+[include:tests/raptor-tp6.ini]
[include:tests/raptor-speedometer.ini]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -115,17 +115,19 @@ class Raptor(object):
test['name'],
self.control_server.port,
benchmark_port)
# must intall raptor addon each time because we dynamically update some content
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
- webext_id = self.profile.addons.addon_details(raptor_webext)['id']
+ # on firefox we can get an addon id; chrome addon actually is just cmd line arg
+ if self.config['app'] == "firefox":
+ webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
self.get_playback_config(test)
# startup the playback tool
self.playback = get_playback(self.config)
self.runner.start()
@@ -141,18 +143,20 @@ class Raptor(object):
self.runner.check_for_crashes()
except NotImplementedError: # not implemented for Chrome
pass
if self.playback is not None:
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
- self.log.info("removing webext %s" % raptor_webext)
- self.profile.addons.remove_addon(webext_id)
+ # applies to firefox only; chrome the addon is actually just cmd line arg
+ if self.config['app'] == "firefox":
+ self.log.info("removing webext %s" % raptor_webext)
+ self.profile.addons.remove_addon(webext_id)
if self.runner.is_running():
self.log("Application timed out after {} seconds".format(timeout))
self.runner.stop()
def process_results(self):
# when running locally output results in build/raptor.json; when running
# in production output to a local.json to be turned into tc job artifact
@@ -174,16 +178,18 @@ class Raptor(object):
self.log.info("finished")
def main(args=sys.argv[1:]):
args = parse_args()
commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
LOG = get_default_logger(component='raptor-main')
+ LOG.info("received command line arguments: %s" % str(args))
+
# if a test name specified on command line, and it exists, just run that one
# otherwise run all available raptor tests that are found for this browser
raptor_test_list = get_raptor_test_list(args)
# ensure we have at least one valid test to run
if len(raptor_test_list) == 0:
LOG.critical("abort: no tests found")
sys.exit(1)
deleted file mode 100644
--- a/testing/raptor/raptor/tests/raptor-firefox-tp6.ini
+++ /dev/null
@@ -1,39 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# raptor tp6 on firefox
-
-[DEFAULT]
-apps = firefox
-type = pageload
-playback = mitmproxy
-playback_binary_manifest = mitmproxy-rel-bin-osx.manifest
-playback_binary_zip_mac = mitmproxy-2.0.2-osx.tar.gz
-playback_pageset_manifest = mitmproxy-playback-set.manifest
-playback_pageset_zip_mac = mitmproxy-recording-set-win10.zip
-page_cycles = 25
-unit = ms
-lower_is_better = true
-alert_threshold = 2.0
-
-[raptor-firefox-tp6-amazon]
-test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
-playback_recordings = mitmproxy-recording-amazon.mp
-measure = fnbpaint
-
-[raptor-firefox-tp6-facebook]
-test_url = https://www.facebook.com
-playback_recordings = mitmproxy-recording-facebook.mp
-measure = fnbpaint
-
-[raptor-firefox-tp6-google]
-test_url = https://www.google.com/#hl=en&q=barack+obama
-playback_recordings = mitmproxy-recording-google.mp
-measure = fnbpaint, hero
-hero = hero
-
-[raptor-firefox-tp6-youtube]
-test_url = https://www.youtube.com
-playback_recordings = mitmproxy-recording-youtube.mp
-measure = fnbpaint
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ b/testing/raptor/raptor/tests/raptor-speedometer.ini
@@ -1,15 +1,20 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# speedometer benchmark for firefox and chrome
-[raptor-speedometer]
-apps = firefox
+[DEFAULT]
type = benchmark
test_url = http://localhost:<port>/Speedometer/index.html?raptor
-page_cycles = 5
+page_cycles = 1
page_timeout = 120000
unit = score
lower_is_better = false
alert_threshold = 2.0
+
+[raptor-speedometer-firefox]
+apps = firefox
+
+[raptor-speedometer-chrome]
+apps = chrome
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6.ini
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6
+
+[DEFAULT]
+type = pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-osx.manifest
+playback_binary_zip_mac = mitmproxy-2.0.2-osx.tar.gz
+playback_pageset_manifest = mitmproxy-playback-set.manifest
+playback_pageset_zip_mac = mitmproxy-recording-set-win10.zip
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+
+[raptor-firefox-tp6-amazon]
+apps = firefox
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = mitmproxy-recording-amazon.mp
+measure = fnbpaint
+
+[raptor-firefox-tp6-facebook]
+apps = firefox
+test_url = https://www.facebook.com
+playback_recordings = mitmproxy-recording-facebook.mp
+measure = fnbpaint
+
+[raptor-firefox-tp6-google]
+apps = firefox
+test_url = https://www.google.com/#hl=en&q=barack+obama
+playback_recordings = mitmproxy-recording-google.mp
+measure = fnbpaint, hero
+hero = hero
+
+[raptor-firefox-tp6-youtube]
+apps = firefox
+test_url = https://www.youtube.com
+playback_recordings = mitmproxy-recording-youtube.mp
+measure = fnbpaint
+
+[raptor-chrome-tp6-amazon]
+apps = chrome
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = mitmproxy-recording-amazon.mp
+measure = fcp
+
+[raptor-chrome-tp6-facebook]
+apps = chrome
+test_url = https://www.facebook.com
+playback_recordings = mitmproxy-recording-facebook.mp
+measure = fcp
+
+[raptor-chrome-tp6-google]
+apps = chrome
+test_url = https://www.google.com/#hl=en&q=barack+obama
+playback_recordings = mitmproxy-recording-google.mp
+measure = fcp, hero
+hero = hero
+
+[raptor-chrome-tp6-youtube]
+apps = chrome
+test_url = https://www.youtube.com
+playback_recordings = mitmproxy-recording-youtube.mp
+measure = fcp
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -11,16 +11,17 @@
"background": {
"scripts": ["auto_gen_test_config.js", "runner.js"]
},
"content_scripts": [
{
"matches": ["*://*.amazon.com/*",
"*://*.facebook.com/*",
"*://*.google.com/*",
+ "*://*.google.ca/*",
"*://*.youtube.com/*"],
"js": ["measure.js"]
},
{
"matches": ["*://*/Speedometer/index.html*"],
"js": ["benchmark-relay.js"]
}
],