Bug 1458571 - Use base testing/profiles in raptor, r?rwood
This gets raptor to use the newly created "perf" profile that talos
also uses. There is a single pref that raptor sets that we can't set
in talos. To that end, this also creates a "raptor" specific profile.
This means to set a pref in talos and raptor, edit:
testing/profiles/perf/user.js
To set a pref in raptor only, edit:
testing/profiles/raptor/user.js
The performance of extensions can now be tested by dropping the
extension into:
testing/profiles/perf/extensions
MozReview-Commit-ID: LEJeytmmiFF
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -415,16 +415,22 @@ ARCHIVE_FILES = {
'raptor': [
{
'source': buildconfig.topsrcdir,
'base': 'testing',
'pattern': 'raptor/**',
},
{
'source': buildconfig.topsrcdir,
+ 'base': 'testing/profiles',
+ 'pattern': '**',
+ 'dest': 'raptor/raptor/profile_data',
+ },
+ {
+ 'source': buildconfig.topsrcdir,
'base': 'third_party/webkit/PerformanceTests',
'pattern': '**',
'dest': 'raptor/raptor/tests/webkit/PerformanceTests/',
},
],
'awsy': [
{
'source': buildconfig.topsrcdir,
--- a/testing/profiles/profiles.json
+++ b/testing/profiles/profiles.json
@@ -1,7 +1,8 @@
{
"mochitest": ["common"],
"profileserver": ["common"],
- "web-platform-tests": ["common"],
+ "raptor": ["perf", "raptor"],
"talos": ["perf"],
- "valgrind": ["common"]
+ "valgrind": ["common"],
+ "web-platform-tests": ["common"]
}
new file mode 100644
--- /dev/null
+++ b/testing/profiles/raptor/extensions/README.txt
@@ -0,0 +1,2 @@
+Dropping extensions here will get them installed in all test harnesses
+that make use of this profile.
new file mode 100644
--- /dev/null
+++ b/testing/profiles/raptor/user.js
@@ -0,0 +1,3 @@
+// Preferences file used by the raptor harness
+/* globals user_pref */
+user_pref("dom.performance.time_to_non_blank_paint.enabled", true);
deleted file mode 100644
--- a/testing/raptor/raptor/preferences/firefox.json
+++ /dev/null
@@ -1,109 +0,0 @@
-{
- "app.normandy.api_url": "https://127.0.0.1/selfsupport-dummy/",
- "app.update.enabled": false,
- "browser.EULA.override": true,
- "browser.aboutHomeSnippets.updateUrl": "https://127.0.0.1/about-dummy/",
- "browser.addon-watch.interval": -1,
- "browser.bookmarks.max_backups": 0,
- "browser.cache.disk.smart_size.enabled": false,
- "browser.cache.disk.smart_size.first_run": false,
- "browser.chrome.dynamictoolbar": false,
- "browser.contentHandlers.types.0.uri": "http://127.0.0.1/rss?url=%s",
- "browser.contentHandlers.types.1.uri": "http://127.0.0.1/rss?url=%s",
- "browser.contentHandlers.types.2.uri": "http://127.0.0.1/rss?url=%s",
- "browser.contentHandlers.types.3.uri": "http://127.0.0.1/rss?url=%s",
- "browser.contentHandlers.types.4.uri": "http://127.0.0.1/rss?url=%s",
- "browser.contentHandlers.types.5.uri": "http://127.0.0.1/rss?url=%s",
- "browser.dom.window.dump.enabled": true,
- "browser.link.open_newwindow": 2,
- "browser.newtabpage.activity-stream.default.sites": "",
- "browser.newtabpage.activity-stream.feeds.section.topstories": false,
- "browser.newtabpage.activity-stream.feeds.snippets": false,
- "browser.newtabpage.activity-stream.telemetry": false,
- "browser.newtabpage.activity-stream.tippyTop.service.endpoint": "",
- "browser.ping-centre.production.endpoint": "https://127.0.0.1/pingcentre/dummy/",
- "browser.ping-centre.staging.endpoint": "https://127.0.0.1/pingcentre/dummy/",
- "browser.reader.detectedFirstArticle": true,
- "browser.safebrowsing.blockedURIs.enabled": false,
- "browser.safebrowsing.downloads.enabled": false,
- "browser.safebrowsing.downloads.remote.url": "http://127.0.0.1/safebrowsing-dummy/downloads",
- "browser.safebrowsing.malware.enabled": false,
- "browser.safebrowsing.passwords.enabled": false,
- "browser.safebrowsing.phishing.enabled": false,
- "browser.safebrowsing.provider.google.gethashURL": "http://127.0.0.1/safebrowsing-dummy/gethash",
- "browser.safebrowsing.provider.google.updateURL": "http://127.0.0.1/safebrowsing-dummy/update",
- "browser.safebrowsing.provider.google4.gethashURL": "http://127.0.0.1/safebrowsing4-dummy/gethash",
- "browser.safebrowsing.provider.google4.updateURL": "http://127.0.0.1/safebrowsing4-dummy/update",
- "browser.safebrowsing.provider.mozilla.gethashURL": "http://127.0.0.1/safebrowsing-dummy/gethash",
- "browser.safebrowsing.provider.mozilla.updateURL": "http://127.0.0.1/safebrowsing-dummy/update",
- "browser.search.countryCode": "US",
- "browser.search.geoSpecificDefaults": false,
- "browser.search.geoip.url": "",
- "browser.search.isUS": true,
- "browser.shell.checkDefaultBrowser": false,
- "browser.snippets.enabled": false,
- "browser.snippets.syncPromo.enabled": false,
- "browser.tabs.remote.autostart": false,
- "browser.urlbar.userMadeSearchSuggestionsChoice": true,
- "browser.warnOnQuit": false,
- "browser.webapps.checkForUpdates": 0,
- "datareporting.healthreport.documentServerURI": "http://127.0.0.1/healthreport/",
- "datareporting.policy.dataSubmissionPolicyBypassNotification": true,
- "devtools.chrome.enabled": false,
- "devtools.debugger.remote-enabled": false,
- "devtools.theme": "light",
- "devtools.timeline.enabled": false,
- "dom.allow_scripts_to_close_windows": true,
- "dom.disable_open_during_load": false,
- "dom.disable_window_flip": true,
- "dom.disable_window_move_resize": true,
- "dom.max_chrome_script_run_time": 0,
- "dom.max_script_run_time": 0,
- "dom.performance.time_to_non_blank_paint.enabled": true,
- "dom.send_after_paint_to_content": true,
- "experiments.manifest.uri": "https://127.0.0.1/experiments-dummy/manifest",
- "extensions.autoDisableScopes": 10,
- "extensions.blocklist.enabled": false,
- "extensions.blocklist.url": "http://127.0.0.1/extensions-dummy/blocklistURL",
- "extensions.checkCompatibility": false,
- "extensions.enabledScopes": 5,
- "extensions.getAddons.get.url": "http://127.0.0.1/extensions-dummy/repositoryGetURL",
- "extensions.getAddons.getWithPerformance.url": "http://127.0.0.1/extensions-dummy/repositoryGetWithPerformanceURL",
- "extensions.getAddons.search.browseURL": "http://127.0.0.1/extensions-dummy/repositoryBrowseURL",
- "extensions.hotfix.url": "http://127.0.0.1/extensions-dummy/hotfixURL",
- "extensions.legacy.enabled": true,
- "extensions.systemAddon.update.url": "http://127.0.0.1/dummy-system-addons.xml",
- "extensions.update.background.url": "http://127.0.0.1/extensions-dummy/updateBackgroundURL",
- "extensions.update.enabled": false,
- "extensions.update.notifyUser": false,
- "extensions.update.url": "http://127.0.0.1/extensions-dummy/updateURL",
- "extensions.webservice.discoverURL": "http://127.0.0.1/extensions-dummy/discoveryURL",
- "general.useragent.updates.enabled": false,
- "hangmonitor.timeout": 0,
- "identity.fxaccounts.auth.uri": "https://127.0.0.1/fxa-dummy/",
- "identity.fxaccounts.migrateToDevEdition": false,
- "lightweightThemes.selectedThemeID": "",
- "media.capturestream_hints.enabled": true,
- "media.gmp-manager.updateEnabled": false,
- "media.gmp-manager.url": "http://127.0.0.1/gmpmanager-dummy/update.xml",
- "media.libavcodec.allow-obsolete": true,
- "media.navigator.enabled": true,
- "media.navigator.permission.disabled": true,
- "media.peerconnection.enabled": true,
- "network.http.speculative-parallel-limit": 0,
- "network.proxy.http": "localhost",
- "network.proxy.http_port": 80,
- "network.proxy.type": 1,
- "places.database.lastMaintenance": 2147483647,
- "plugin.state.flash": 0,
- "plugins.flashBlock.enabled": false,
- "privacy.trackingprotection.annotate_channels": false,
- "privacy.trackingprotection.enabled": false,
- "privacy.trackingprotection.introURL": "http://127.0.0.1/trackingprotection/tour",
- "privacy.trackingprotection.pbmode.enabled": false,
- "security.enable_java": false,
- "security.fileuri.strict_origin_policy": false,
- "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer": true,
- "toolkit.telemetry.server": "https://127.0.0.1/telemetry-dummy/",
- "xpinstall.signatures.required": false
-}
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -16,16 +16,22 @@ from mozlog import commandline, get_defa
from mozprofile import create_profile
from mozrunner import runners
# need this so raptor imports work both from /raptor and via mach
here = os.path.abspath(os.path.dirname(__file__))
webext_dir = os.path.join(os.path.dirname(here), 'webext')
sys.path.insert(0, here)
+try:
+ from mozbuild.base import MozbuildObject
+ build = MozbuildObject.from_environment(cwd=here)
+except ImportError:
+ build = None
+
from cmdline import parse_args
from control_server import RaptorControlServer
from gen_test_config import gen_test_config
from outputhandler import OutputHandler
from playback import get_playback
from manifest import get_raptor_test_list
@@ -39,36 +45,44 @@ class Raptor(object):
self.config['platform'] = mozinfo.os
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
self.log = get_default_logger(component='raptor')
self.control_server = None
self.playback = None
# Create the profile
- pref_file = os.path.join(here, 'preferences', '{}.json'.format(self.config['app']))
- prefs = {}
- if os.path.isfile(pref_file):
- with open(pref_file, 'r') as fh:
- prefs = json.load(fh)
+ self.profile = create_profile(self.config['app'])
- try:
- self.profile = create_profile(self.config['app'], preferences=prefs)
- except NotImplementedError:
- self.profile = None
+ # Merge in base profiles
+ with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
+ base_profiles = json.load(fh)['raptor']
+
+ for name in base_profiles:
+ path = os.path.join(self.profile_data_dir, name)
+ self.log.info("Merging profile: {}".format(path))
+ self.profile.merge(path)
# Create the runner
self.output_handler = OutputHandler()
process_args = {
'processOutputLine': [self.output_handler],
}
runner_cls = runners[app]
self.runner = runner_cls(
binary, profile=self.profile, process_args=process_args)
+ @property
+ def profile_data_dir(self):
+ if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
+ return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
+ if build:
+ return os.path.join(build.topsrcdir, 'testing', 'profiles')
+ return os.path.join(here, 'profile_data')
+
def start_control_server(self):
self.control_server = RaptorControlServer()
self.control_server.start()
def get_playback_config(self, test):
self.config['playback_tool'] = test.get('playback')
self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)