Bug 1458571 - Use base testing/profiles in raptor, r?rwood draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 04 May 2018 11:19:49 -0400
changeset 794166 151b8ac8e37b9e11c523f667a5262aaf7351c075
parent 794165 9b5c0274098f115035d3bcd54e810b6490a225e5
child 794167 9ee954fde02518f7f2b0bc5411cff5d5fa4c8931
push id109588
push userahalberstadt@mozilla.com
push dateFri, 11 May 2018 13:03:42 +0000
reviewersrwood
bugs1458571
milestone62.0a1
Bug 1458571 - Use base testing/profiles in raptor, r?rwood This gets raptor to use the newly created "perf" profile that talos also uses. There is a single pref that raptor sets that we can't set in talos. To that end, this also creates a "raptor" specific profile. This means to set a pref in talos and raptor, edit: testing/profiles/perf/user.js To set a pref in raptor only, edit: testing/profiles/raptor/user.js The performance of extensions can now be tested by dropping the extension into: testing/profiles/perf/extensions MozReview-Commit-ID: LEJeytmmiFF
python/mozbuild/mozbuild/action/test_archive.py
testing/profiles/profiles.json
testing/profiles/raptor/extensions/README.txt
testing/profiles/raptor/user.js
testing/raptor/raptor/preferences/firefox.json
testing/raptor/raptor/raptor.py
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -415,16 +415,22 @@ ARCHIVE_FILES = {
     'raptor': [
         {
             'source': buildconfig.topsrcdir,
             'base': 'testing',
             'pattern': 'raptor/**',
         },
         {
             'source': buildconfig.topsrcdir,
+            'base': 'testing/profiles',
+            'pattern': '**',
+            'dest': 'raptor/raptor/profile_data',
+        },
+        {
+            'source': buildconfig.topsrcdir,
             'base': 'third_party/webkit/PerformanceTests',
             'pattern': '**',
             'dest': 'raptor/raptor/tests/webkit/PerformanceTests/',
         },
     ],
     'awsy': [
         {
             'source': buildconfig.topsrcdir,
--- a/testing/profiles/profiles.json
+++ b/testing/profiles/profiles.json
@@ -1,7 +1,8 @@
 {
     "mochitest": ["common"],
     "profileserver": ["common"],
-    "web-platform-tests": ["common"],
+    "raptor": ["perf", "raptor"],
     "talos": ["perf"],
-    "valgrind": ["common"]
+    "valgrind": ["common"],
+    "web-platform-tests": ["common"]
 }
new file mode 100644
--- /dev/null
+++ b/testing/profiles/raptor/extensions/README.txt
@@ -0,0 +1,2 @@
+Dropping extensions here will get them installed in all test harnesses
+that make use of this profile.
new file mode 100644
--- /dev/null
+++ b/testing/profiles/raptor/user.js
@@ -0,0 +1,3 @@
+// Preferences file used by the raptor harness
+/* globals user_pref */
+user_pref("dom.performance.time_to_non_blank_paint.enabled", true);
deleted file mode 100644
--- a/testing/raptor/raptor/preferences/firefox.json
+++ /dev/null
@@ -1,109 +0,0 @@
-{
-    "app.normandy.api_url": "https://127.0.0.1/selfsupport-dummy/",
-    "app.update.enabled": false,
-    "browser.EULA.override": true,
-    "browser.aboutHomeSnippets.updateUrl": "https://127.0.0.1/about-dummy/",
-    "browser.addon-watch.interval": -1,
-    "browser.bookmarks.max_backups": 0,
-    "browser.cache.disk.smart_size.enabled": false,
-    "browser.cache.disk.smart_size.first_run": false,
-    "browser.chrome.dynamictoolbar": false,
-    "browser.contentHandlers.types.0.uri": "http://127.0.0.1/rss?url=%s",
-    "browser.contentHandlers.types.1.uri": "http://127.0.0.1/rss?url=%s",
-    "browser.contentHandlers.types.2.uri": "http://127.0.0.1/rss?url=%s",
-    "browser.contentHandlers.types.3.uri": "http://127.0.0.1/rss?url=%s",
-    "browser.contentHandlers.types.4.uri": "http://127.0.0.1/rss?url=%s",
-    "browser.contentHandlers.types.5.uri": "http://127.0.0.1/rss?url=%s",
-    "browser.dom.window.dump.enabled": true,
-    "browser.link.open_newwindow": 2,
-    "browser.newtabpage.activity-stream.default.sites": "",
-    "browser.newtabpage.activity-stream.feeds.section.topstories": false,
-    "browser.newtabpage.activity-stream.feeds.snippets": false,
-    "browser.newtabpage.activity-stream.telemetry": false,
-    "browser.newtabpage.activity-stream.tippyTop.service.endpoint": "",
-    "browser.ping-centre.production.endpoint": "https://127.0.0.1/pingcentre/dummy/",
-    "browser.ping-centre.staging.endpoint": "https://127.0.0.1/pingcentre/dummy/",
-    "browser.reader.detectedFirstArticle": true,
-    "browser.safebrowsing.blockedURIs.enabled": false,
-    "browser.safebrowsing.downloads.enabled": false,
-    "browser.safebrowsing.downloads.remote.url": "http://127.0.0.1/safebrowsing-dummy/downloads",
-    "browser.safebrowsing.malware.enabled": false,
-    "browser.safebrowsing.passwords.enabled": false,
-    "browser.safebrowsing.phishing.enabled": false,
-    "browser.safebrowsing.provider.google.gethashURL": "http://127.0.0.1/safebrowsing-dummy/gethash",
-    "browser.safebrowsing.provider.google.updateURL": "http://127.0.0.1/safebrowsing-dummy/update",
-    "browser.safebrowsing.provider.google4.gethashURL": "http://127.0.0.1/safebrowsing4-dummy/gethash",
-    "browser.safebrowsing.provider.google4.updateURL": "http://127.0.0.1/safebrowsing4-dummy/update",
-    "browser.safebrowsing.provider.mozilla.gethashURL": "http://127.0.0.1/safebrowsing-dummy/gethash",
-    "browser.safebrowsing.provider.mozilla.updateURL": "http://127.0.0.1/safebrowsing-dummy/update",
-    "browser.search.countryCode": "US",
-    "browser.search.geoSpecificDefaults": false,
-    "browser.search.geoip.url": "",
-    "browser.search.isUS": true,
-    "browser.shell.checkDefaultBrowser": false,
-    "browser.snippets.enabled": false,
-    "browser.snippets.syncPromo.enabled": false,
-    "browser.tabs.remote.autostart": false,
-    "browser.urlbar.userMadeSearchSuggestionsChoice": true,
-    "browser.warnOnQuit": false,
-    "browser.webapps.checkForUpdates": 0,
-    "datareporting.healthreport.documentServerURI": "http://127.0.0.1/healthreport/",
-    "datareporting.policy.dataSubmissionPolicyBypassNotification": true,
-    "devtools.chrome.enabled": false,
-    "devtools.debugger.remote-enabled": false,
-    "devtools.theme": "light",
-    "devtools.timeline.enabled": false,
-    "dom.allow_scripts_to_close_windows": true,
-    "dom.disable_open_during_load": false,
-    "dom.disable_window_flip": true,
-    "dom.disable_window_move_resize": true,
-    "dom.max_chrome_script_run_time": 0,
-    "dom.max_script_run_time": 0,
-    "dom.performance.time_to_non_blank_paint.enabled": true,
-    "dom.send_after_paint_to_content": true,
-    "experiments.manifest.uri": "https://127.0.0.1/experiments-dummy/manifest",
-    "extensions.autoDisableScopes": 10,
-    "extensions.blocklist.enabled": false,
-    "extensions.blocklist.url": "http://127.0.0.1/extensions-dummy/blocklistURL",
-    "extensions.checkCompatibility": false,
-    "extensions.enabledScopes": 5,
-    "extensions.getAddons.get.url": "http://127.0.0.1/extensions-dummy/repositoryGetURL",
-    "extensions.getAddons.getWithPerformance.url": "http://127.0.0.1/extensions-dummy/repositoryGetWithPerformanceURL",
-    "extensions.getAddons.search.browseURL": "http://127.0.0.1/extensions-dummy/repositoryBrowseURL",
-    "extensions.hotfix.url": "http://127.0.0.1/extensions-dummy/hotfixURL",
-    "extensions.legacy.enabled": true,
-    "extensions.systemAddon.update.url": "http://127.0.0.1/dummy-system-addons.xml",
-    "extensions.update.background.url": "http://127.0.0.1/extensions-dummy/updateBackgroundURL",
-    "extensions.update.enabled": false,
-    "extensions.update.notifyUser": false,
-    "extensions.update.url": "http://127.0.0.1/extensions-dummy/updateURL",
-    "extensions.webservice.discoverURL": "http://127.0.0.1/extensions-dummy/discoveryURL",
-    "general.useragent.updates.enabled": false,
-    "hangmonitor.timeout": 0,
-    "identity.fxaccounts.auth.uri": "https://127.0.0.1/fxa-dummy/",
-    "identity.fxaccounts.migrateToDevEdition": false,
-    "lightweightThemes.selectedThemeID": "",
-    "media.capturestream_hints.enabled": true,
-    "media.gmp-manager.updateEnabled": false,
-    "media.gmp-manager.url": "http://127.0.0.1/gmpmanager-dummy/update.xml",
-    "media.libavcodec.allow-obsolete": true,
-    "media.navigator.enabled": true,
-    "media.navigator.permission.disabled": true,
-    "media.peerconnection.enabled": true,
-    "network.http.speculative-parallel-limit": 0,
-    "network.proxy.http": "localhost",
-    "network.proxy.http_port": 80,
-    "network.proxy.type": 1,
-    "places.database.lastMaintenance": 2147483647,
-    "plugin.state.flash": 0,
-    "plugins.flashBlock.enabled": false,
-    "privacy.trackingprotection.annotate_channels": false,
-    "privacy.trackingprotection.enabled": false,
-    "privacy.trackingprotection.introURL": "http://127.0.0.1/trackingprotection/tour",
-    "privacy.trackingprotection.pbmode.enabled": false,
-    "security.enable_java": false,
-    "security.fileuri.strict_origin_policy": false,
-    "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer": true,
-    "toolkit.telemetry.server": "https://127.0.0.1/telemetry-dummy/",
-    "xpinstall.signatures.required": false
-}
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -16,16 +16,22 @@ from mozlog import commandline, get_defa
 from mozprofile import create_profile
 from mozrunner import runners
 
 # need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext')
 sys.path.insert(0, here)
 
+try:
+    from mozbuild.base import MozbuildObject
+    build = MozbuildObject.from_environment(cwd=here)
+except ImportError:
+    build = None
+
 from cmdline import parse_args
 from control_server import RaptorControlServer
 from gen_test_config import gen_test_config
 from outputhandler import OutputHandler
 from playback import get_playback
 from manifest import get_raptor_test_list
 
 
@@ -39,36 +45,44 @@ class Raptor(object):
         self.config['platform'] = mozinfo.os
 
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.log = get_default_logger(component='raptor')
         self.control_server = None
         self.playback = None
 
         # Create the profile
-        pref_file = os.path.join(here, 'preferences', '{}.json'.format(self.config['app']))
-        prefs = {}
-        if os.path.isfile(pref_file):
-            with open(pref_file, 'r') as fh:
-                prefs = json.load(fh)
+        self.profile = create_profile(self.config['app'])
 
-        try:
-            self.profile = create_profile(self.config['app'], preferences=prefs)
-        except NotImplementedError:
-            self.profile = None
+        # Merge in base profiles
+        with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
+            base_profiles = json.load(fh)['raptor']
+
+        for name in base_profiles:
+            path = os.path.join(self.profile_data_dir, name)
+            self.log.info("Merging profile: {}".format(path))
+            self.profile.merge(path)
 
         # Create the runner
         self.output_handler = OutputHandler()
         process_args = {
             'processOutputLine': [self.output_handler],
         }
         runner_cls = runners[app]
         self.runner = runner_cls(
             binary, profile=self.profile, process_args=process_args)
 
+    @property
+    def profile_data_dir(self):
+        if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
+            return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
+        if build:
+            return os.path.join(build.topsrcdir, 'testing', 'profiles')
+        return os.path.join(here, 'profile_data')
+
     def start_control_server(self):
         self.control_server = RaptorControlServer()
         self.control_server.start()
 
     def get_playback_config(self, test):
         self.config['playback_tool'] = test.get('playback')
         self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
         self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)