Bug 1455872 - Raptor fixes for production; r?ahal draft
authorRob Wood <rwood@mozilla.com>
Tue, 01 May 2018 14:02:39 -0400
changeset 793132 e26fddcb0e53256a2bd54b893982a77f0ec576ec
parent 793131 b18672eca2d3922367d3e4e50ff4b451a06e479e
child 793133 ef48eb36bc54812872de3aef6e6c0b06214599fa
push id109287
push userrwood@mozilla.com
push dateWed, 09 May 2018 14:49:16 +0000
reviewersahal
bugs1455872
milestone62.0a1
Bug 1455872 - Raptor fixes for production; r?ahal MozReview-Commit-ID: KVqypuqHjF9
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/raptor/MANIFEST.in
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/gen_test_config.py
testing/raptor/raptor/playback/mitmproxy.py
testing/raptor/webext/raptor/runner.js
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -158,34 +158,37 @@ class Raptor(TestingMixin, MercurialScri
         if not binary_path:
             self.fatal("Raptor requires a path to the binary.  You can specify binary_path or add download-and-extract to your action list.")
         # raptor options
         if binary_path.endswith('.exe'):
             binary_path = binary_path[:-4]
         options = []
         kw_options = {'binary': binary_path}
         # options overwritten from **kw
-        if 'suite' in self.config:
-            kw_options['suite'] = self.config['suite']
+        if 'test' in self.config:
+            kw_options['test'] = self.config['test']
         if self.config.get('branch'):
             kw_options['branchName'] = self.config['branch']
         if self.symbols_path:
             kw_options['symbolsPath'] = self.symbols_path
         kw_options.update(kw)
         # configure profiling options
         options.extend(self.query_gecko_profile_options())
         # extra arguments
         if args is not None:
             options += args
         if 'raptor_extra_options' in self.config:
             options += self.config['raptor_extra_options']
         if self.config.get('code_coverage', False):
             options.extend(['--code-coverage'])
         for key, value in kw_options.items():
-            options.extend(['--%s' % key, value])
+            if key == "test":
+                options.extend([value])
+            else:
+                options.extend(['--%s' % key, value])
         return options
 
     def populate_webroot(self):
         """Populate the production test slaves' webroots"""
         self.raptor_path = os.path.join(
             self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
         )
 
@@ -315,18 +318,19 @@ class Raptor(TestingMixin, MercurialScri
             self.mkdir_p(env['MOZ_UPLOAD_DIR'])
         env = self.query_env(partial_env=env, log_level=INFO)
         # adjust PYTHONPATH to be able to use raptor as a python package
         if 'PYTHONPATH' in env:
             env['PYTHONPATH'] = self.raptor_path + os.pathsep + env['PYTHONPATH']
         else:
             env['PYTHONPATH'] = self.raptor_path
 
-        # mitmproxy needs path to mozharness when installing the cert
+        # mitmproxy needs path to mozharness when installing the cert, and tooltool
         env['SCRIPTSPATH'] = scripts_path
+        env['EXTERNALTOOLSPATH'] = external_tools_path
 
         if self.repo_path is not None:
             env['MOZ_DEVELOPER_REPO_DIR'] = self.repo_path
         if self.obj_path is not None:
             env['MOZ_DEVELOPER_OBJ_DIR'] = self.obj_path
 
         # sets a timeout for how long raptor should run without output
         output_timeout = self.config.get('raptor_output_timeout', 3600)
--- a/testing/raptor/MANIFEST.in
+++ b/testing/raptor/MANIFEST.in
@@ -1,3 +1,1 @@
-include raptor/preferences/*.json
-include raptor/tests/*.ini
-include requirements.txt
+recursive-include raptor *
\ No newline at end of file
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -14,17 +14,20 @@ def create_parser(mach_interface=False):
     add_arg = parser.add_argument
 
     if not mach_interface:
         add_arg('--app', default='firefox', dest='app',
                 help="name of the application we are testing (default: firefox)",
                 choices=['firefox', 'chrome'])
         add_arg('-b', '--binary', required=True, dest='binary',
                 help="path to the browser executable that we are testing")
-
+        add_arg('--branchName', dest="branch_name", default='',
+                help="Name of the branch we are testing on")
+        add_arg('--symbolsPath', dest='symbols_path',
+                help="Path to the symbols for the build we are testing")
     # remaining arg is test name
     add_arg("test",
             nargs="*",
             help="name of raptor test to run")
 
     add_logging_group(parser)
     return parser
 
--- a/testing/raptor/raptor/gen_test_config.py
+++ b/testing/raptor/raptor/gen_test_config.py
@@ -13,20 +13,22 @@ webext_dir = os.path.join(os.path.dirnam
 LOG = get_proxy_logger(component="gen_test_url")
 
 
 def gen_test_config(browser, test, cs_port):
     LOG.info("writing test settings url background js, so webext can get it")
 
     data = """// this file is auto-generated by raptor, do not edit directly
 function getTestConfig() {
-    return {"browser": "%s", "test_settings_url": "http://localhost:%d/%s.json"};
+    return {"browser": "%s",
+            "cs_port": "%d",
+            "test_settings_url": "http://localhost:%d/%s.json"};
 }
 
-""" % (browser, cs_port, test)
+""" % (browser, cs_port, cs_port, test)
 
     webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
 
     file = open(webext_background_script, "w")
     file.write(data)
     file.close()
 
     LOG.info("finished writing test config into webext")
--- a/testing/raptor/raptor/playback/mitmproxy.py
+++ b/testing/raptor/raptor/playback/mitmproxy.py
@@ -19,17 +19,24 @@ from mozprocess import ProcessHandler
 from .base import Playback
 
 here = os.path.dirname(os.path.realpath(__file__))
 LOG = get_proxy_logger(component='mitmproxy')
 
 mozharness_dir = os.path.join(here, '../../../mozharness')
 sys.path.insert(0, mozharness_dir)
 
-TOOLTOOL_PATH = os.path.join(mozharness_dir, 'external_tools', 'tooltool.py')
+external_tools_path = os.environ.get('EXTERNALTOOLSPATH', None)
+
+if external_tools_path is not None:
+    # running in production via mozharness
+    TOOLTOOL_PATH = os.path.join(external_tools_path, 'tooltool.py')
+else:
+    # running locally via mach
+    TOOLTOOL_PATH = os.path.join(mozharness_dir, 'external_tools', 'tooltool.py')
 
 # path for mitmproxy certificate, generated auto after mitmdump is started
 # on local machine it is 'HOME', however it is different on production machines
 try:
     DEFAULT_CERT_PATH = os.path.join(os.getenv('HOME'),
                                      '.mitmproxy', 'mitmproxy-ca-cert.cer')
 except Exception:
     DEFAULT_CERT_PATH = os.path.join(os.getenv('HOMEDRIVE'), os.getenv('HOMEPATH'),
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -12,16 +12,17 @@
 // repo) or 'webkit/PerformanceTests' dir (for benchmarks) first run:
 // 'python -m SimpleHTTPServer 8081'
 // to serve out the pages that we want to prototype with. Also
 // update the manifest content 'matches' accordingly
 
 var browserName;
 var ext;
 var settingsURL = null;
+var cs_port = null;
 var testType;
 var pageCycles = 0;
 var pageCycle = 0;
 var pageCycleDelay = 1000;
 var testURL;
 var testTabID = 0;
 var results = {"page": "", "measurements": {}};
 var getHero = false;
@@ -281,17 +282,17 @@ function verifyResults() {
                   + x + " but only have " + count);
     }
   }
   postToControlServer("results", results);
 }
 
 function postToControlServer(msgType, msgData) {
   // requires 'control server' running at port 8000 to receive results
-  var url = "http://127.0.0.1:8000/";
+  var url = "http://127.0.0.1:" + cs_port + "/";
   var client = new XMLHttpRequest();
   client.onreadystatechange = function() {
     if (client.readyState == XMLHttpRequest.DONE && client.status == 200) {
       console.log("post success");
     }
   };
 
   client.open("POST", url, true);
@@ -327,16 +328,17 @@ function cleanUp() {
   if (browserName === "firefox")
     window.dump("\n__raptor_shutdownBrowser\n");
 
 }
 
 function runner() {
   let config = getTestConfig();
   settingsURL = config.test_settings_url;
+  cs_port = config.cs_port;
   browserName = config.browser;
   getBrowserInfo().then(function() {
     getTestSettings().then(function() {
       if (testType == "benchmark") {
         // webkit benchmark type of test
         console.log("benchmark test start");
       } else if (testType == "pageload") {
         // standard pageload test