Bug 1286312 - Add mochitest option to run tests using https draft
authorpyang <pyang@mozilla.com>
Fri, 30 Dec 2016 18:08:28 +0800
changeset 454792 b518721d32e1cb34b13c251641d4a0210baf957c
parent 454351 143bb4b9249e528e658f6ccc449991794b8675f8
child 454793 3e20b0fbb57cf39b92301ad76d80d8e79c9decce
push id40048
push userbmo:pyang@mozilla.com
push dateFri, 30 Dec 2016 10:09:51 +0000
bugs1286312
milestone53.0a1
Bug 1286312 - Add mochitest option to run tests using https MozReview-Commit-ID: 8ysuhB9nOJw
testing/mochitest/runtests.py
testing/mochitest/tests/SimpleTest/setup.js
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -786,17 +786,17 @@ class MochitestDesktop(object):
         self.websocketProcessBridge = None
         self.sslTunnel = None
         self._active_tests = None
         self._locations = None
 
         self.marionette = None
         self.start_script = None
         self.mozLogs = None
-        self.start_script_args = []
+        self.start_script_kwargs = {}
         self.urlOpts = []
 
         if self.log is None:
             commandline.log_formatters["tbpl"] = (
                 MochitestFormatter,
                 "Mochitest specific tbpl formatter")
             self.log = commandline.setup_logging("mochitest",
                                                  logger_options,
@@ -1009,18 +1009,21 @@ class MochitestDesktop(object):
             self.testRoot = options.flavor
 
             if options.flavor == 'browser' and options.immersiveMode:
                 self.testRoot = 'metro'
         else:
             self.testRoot = self.TEST_PATH
         self.testRootAbs = os.path.join(SCRIPT_DIR, self.testRoot)
 
-    def buildTestURL(self, options):
-        testHost = "http://mochi.test:8888"
+    def buildTestURL(self, options, scheme='http'):
+        if scheme == 'https':
+            testHost = "https://example.com:443"
+        else:
+            testHost = "http://mochi.test:8888"
         testURL = "/".join([testHost, self.TEST_PATH])
 
         if len(options.test_paths) == 1:
             if options.repeat > 0 and os.path.isfile(
                 os.path.join(
                     self.oldcwd,
                     os.path.dirname(__file__),
                     self.TEST_PATH,
@@ -1032,38 +1035,39 @@ class MochitestDesktop(object):
         if options.flavor in ('a11y', 'chrome'):
             testURL = "/".join([testHost, self.CHROME_PATH])
         elif options.flavor in ('browser', 'jetpack-addon', 'jetpack-package'):
             testURL = "about:blank"
         if options.nested_oop:
             testURL = "/".join([testHost, self.NESTED_OOP_TEST_PATH])
         return testURL
 
-    def buildTestPath(self, options, testsToFilter=None, disabled=True):
+    def getTestsByScheme(self, options, testsToFilter=None, disabled=True):
         """ Build the url path to the specific test harness and test file or directory
             Build a manifest of tests to run and write out a json file for the harness to read
             testsToFilter option is used to filter/keep the tests provided in the list
 
             disabled -- This allows to add all disabled tests on the build side
                         and then on the run side to only run the enabled ones
         """
 
         tests = self.getActiveTests(options, disabled)
         paths = []
         for test in tests:
             if testsToFilter and (test['path'] not in testsToFilter):
                 continue
             paths.append(test)
 
-        # Bug 883865 - add this functionality into manifestparser
-        with open(os.path.join(SCRIPT_DIR, options.testRunManifestFile), 'w') as manifestFile:
-            manifestFile.write(json.dumps({'tests': paths}))
-        options.manifestFile = options.testRunManifestFile
-
-        return self.buildTestURL(options)
+        # Generate test by schemes
+        for (scheme, grouped_tests) in self.groupTestsByScheme(paths).items():
+            # Bug 883865 - add this functionality into manifestparser
+            with open(os.path.join(SCRIPT_DIR, options.testRunManifestFile), 'w') as manifestFile:
+                manifestFile.write(json.dumps({'tests': grouped_tests}))
+            options.manifestFile = options.testRunManifestFile
+            yield (scheme, grouped_tests)
 
     def startWebSocketServer(self, options, debuggerInfo):
         """ Launch the websocket server """
         self.wsserver = WebSocketServer(
             options,
             SCRIPT_DIR,
             self.log,
             debuggerInfo)
@@ -1384,16 +1388,18 @@ toolbar#nav-bar {
                     (test['name'], test['manifest']))
                 continue
 
             testob = {'path': tp}
             if 'disabled' in test:
                 testob['disabled'] = test['disabled']
             if 'expected' in test:
                 testob['expected'] = test['expected']
+            if 'scheme' in test:
+                testob['scheme'] = test['scheme']
             paths.append(testob)
 
         def path_sort(ob1, ob2):
             path1 = ob1['path'].split('/')
             path2 = ob2['path'].split('/')
             return cmp(path1, path2)
 
         paths.sort(path_sort)
@@ -1557,17 +1563,22 @@ toolbar#nav-bar {
 
         if os.path.isfile(self.start_script):
             with open(self.start_script, 'r') as fh:
                 script = fh.read()
         else:
             script = self.start_script
 
         with self.marionette.using_context('chrome'):
-            return self.marionette.execute_script(script, script_args=self.start_script_args)
+            start_script_args = []
+            if 'flavor' in self.start_script_kwargs:
+                start_script_args.append(self.start_script_kwargs['flavor'])
+            if 'testUrl' in self.start_script_kwargs:
+                start_script_args.append(self.start_script_kwargs['testUrl'])
+            return self.marionette.execute_script(script, script_args=start_script_args)
 
     def fillCertificateDB(self, options):
         # TODO: move -> mozprofile:
         # https://bugzilla.mozilla.org/show_bug.cgi?id=746243#c35
 
         pwfilePath = os.path.join(options.profilePath, ".crtdbpw")
         with open(pwfilePath, "w") as pwfile:
             pwfile.write("\n")
@@ -1934,17 +1945,17 @@ toolbar#nav-bar {
 
             # build command line
             cmd = os.path.abspath(app)
             args = list(extraArgs)
             args.append('-marionette')
             # TODO: mozrunner should use -foreground at least for mac
             # https://bugzilla.mozilla.org/show_bug.cgi?id=916512
             args.append('-foreground')
-            self.start_script_args.append(testUrl or 'about:blank')
+            self.start_script_kwargs['testUrl'] = testUrl or 'about:blank'
 
             if detectShutdownLeaks and not self.disable_leak_checking:
                 shutdownLeaks = ShutdownLeaks(self.log)
             else:
                 shutdownLeaks = None
 
             if mozinfo.info["asan"] and (mozinfo.isLinux or mozinfo.isMac) \
                     and not self.disable_leak_checking:
@@ -1998,17 +2009,17 @@ toolbar#nav-bar {
                          interactive=interactive,
                          outputTimeout=timeout)
             proc = runner.process_handler
             self.log.info("runtests.py | Application pid: %d" % proc.pid)
             self.log.process_start("Main app process")
 
             # start marionette and kick off the tests
             marionette_args = marionette_args or {}
-            port_timeout = marionette_args.pop('port_timeout')
+            port_timeout = marionette_args.pop('port_timeout', 60)
             self.marionette = Marionette(**marionette_args)
             self.marionette.start_session(timeout=port_timeout)
 
             # install specialpowers and mochikit as temporary addons
             addons = Addons(self.marionette)
 
             if mozinfo.info.get('toolkit') != 'gonk':
                 addons.install(os.path.join(here, 'extensions', 'specialpowers'), temp=True)
@@ -2062,16 +2073,18 @@ toolbar#nav-bar {
 
             if crash_count or zombieProcesses:
                 status = 1
 
         finally:
             # cleanup
             if os.path.exists(processLog):
                 os.remove(processLog)
+            self.urlOpts = []
+            del(self.start_script_kwargs['testUrl'])
 
         return status
 
     def initializeLooping(self, options):
         """
           This method is used to clear the contents before each run of for loop.
           This method is used for --run-by-dir and --bisect-chunk.
         """
@@ -2164,16 +2177,30 @@ toolbar#nav-bar {
         # We need to print the summary only if options.bisectChunk has a value.
         # Also we need to make sure that we do not print the summary in between
         # running tests via --run-by-dir.
         if options.bisectChunk and options.bisectChunk in self.result:
             bisect.print_summary()
 
         return result
 
+    def groupTestsByScheme(self, tests):
+        """
+        split tests into groups by schemes. test is classified as http if
+        no scheme specified
+        """
+        httpTests = []
+        httpsTests = []
+        for test in tests:
+            if not test.get('scheme') or test.get('scheme') == 'http':
+                httpTests.append(test)
+            elif test.get('scheme') == 'https':
+                httpsTests.append(test)
+        return {'http': httpTests, 'https': httpsTests}
+
     def runTests(self, options):
         """ Prepare, configure, run tests and cleanup """
 
         # a11y and chrome tests don't run with e10s enabled in CI. Need to set
         # this here since |mach mochitest| sets the flavor after argument parsing.
         if options.flavor in ('a11y', 'chrome'):
             options.e10s = False
         mozinfo.update({"e10s": options.e10s})  # for test manifest parsing.
@@ -2338,38 +2365,16 @@ toolbar#nav-bar {
 
         if self.mozLogs:
             self.browserEnv["MOZ_LOG_FILE"] = "{}/moz-pid=%PID-uid={}.log".format(
                 self.browserEnv["MOZ_UPLOAD_DIR"], str(uuid.uuid4()))
 
         try:
             self.startServers(options, debuggerInfo)
 
-            # testsToFilter parameter is used to filter out the test list that
-            # is sent to buildTestPath
-            testURL = self.buildTestPath(options, testsToFilter)
-
-            # read the number of tests here, if we are not going to run any,
-            # terminate early
-            if os.path.exists(
-                os.path.join(
-                    SCRIPT_DIR,
-                    options.testRunManifestFile)):
-                with open(os.path.join(SCRIPT_DIR, options.testRunManifestFile)) as fHandle:
-                    tests = json.load(fHandle)
-                count = 0
-                for test in tests['tests']:
-                    count += 1
-                if count == 0:
-                    return 1
-
-            self.buildURLOptions(options, self.browserEnv)
-            if self.urlOpts:
-                testURL += "?" + "&".join(self.urlOpts)
-
             if options.immersiveMode:
                 options.browserArgs.extend(('-firefoxpath', options.app))
                 options.app = self.immersiveHelperPath
 
             if options.jsdebugger:
                 options.browserArgs.extend(['-jsdebugger'])
 
             # Remove the leak detection file so it can't "leak" to the tests run.
@@ -2388,47 +2393,62 @@ toolbar#nav-bar {
 
             # Detect shutdown leaks for m-bc runs if
             # code coverage is not enabled.
             detectShutdownLeaks = False
             if options.jscov_dir_prefix is None:
                 detectShutdownLeaks = mozinfo.info[
                     "debug"] and options.flavor == 'browser'
 
-            self.start_script_args.append(self.normflavor(options.flavor))
+            self.start_script_kwargs['flavor'] = self.normflavor(options.flavor)
             marionette_args = {
                 'symbols_path': options.symbolsPath,
                 'socket_timeout': options.marionette_socket_timeout,
                 'port_timeout': options.marionette_port_timeout,
             }
 
             if options.marionette:
                 host, port = options.marionette.split(':')
                 marionette_args['host'] = host
                 marionette_args['port'] = int(port)
 
-            self.log.info("runtests.py | Running with e10s: {}".format(options.e10s))
-            self.log.info("runtests.py | Running tests: start.\n")
-            status = self.runApp(testURL,
-                                 self.browserEnv,
-                                 options.app,
-                                 profile=self.profile,
-                                 extraArgs=options.browserArgs,
-                                 utilityPath=options.utilityPath,
-                                 debuggerInfo=debuggerInfo,
-                                 valgrindPath=valgrindPath,
-                                 valgrindArgs=valgrindArgs,
-                                 valgrindSuppFiles=valgrindSuppFiles,
-                                 symbolsPath=options.symbolsPath,
-                                 timeout=timeout,
-                                 detectShutdownLeaks=detectShutdownLeaks,
-                                 screenshotOnFail=options.screenshotOnFail,
-                                 bisectChunk=options.bisectChunk,
-                                 marionette_args=marionette_args,
-                                 )
+            # testsToFilter parameter is used to filter out the test list that
+            # is sent to getTestsByScheme
+            for (scheme, tests) in self.getTestsByScheme(options, testsToFilter):
+                # read the number of tests here, if we are not going to run any,
+                # terminate early
+                if not tests:
+                    continue
+
+                testURL = self.buildTestURL(options, scheme=scheme)
+
+                self.buildURLOptions(options, self.browserEnv)
+                if self.urlOpts:
+                    testURL += "?" + "&".join(self.urlOpts)
+
+                self.log.info("runtests.py | Running with e10s: {}".format(options.e10s))
+                self.log.info("runtests.py | Running tests: start.\n")
+                self.log.info("runApp args: {}".format(self.start_script_kwargs))
+                status = self.runApp(testURL,
+                                     self.browserEnv,
+                                     options.app,
+                                     profile=self.profile,
+                                     extraArgs=options.browserArgs,
+                                     utilityPath=options.utilityPath,
+                                     debuggerInfo=debuggerInfo,
+                                     valgrindPath=valgrindPath,
+                                     valgrindArgs=valgrindArgs,
+                                     valgrindSuppFiles=valgrindSuppFiles,
+                                     symbolsPath=options.symbolsPath,
+                                     timeout=timeout,
+                                     detectShutdownLeaks=detectShutdownLeaks,
+                                     screenshotOnFail=options.screenshotOnFail,
+                                     bisectChunk=options.bisectChunk,
+                                     marionette_args=marionette_args,
+                                     )
         except KeyboardInterrupt:
             self.log.info("runtests.py | Received keyboard interrupt.\n")
             status = -1
         except:
             traceback.print_exc()
             self.log.error(
                 "Automation Error: Received unexpected exception while running application\n")
             status = 1
--- a/testing/mochitest/tests/SimpleTest/setup.js
+++ b/testing/mochitest/tests/SimpleTest/setup.js
@@ -155,17 +155,17 @@ TestRunner.logger.addListener("dumpListe
 
 var gTestList = [];
 var RunSet = {};
 RunSet.runall = function(e) {
   // Filter tests to include|exclude tests based on data in params.filter.
   // This allows for including or excluding tests from the gTestList
   // TODO Only used by ipc tests, remove once those are implemented sanely
   if (params.testManifest) {
-    getTestManifest("http://mochi.test:8888/" + params.testManifest, params, function(filter) { gTestList = filterTests(filter, gTestList, params.runOnly); RunSet.runtests(); });
+    getTestManifest(getTestManifestURL(params.testManifest), params, function(filter) { gTestList = filterTests(filter, gTestList, params.runOnly); RunSet.runtests(); });
   } else {
     RunSet.runtests();
   }
 }
 
 RunSet.runtests = function(e) {
   // Which tests we're going to run
   var my_tests = gTestList;
@@ -230,17 +230,17 @@ function toggleNonTests (e) {
   } else {
     $("toggleNonTests").innerHTML = "Show Non-Tests";
   }
 }
 
 // hook up our buttons
 function hookup() {
   if (params.manifestFile) {
-    getTestManifest("http://mochi.test:8888/" + params.manifestFile, params, hookupTests);
+    getTestManifest(getTestManifestURL(params.manifestFile), params, hookupTests);
   } else {
     hookupTests(gTestList);
   }
 }
 
 function hookupTests(testList) {
   if (testList.length > 0) {
     gTestList = testList;
@@ -253,8 +253,18 @@ function hookupTests(testList) {
 
   document.getElementById('runtests').onclick = RunSet.reloadAndRunAll;
   document.getElementById('toggleNonTests').onclick = toggleNonTests;
   // run automatically if autorun specified
   if (params.autorun) {
     RunSet.runall();
   }
 }
+
+function getTestManifestURL(path) {
+  // The test manifest url scheme should be the same protocol as the containing
+  // window... unless it's not http(s)
+  if (window.location.protocol == "http:" ||
+      window.location.protocol == "https:") {
+    return window.location.protocol + "//" + window.location.host + "/" + path;
+  }
+  return "http://mochi.test:8888/" + path;
+}