Bug 1244816 - Run the SPDY and H2 servers for mochitests. r?ted draft
authorKit Cambridge <kcambridge@mozilla.com>
Tue, 02 Feb 2016 13:24:04 -0800
changeset 328352 f0be03e1630d4f68f9f5019ad41f8e2d6495c973
parent 327605 d07dbd40dcd209124149f331f60dd55c8da33fbe
child 328353 1cc1d55377229d100b8c4a9ec5aad04b05e62eae
push id10356
push userkcambridge@mozilla.com
push dateWed, 03 Feb 2016 02:26:08 +0000
reviewersted
bugs1244816
milestone47.0a1
Bug 1244816 - Run the SPDY and H2 servers for mochitests. r?ted
testing/mochitest/moz.build
testing/mochitest/runtests.py
--- a/testing/mochitest/moz.build
+++ b/testing/mochitest/moz.build
@@ -32,16 +32,20 @@ TEST_HARNESS_FILES.testing.mochitest += 
     '/netwerk/test/httpserver/httpd.js',
     '/testing/mozbase/mozdevice/mozdevice/devicemanager.py',
     '/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py',
     '/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py',
     '/testing/mozbase/mozdevice/mozdevice/droid.py',
     '/testing/mozbase/mozdevice/mozdevice/version_codes.py',
     '/testing/mozbase/mozdevice/mozdevice/Zeroconf.py',
     '/testing/mozbase/moznetwork/moznetwork/moznetwork.py',
+    '/testing/xpcshell/moz-http2',
+    '/testing/xpcshell/moz-spdy',
+    '/testing/xpcshell/node-http2',
+    '/testing/xpcshell/node-spdy',
     'b2g_start_script.js',
     'bisection.py',
     'browser-harness.xul',
     'browser-test-overlay.xul',
     'browser-test.js',
     'cc-analyzer.js',
     'chrome-harness.js',
     'chunkifyTests.js',
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -514,16 +514,18 @@ class MochitestBase(object):
     urlOpts = []
     log = None
 
     def __init__(self, logger_options):
         self.update_mozinfo()
         self.server = None
         self.wsserver = None
         self.sslTunnel = None
+        self.nodeProc = {}
+        self.nodeEnv = {}
         self._active_tests = None
         self._locations = None
 
         if self.log is None:
             commandline.log_formatters["tbpl"] = (
                 MochitestFormatter,
                 "Mochitest specific tbpl formatter")
             self.log = commandline.setup_logging("mochitest",
@@ -1263,16 +1265,79 @@ overlay chrome://browser/content/browser
                 if parts[2] == pname and parts[1] == '1':
                     self.log.info("killing %s orphan with pid %d" % (pname, pid))
                     killPid(pid, self.log)
         process = mozprocess.ProcessHandler(['ps', '-o', 'pid,ppid,comm'],
                                             processOutputLine=_psKill)
         process.run()
         process.wait()
 
+    def trySetupNode(self, options):
+        """
+          Run node for SPDY tests, if available, and updates mozinfo as appropriate.
+        """
+        nodeMozInfo = {'hasNode': False} # Assume the worst
+        nodeBin = None
+
+        # We try to find the node executable in the path given to us by the user in
+        # the MOZ_NODE_PATH environment variable
+        localPath = os.getenv('MOZ_NODE_PATH', None)
+        if localPath and os.path.exists(localPath) and os.path.isfile(localPath):
+            nodeBin = localPath
+
+        if nodeBin:
+            self.log.info('Found node at %s' % (nodeBin,))
+
+            def startServer(name, env, serverJs):
+                if os.path.exists(serverJs):
+                    # OK, we found our SPDY server, let's try to get it running
+                    self.log.info('Found %s at %s' % (name, serverJs))
+                    # We pipe stdin to node because the spdy server will exit when its
+                    # stdin reaches EOF
+                    process = mozprocess.ProcessHandler([nodeBin, serverJs],
+                            stdin=subprocess.PIPE,
+                            env=self.environment(xrePath=options.xrePath),
+                            cwd=os.getcwd())
+                    process.run()
+                    self.nodeProc[name] = process
+
+                    while len(process.output) == 0 and process.poll() is None:
+                        process.wait(timeout=0.1)
+
+                    # Check to make sure the server starts properly by waiting for it to
+                    # tell us it's started
+                    if len(process.output) > 0:
+                        msg = process.output[0]
+                        if 'server listening' in msg:
+                            nodeMozInfo['hasNode'] = True
+                            searchObj = re.search( r'(?:SPDY|HTTP2) server listening on port (.*)', msg, 0)
+                            if searchObj:
+                                self.nodeEnv[env] = searchObj.group(1)
+                else:
+                    self.log.error('Path does not exist! %s' % serverJs)
+
+            startServer('moz-spdy', 'MOZSPDY_PORT', os.path.join(SCRIPT_DIR, 'moz-spdy', 'moz-spdy.js'))
+            startServer('moz-http2', 'MOZHTTP2_PORT', os.path.join(SCRIPT_DIR, 'moz-http2', 'moz-http2.js'))
+        elif os.getenv('MOZ_ASSUME_NODE_RUNNING', None):
+            self.log.info('Assuming required node servers are already running')
+            nodeMozInfo['hasNode'] = True
+
+        mozinfo.update(nodeMozInfo)
+
+    def shutdownNode(self):
+        """
+          Shut down our node process, if it exists
+        """
+        for name, proc in self.nodeProc.iteritems():
+            self.log.info('Node %s server shutting down ...' % name)
+            if proc.poll() is not None:
+                self.log.info('Node server %s already dead %s' % (name, proc.poll()))
+            else:
+                proc.kill(sig=signal.SIGTERM)
+
 
 class SSLTunnel:
 
     def __init__(self, options, logger, ignoreSSLTunnelExts=False):
         self.log = logger
         self.process = None
         self.utilityPath = options.utilityPath
         self.xrePath = options.xrePath
@@ -1652,16 +1717,17 @@ class MochitestDesktop(MochitestBase):
             prefs.update(Preferences.read_prefs(path))
 
         prefs.update(self.extraPrefs(options.extraPrefs))
 
         # interpolate preferences
         interpolation = {
             "server": "%s:%s" %
             (options.webServer, options.httpPort)}
+        interpolation.update(self.nodeEnv)
 
         # TODO: Remove OOP once bug 1072443 is fixed
         if mozinfo.info.get('buildapp') == 'mulet':
             interpolation["OOP"] = "false"
 
         prefs = json.loads(json.dumps(prefs) % interpolation)
         for pref in prefs:
             prefs[pref] = Preferences.cast(prefs[pref])
@@ -2152,58 +2218,65 @@ class MochitestDesktop(MochitestBase):
         # Despite our efforts to clean up servers started by this script, in practice
         # we still see infrequent cases where a process is orphaned and interferes
         # with future tests, typically because the old server is keeping the port in use.
         # Try to avoid those failures by checking for and killing orphan servers before
         # trying to start new ones.
         self.killNamedOrphans('ssltunnel')
         self.killNamedOrphans('xpcshell')
 
-        # Until we have all green, this only runs on bc*/dt*/mochitest-chrome
-        # jobs, not webapprt*, jetpack*, a11yr (for perf reasons), or plain
-
-        testsToRun = self.getTestsToRun(options)
-        if not options.runByDir:
-            return self.runMochitests(options, testsToRun, onLaunch)
-
-        # code for --run-by-dir
-        dirs = self.getDirectories(options)
-
-        result = 1  # default value, if no tests are run.
-        for d in dirs:
-            print "dir: %s" % d
-            tests_in_dir = [t for t in testsToRun if os.path.dirname(t) == d]
-
-            # If we are using --run-by-dir, we should not use the profile path (if) provided
-            # by the user, since we need to create a new directory for each run. We would face problems
-            # if we use the directory provided by the user.
-            result = self.runMochitests(options, tests_in_dir, onLaunch)
-
-            # Dump the logging buffer
-            self.message_logger.dump_buffered()
-
-            if result == -1:
-                break
-
-        # printing total number of tests
-        if options.browserChrome:
-            print "TEST-INFO | checking window state"
-            print "Browser Chrome Test Summary"
-            print "\tPassed: %s" % self.countpass
-            print "\tFailed: %s" % self.countfail
-            print "\tTodo: %s" % self.counttodo
-            print "*** End BrowserChrome Test Results ***"
-        else:
-            print "0 INFO TEST-START | Shutdown"
-            print "1 INFO Passed:  %s" % self.countpass
-            print "2 INFO Failed:  %s" % self.countfail
-            print "3 INFO Todo:    %s" % self.counttodo
-            print "4 INFO SimpleTest FINISHED"
-
-        return result
+        try:
+            # We have to do this before we build the test list so we know whether or
+            # not to run tests that depend on having the node spdy server
+            self.trySetupNode(options)
+
+            # Until we have all green, this only runs on bc*/dt*/mochitest-chrome
+            # jobs, not webapprt*, jetpack*, a11yr (for perf reasons), or plain
+
+            testsToRun = self.getTestsToRun(options)
+            if not options.runByDir:
+                return self.runMochitests(options, testsToRun, onLaunch)
+
+            # code for --run-by-dir
+            dirs = self.getDirectories(options)
+
+            result = 1  # default value, if no tests are run.
+            for d in dirs:
+                print "dir: %s" % d
+                tests_in_dir = [t for t in testsToRun if os.path.dirname(t) == d]
+
+                # If we are using --run-by-dir, we should not use the profile path (if) provided
+                # by the user, since we need to create a new directory for each run. We would face problems
+                # if we use the directory provided by the user.
+                result = self.runMochitests(options, tests_in_dir, onLaunch)
+
+                # Dump the logging buffer
+                self.message_logger.dump_buffered()
+
+                if result == -1:
+                    break
+
+            # printing total number of tests
+            if options.browserChrome:
+                print "TEST-INFO | checking window state"
+                print "Browser Chrome Test Summary"
+                print "\tPassed: %s" % self.countpass
+                print "\tFailed: %s" % self.countfail
+                print "\tTodo: %s" % self.counttodo
+                print "*** End BrowserChrome Test Results ***"
+            else:
+                print "0 INFO TEST-START | Shutdown"
+                print "1 INFO Passed:  %s" % self.countpass
+                print "2 INFO Failed:  %s" % self.countfail
+                print "3 INFO Todo:    %s" % self.counttodo
+                print "4 INFO SimpleTest FINISHED"
+
+            return result
+        finally:
+            self.shutdownNode()
 
     def doTests(self, options, onLaunch=None, testsToFilter=None):
         # A call to initializeLooping method is required in case of --run-by-dir or --bisect-chunk
         # since we need to initialize variables for each loop.
         if options.bisectChunk or options.runByDir:
             self.initializeLooping(options)
 
         # get debugger info, a dict of:
@@ -2258,26 +2331,24 @@ class MochitestDesktop(MochitestBase):
             return 1
 
         self.leak_report_file = os.path.join(
             options.profilePath,
             "runtests_leaks.log")
 
         self.browserEnv = self.buildBrowserEnv(
             options,
-            debuggerInfo is not None)
+            debuggerInfo is not None,
+            env=self.nodeEnv)
 
         # If there are any Mulet-specific tests doing remote network access,
         # we will not be aware since we are explicitely allowing this, as for
         # B2G
-        #
-        # In addition, the push subsuite directly accesses the production
-        # push service.
-        if 'MOZ_DISABLE_NONLOCAL_CONNECTIONS' in self.browserEnv:
-            if mozinfo.info.get('buildapp') == 'mulet' or options.subsuite == 'push':
+        if 'MOZ_DISABLE_NONLOCAL_CONNECTIONS' in self.browserEnv and\
+            mozinfo.info.get('buildapp') == 'mulet':
                 del self.browserEnv['MOZ_DISABLE_NONLOCAL_CONNECTIONS']
                 os.environ["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "0"
 
         if self.browserEnv is None:
             return 1
 
         try:
             self.startServers(options, debuggerInfo)