--- a/testing/web-platform/harness/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py
@@ -56,16 +56,18 @@ def do_delayed_imports():
 
 class MarionetteProtocol(Protocol):
     def __init__(self, executor, browser):
         do_delayed_imports()
 
         Protocol.__init__(self, executor, browser)
         self.marionette = None
         self.marionette_port = browser.marionette_port
+        self.timeout = None
+        self.runner_handle = None
 
     def setup(self, runner):
         """Connect to browser via Marionette."""
         Protocol.setup(self, runner)
 
         self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
         self.marionette = marionette.Marionette(host='localhost',
                                                 port=self.marionette_port,
@@ -118,32 +120,56 @@ class MarionetteProtocol(Protocol):
             self.marionette.current_window_handle
         except Exception:
             return False
         return True
 
     def after_connect(self):
         self.load_runner("http")
 
+    def set_timeout(self, timeout):
+        self.marionette.set_script_timeout(timeout * 1000)
+        self.timeout = timeout
+
     def load_runner(self, protocol):
         # Check if we previously had a test window open, and if we did make sure it's closed
         self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
         url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
         self.logger.debug("Loading %s" % url)
+        self.runner_handle = self.marionette.current_window_handle
         try:
             self.marionette.navigate(url)
         except Exception as e:
             self.logger.critical(
                 "Loading initial page %s failed. Ensure that the "
                 "there are no other programs bound to this port and "
                 "that your firewall rules or network setup does not "
                 "prevent access.\e%s" % (url, traceback.format_exc(e)))
         self.marionette.execute_script(
             "document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
 
+    def close_old_windows(self, protocol):
+        handles = self.marionette.window_handles
+        runner_handle = None
+        try:
+            handles.remove(self.runner_handle)
+            runner_handle = self.runner_handle
+        except ValueError:
+            # The runner window probably changed id but we can restore it
+            runner_handle = handles[0]
+            handles = handles[1:]
+
+        for handle in handles:
+            self.marionette.switch_to_window(handle)
+            self.marionette.close()
+
+        self.marionette.switch_to_window(runner_handle)
+        if runner_handle != self.runner_handle:
+            self.load_runner(protocol)
+
     def wait(self):
         socket_timeout = self.marionette.client.sock.gettimeout()
         if socket_timeout:
             self.marionette.set_script_timeout((socket_timeout / 2) * 1000)
 
         while True:
             try:
                 self.marionette.execute_async_script("")
@@ -285,36 +311,38 @@ class RemoteMarionetteProtocol(Protocol)
         """
         conn = httplib.HTTPConnection(self.server.host, self.server.port)
         conn.request("HEAD", self.server.base_path + "invalid")
         res = conn.getresponse()
         return res.status == 404
 
 
 class ExecuteAsyncScriptRun(object):
-    def __init__(self, logger, func, marionette, url, timeout):
+    def __init__(self, logger, func, protocol, url, timeout):
         self.logger = logger
         self.result = (None, None)
-        self.marionette = marionette
+        self.protocol = protocol
+        self.marionette = protocol.marionette
         self.func = func
         self.url = url
         self.timeout = timeout
         self.result_flag = threading.Event()
 
     def run(self):
         timeout = self.timeout
 
         try:
             if timeout is not None:
-                self.marionette.set_script_timeout((timeout + extra_timeout) * 1000)
+                if timeout + extra_timeout != self.protocol.timeout:
+                    self.protocol.set_timeout(timeout + extra_timeout)
             else:
                 # We just want it to never time out, really, but marionette doesn't
                 # make that possible. It also seems to time out immediately if the
                 # timeout is set too high. This works at least.
-                self.marionette.set_script_timeout(2**31 - 1)
+                self.protocol.set_timeout(2**28 - 1)
         except IOError:
             self.logger.error("Lost marionette connection before starting test")
             return Stop
 
         executor = threading.Thread(target = self._run)
         executor.start()
 
         if timeout is not None:
@@ -378,27 +406,28 @@ class MarionetteTestharnessExecutor(Test
             self.protocol.load_runner(new_environment["protocol"])
 
     def do_test(self, test):
         timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
                    else None)
 
         success, data = ExecuteAsyncScriptRun(self.logger,
                                               self.do_testharness,
-                                              self.protocol.marionette,
+                                              self.protocol,
                                               self.test_url(test),
                                               timeout).run()
         if success:
             return self.convert_result(test, data)
 
         return (test.result_cls(*data), [])
 
     def do_testharness(self, marionette, url, timeout):
         if self.close_after_done:
             marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
+            self.protocol.close_old_windows(self.protocol)
 
         if timeout is not None:
             timeout_ms = str(timeout * 1000)
         else:
             timeout_ms = "null"
 
         script = self.script % {"abs_url": url,
                                 "url": strip_server(url),
@@ -461,17 +490,17 @@ class MarionetteRefTestExecutor(RefTestE
         assert dpi is None
 
         timeout =  self.timeout_multiplier * test.timeout if self.debug_info is None else None
 
         test_url = self.test_url(test)
 
         return ExecuteAsyncScriptRun(self.logger,
                              self._screenshot,
-                             self.protocol.marionette,
+                             self.protocol,
                              test_url,
                              timeout).run()
 
     def _screenshot(self, marionette, url, timeout):
         marionette.navigate(url)
 
         marionette.execute_async_script(self.wait_script)