--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/base.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/base.py
@@ -78,18 +78,21 @@ class Browser(object):
def __exit__(self, *args, **kwargs):
self.cleanup()
def setup(self):
"""Used for browser-specific setup that happens at the start of a test run"""
pass
+ def settings(self, test):
+ return {}
+
@abstractmethod
- def start(self):
+ def start(self, **kwargs):
"""Launch the browser object and get it into a state where is is ready to run tests"""
pass
@abstractmethod
def stop(self, force=False):
"""Stop the running browser process."""
pass
@@ -121,17 +124,17 @@ class Browser(object):
in the browser, or an empty list if no crashes occurred"""
self.logger.crash(process, test)
class NullBrowser(Browser):
def __init__(self, logger, **kwargs):
super(NullBrowser, self).__init__(logger)
- def start(self):
+ def start(self, **kwargs):
"""No-op browser to use in scenarios where the TestRunnerManager shouldn't
actually own the browser process (e.g. Servo where we start one browser
per test)"""
pass
def stop(self, force=False):
pass
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py
@@ -15,17 +15,17 @@ from ..executors.executorselenium import
"env_extras": "env_extras",
"env_options": "env_options"}
def check_args(**kwargs):
require_arg(kwargs, "webdriver_binary")
-def browser_kwargs(**kwargs):
+def browser_kwargs(test_type, run_info_data, **kwargs):
return {"binary": kwargs["binary"],
"webdriver_binary": kwargs["webdriver_binary"],
"webdriver_args": kwargs.get("webdriver_args")}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
from selenium.webdriver import DesiredCapabilities
@@ -67,17 +67,17 @@ class ChromeBrowser(Browser):
"""Creates a new representation of Chrome. The `binary` argument gives
the browser binary to use for testing."""
Browser.__init__(self, logger)
self.binary = binary
self.server = ChromeDriverServer(self.logger,
binary=webdriver_binary,
args=webdriver_args)
- def start(self):
+ def start(self, **kwargs):
self.server.start(block=False)
def stop(self, force=False):
self.server.stop(force=force)
def pid(self):
return self.server.pid
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/edge.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/edge.py
@@ -46,17 +46,17 @@ class EdgeBrowser(Browser):
def __init__(self, logger, webdriver_binary, webdriver_args=None):
Browser.__init__(self, logger)
self.server = EdgeDriverServer(self.logger,
binary=webdriver_binary,
args=webdriver_args)
self.webdriver_host = "localhost"
self.webdriver_port = self.server.port
- def start(self):
+ def start(self, **kwargs):
print self.server.url
self.server.start()
def stop(self):
self.server.stop()
def pid(self):
return self.server.pid
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -1,15 +1,16 @@
import os
import platform
import signal
import subprocess
import sys
import mozinfo
+import mozleak
from mozprocess import ProcessHandler
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import get_stack_fixer_function
from mozcrash import mozcrash
from .base import (get_free_port,
@@ -36,49 +37,61 @@ here = os.path.join(os.path.split(__file
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_extras": "env_extras",
"env_options": "env_options",
"run_info_extras": "run_info_extras",
"update_properties": "update_properties"}
+def get_timeout_multiplier(test_type, run_info_data, **kwargs):
+ if kwargs["timeout_multiplier"] is not None:
+ return kwargs["timeout_multiplier"]
+ if test_type == "reftest":
+ if run_info_data["debug"] or run_info_data.get("asan"):
+ return 4
+ else:
+ return 2
+ elif run_info_data["debug"] or run_info_data.get("asan"):
+ return 3
+ return 1
+
+
def check_args(**kwargs):
require_arg(kwargs, "binary")
if kwargs["ssl_type"] != "none":
require_arg(kwargs, "certutil_binary")
-def browser_kwargs(**kwargs):
+def browser_kwargs(test_type, run_info_data, **kwargs):
return {"binary": kwargs["binary"],
"prefs_root": kwargs["prefs_root"],
"extra_prefs": kwargs["extra_prefs"],
"debug_info": kwargs["debug_info"],
"symbols_path": kwargs["symbols_path"],
"stackwalk_binary": kwargs["stackwalk_binary"],
"certutil_binary": kwargs["certutil_binary"],
"ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
"e10s": kwargs["gecko_e10s"],
"stackfix_dir": kwargs["stackfix_dir"],
- "binary_args": kwargs["binary_args"]}
+ "binary_args": kwargs["binary_args"],
+ "timeout_multiplier": get_timeout_multiplier(test_type,
+ run_info_data,
+ **kwargs),
+ "leak_check": kwargs["leak_check"]}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
executor_kwargs = base_executor_kwargs(test_type, server_config,
cache_manager, **kwargs)
executor_kwargs["close_after_done"] = test_type != "reftest"
- if kwargs["timeout_multiplier"] is None:
- if test_type == "reftest":
- if run_info_data["debug"] or run_info_data.get("asan"):
- executor_kwargs["timeout_multiplier"] = 4
- else:
- executor_kwargs["timeout_multiplier"] = 2
- elif run_info_data["debug"] or run_info_data.get("asan"):
- executor_kwargs["timeout_multiplier"] = 3
+ executor_kwargs["timeout_multiplier"] = get_timeout_multiplier(test_type,
+ run_info_data,
+ **kwargs)
if test_type == "wdspec":
executor_kwargs["binary"] = kwargs["binary"]
executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
executor_kwargs["webdriver_args"] = kwargs.get("webdriver_args")
fxOptions = {}
if kwargs["binary"]:
fxOptions["binary"] = kwargs["binary"]
if kwargs["binary_args"]:
@@ -114,17 +127,17 @@ def update_properties():
class FirefoxBrowser(Browser):
used_ports = set()
init_timeout = 60
shutdown_timeout = 60
def __init__(self, logger, binary, prefs_root, extra_prefs=None, debug_info=None,
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
ca_certificate_path=None, e10s=False, stackfix_dir=None,
- binary_args=None):
+ binary_args=None, timeout_multiplier=None, leak_check=False):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.extra_prefs = extra_prefs
self.marionette_port = None
self.runner = None
self.debug_info = debug_info
self.profile = None
@@ -135,38 +148,55 @@ class FirefoxBrowser(Browser):
self.e10s = e10s
self.binary_args = binary_args
if self.symbols_path and stackfix_dir:
self.stack_fixer = get_stack_fixer_function(stackfix_dir,
self.symbols_path)
else:
self.stack_fixer = None
- def start(self):
- self.marionette_port = get_free_port(2828, exclude=self.used_ports)
- self.used_ports.add(self.marionette_port)
+ if timeout_multiplier:
+ self.init_timeout = self.init_timeout * timeout_multiplier
+
+ self.leak_report_file = None
+ self.leak_check = leak_check
+
+ def settings(self, test):
+ return {"check_leaks": self.leak_check and not test.leaks}
+
+ def start(self, **kwargs):
+ if self.marionette_port is None:
+ self.marionette_port = get_free_port(2828, exclude=self.used_ports)
+ self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
- self.profile.set_preferences({"marionette.enabled": True,
- "marionette.port": self.marionette_port,
+ self.profile.set_preferences({"marionette.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(hostnames),
"network.proxy.type": 0,
"places.history.enabled": False})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
+ if self.leak_check and kwargs.get("check_leaks", True):
+ self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
+ if os.path.exists(self.leak_report_file):
+ os.remove(self.leak_report_file)
+ env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
+ else:
+ self.leak_report_file = None
+
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft") and
'5.1' in platform.version()):
self.profile.set_preferences({"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
@@ -212,16 +242,34 @@ class FirefoxBrowser(Browser):
if not force or not clean:
retcode = stop_f()
if retcode is not None:
self.logger.info("Browser exited with return code %s" % retcode)
break
except OSError:
# This can happen on Windows if the process is already dead
pass
+ self.logger.debug("stopped")
+
+ def process_leaks(self):
+ self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file)
+ if self.leak_report_file is None:
+ return
+ mozleak.process_leak_log(
+ self.leak_report_file,
+ leak_thresholds={
+ "default": 0,
+ "tab": 10000, # See dependencies of bug 1051230.
+ # GMP rarely gets a log, but when it does, it leaks a little.
+ "geckomediaplugin": 20000,
+ },
+ ignore_missing_leaks=["geckomediaplugin"],
+ log=self.logger,
+ stack_fixer=self.stack_fixer
+ )
def pid(self):
if self.runner.process_handler is None:
return None
try:
return self.runner.process_handler.pid
except AttributeError:
@@ -238,16 +286,17 @@ class FirefoxBrowser(Browser):
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self):
self.stop()
+ self.process_leaks()
def executor_browser(self):
assert self.marionette_port is not None
return ExecutorBrowser, {"marionette_port": self.marionette_port}
def log_crash(self, process, test):
dump_dir = os.path.join(self.profile.profile, "minidumps")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/sauce.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/sauce.py
@@ -87,17 +87,17 @@ def get_sauce_config(**kwargs):
def check_args(**kwargs):
require_arg(kwargs, "sauce_browser")
require_arg(kwargs, "sauce_platform")
require_arg(kwargs, "sauce_version")
require_arg(kwargs, "sauce_user")
require_arg(kwargs, "sauce_key")
-def browser_kwargs(**kwargs):
+def browser_kwargs(test_type, run_info_data, **kwargs):
sauce_config = get_sauce_config(**kwargs)
return {"sauce_config": sauce_config}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
executor_kwargs = base_executor_kwargs(test_type, server_config,
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py
@@ -22,17 +22,17 @@ here = os.path.join(os.path.split(__file
"update_properties": "update_properties",
}
def check_args(**kwargs):
require_arg(kwargs, "binary")
-def browser_kwargs(**kwargs):
+def browser_kwargs(test_type, run_info_data, **kwargs):
return {
"binary": kwargs["binary"],
"debug_info": kwargs["debug_info"],
"binary_args": kwargs["binary_args"],
"user_stylesheets": kwargs.get("user_stylesheets"),
"ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
}
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py
@@ -34,17 +34,17 @@ 127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-pl
127.0.0.1 xn--lve-6lad.web-platform.test
"""
def check_args(**kwargs):
require_arg(kwargs, "binary")
-def browser_kwargs(**kwargs):
+def browser_kwargs(test_type, run_info_data, **kwargs):
return {
"binary": kwargs["binary"],
"debug_info": kwargs["debug_info"],
"user_stylesheets": kwargs.get("user_stylesheets"),
}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data, **kwargs):
@@ -86,17 +86,17 @@ class ServoWebDriverBrowser(Browser):
self.webdriver_host = webdriver_host
self.webdriver_port = None
self.proc = None
self.debug_info = debug_info
self.hosts_path = make_hosts_file()
self.command = None
self.user_stylesheets = user_stylesheets if user_stylesheets else []
- def start(self):
+ def start(self, **kwargs):
self.webdriver_port = get_free_port(4444, exclude=self.used_ports)
self.used_ports.add(self.webdriver_port)
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
debug_args, command = browser_command(
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
@@ -43,38 +43,41 @@ def do_delayed_imports():
try:
import marionette
from marionette import errors
except ImportError:
from marionette_driver import marionette, errors
class MarionetteProtocol(Protocol):
- def __init__(self, executor, browser):
+ def __init__(self, executor, browser, timeout_multiplier=1):
do_delayed_imports()
Protocol.__init__(self, executor, browser)
self.marionette = None
self.marionette_port = browser.marionette_port
+ self.timeout_multiplier = timeout_multiplier
self.timeout = None
self.runner_handle = None
def setup(self, runner):
"""Connect to browser via Marionette."""
Protocol.setup(self, runner)
self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
+ startup_timeout = marionette.Marionette.DEFAULT_STARTUP_TIMEOUT * self.timeout_multiplier
self.marionette = marionette.Marionette(host='localhost',
port=self.marionette_port,
- socket_timeout=None)
+ socket_timeout=None,
+ startup_timeout=startup_timeout)
# XXX Move this timeout somewhere
self.logger.debug("Waiting for Marionette connection")
while True:
- success = self.marionette.wait_for_port(60)
+ success = self.marionette.wait_for_port(60 * self.timeout_multiplier)
#When running in a debugger wait indefinitely for firefox to start
if success or self.executor.debug_info is None:
break
session_started = False
if success:
try:
self.logger.debug("Starting Marionette session")
@@ -402,17 +405,17 @@ class ExecuteAsyncScriptRun(object):
class MarionetteTestharnessExecutor(TestharnessExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
close_after_done=True, debug_info=None, **kwargs):
"""Marionette-based executor for testharness.js tests"""
TestharnessExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
- self.protocol = MarionetteProtocol(self, browser)
+ self.protocol = MarionetteProtocol(self, browser, timeout_multiplier)
self.script = open(os.path.join(here, "testharness_marionette.js")).read()
self.close_after_done = close_after_done
self.window_id = str(uuid.uuid4())
self.original_pref_values = {}
if marionette is None:
do_delayed_imports()
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
@@ -107,16 +107,20 @@ class ExpectedManifest(ManifestItem):
def disabled(self):
return bool_prop("disabled", self)
@property
def restart_after(self):
return bool_prop("restart-after", self)
@property
+ def leaks(self):
+ return bool_prop("leaks", self)
+
+ @property
def tags(self):
return tags(self)
@property
def prefs(self):
return prefs(self)
@@ -125,16 +129,20 @@ class DirectoryManifest(ManifestItem):
def disabled(self):
return bool_prop("disabled", self)
@property
def restart_after(self):
return bool_prop("restart-after", self)
@property
+ def leaks(self):
+ return bool_prop("leaks", self)
+
+ @property
def tags(self):
return tags(self)
@property
def prefs(self):
return prefs(self)
@@ -170,16 +178,20 @@ class TestNode(ManifestItem):
def disabled(self):
return bool_prop("disabled", self)
@property
def restart_after(self):
return bool_prop("restart-after", self)
@property
+ def leaks(self):
+ return bool_prop("leaks", self)
+
+ @property
def tags(self):
return tags(self)
@property
def prefs(self):
return prefs(self)
def append(self, node):
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py
@@ -87,18 +87,18 @@ class EqualTimeChunker(TestChunker):
for i, (test_type, test_path, tests) in enumerate(manifest_items):
test_dir = tuple(os.path.split(test_path)[0].split(os.path.sep)[:3])
if not test_dir in by_dir:
by_dir[test_dir] = PathData(test_dir)
data = by_dir[test_dir]
- time = sum(wpttest.DEFAULT_TIMEOUT if test.timeout !=
- "long" else wpttest.LONG_TIMEOUT for test in tests)
+ time = sum(test.default_timeout if test.timeout !=
+ "long" else test.long_timeout for test in tests)
data.time += time
total_time += time
data.tests.append((test_type, test_path, tests))
return by_dir, total_time
def _maybe_remove(self, chunks, i, direction):
"""Trial removing a chunk from one chunk to an adjacent one.
@@ -622,17 +622,16 @@ class PathGroupedSource(TestSource):
if not self.current_queue or self.current_queue.empty():
try:
data = self.test_queue.get(block=True, timeout=1)
self.current_queue = Queue()
for item in data:
self.current_queue.put(item)
except Empty:
return None
-
return self.current_queue
def requeue_test(self, test):
self.current_queue.put(test)
def __exit__(self, *args, **kwargs):
if self.current_queue:
self.current_queue.close()
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
@@ -148,21 +148,30 @@ def next_manager_number():
class BrowserManager(object):
init_lock = threading.Lock()
def __init__(self, logger, browser, command_queue, no_timeout=False):
self.logger = logger
self.browser = browser
self.no_timeout = no_timeout
+ self.browser_settings = None
self.started = False
self.init_timer = None
+ def update_settings(self, test):
+ browser_settings = self.browser.settings(test)
+ restart_required = ((self.browser_settings is not None and
+ self.browser_settings != browser_settings) or
+ test.expected() == "CRASH")
+ self.browser_settings = browser_settings
+ return restart_required
+
def init(self):
"""Launch the browser that is being tested,
and the TestRunner process that will run the tests."""
# It seems that this lock is helpful to prevent some race that otherwise
# sometimes stops the spawned processes initalising correctly, and
# leaves this thread hung
if self.init_timer is not None:
self.init_timer.cancel()
@@ -173,17 +182,18 @@ class BrowserManager(object):
# Guard against problems initialising the browser or the browser
# remote control method
if not self.no_timeout:
self.init_timer = threading.Timer(self.browser.init_timeout,
self.init_timeout)
try:
if self.init_timer is not None:
self.init_timer.start()
- self.browser.start()
+ self.logger.debug("Starting browser with settings %r" % self.browser_settings)
+ self.browser.start(**self.browser_settings)
self.browser_pid = self.browser.pid()
except:
self.logger.warning("Failure during init %s" % traceback.format_exc())
if self.init_timer is not None:
self.init_timer.cancel()
self.logger.error(traceback.format_exc())
succeeded = False
else:
@@ -432,16 +442,18 @@ class TestRunnerManager(threading.Thread
return RunnerManagerState.initalizing(test, test_queue, 0)
def init(self):
assert isinstance(self.state, RunnerManagerState.initalizing)
if self.state.failure_count > self.max_restarts:
self.logger.error("Max restarts exceeded")
return RunnerManagerState.error()
+ self.browser.update_settings(self.state.test)
+
result = self.browser.init()
if result is Stop:
return RunnerManagerState.error()
elif not result:
return RunnerManagerState.initalizing(self.state.test,
self.state.test_queue,
self.state.failure_count + 1)
else:
@@ -490,25 +502,31 @@ class TestRunnerManager(threading.Thread
while test is None:
if test_queue is None:
test_queue = self.test_source.get_queue()
if test_queue is None:
self.logger.info("No more tests")
return None, None
try:
# Need to block here just to allow for contention with other processes
- test = test_queue.get(block=True, timeout=1)
+ test = test_queue.get(block=True, timeout=2)
except Empty:
- pass
+ if test_queue.empty():
+ test_queue = None
return test, test_queue
def run_test(self):
assert isinstance(self.state, RunnerManagerState.running)
assert self.state.test is not None
+ if self.browser.update_settings(self.state.test):
+ self.logger.info("Restarting browser for new test environment")
+ return RunnerManagerState.restarting(self.state.test,
+ self.state.test_queue)
+
self.logger.test_start(self.state.test.id)
self.send_message("run_test", self.state.test)
def test_ended(self, test, results):
"""Handle the end of a test.
Output the result of each subtest, and the result of the overall
harness to the logs.
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_chunker.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_chunker.py
@@ -7,16 +7,18 @@ import pytest
sys.path.insert(0, join(dirname(__file__), "..", ".."))
from wptrunner.testloader import EqualTimeChunker
structured.set_default_logger(structured.structuredlog.StructuredLogger("TestChunker"))
class MockTest(object):
+ default_timeout = 10
+
def __init__(self, id, timeout=10):
self.id = id
self.item_type = "testharness"
self.timeout = timeout
def make_mock_manifest(*items):
rv = []
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/sync.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/sync.py
@@ -1,9 +1,11 @@
+import fnmatch
import os
+import re
import shutil
import sys
import uuid
from .. import testloader
from base import Step, StepRunner
from tree import Commit
@@ -37,28 +39,43 @@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NO
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
"""
-def copy_wpt_tree(tree, dest):
+def copy_wpt_tree(tree, dest, excludes=None, includes=None):
"""Copy the working copy of a Tree to a destination directory.
:param tree: The Tree to copy.
:param dest: The destination directory"""
if os.path.exists(dest):
assert os.path.isdir(dest)
shutil.rmtree(dest)
+
os.mkdir(dest)
+ if excludes is None:
+ excludes = []
+
+ excludes = [re.compile(fnmatch.translate(item)) for item in excludes]
+
+ if includes is None:
+ includes = []
+
+ includes = [re.compile(fnmatch.translate(item)) for item in includes]
+
for tree_path in tree.paths():
+ if (any(item.match(tree_path) for item in excludes) and
+ not any(item.match(tree_path) for item in includes)):
+ continue
+
source_path = os.path.join(tree.root, tree_path)
dest_path = os.path.join(dest, tree_path)
dest_dir = os.path.split(dest_path)[0]
if not os.path.isdir(source_path):
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copy2(source_path, dest_path)
@@ -74,16 +91,17 @@ def copy_wpt_tree(tree, dest):
def add_license(dest):
"""Write the bsd license string to a LICENSE file.
:param dest: Directory in which to place the LICENSE file."""
with open(os.path.join(dest, "LICENSE"), "w") as f:
f.write(bsd_license)
+
class UpdateCheckout(Step):
"""Pull changes from upstream into the local sync tree."""
provides = ["local_branch"]
def create(self, state):
sync_tree = state.sync_tree
state.local_branch = uuid.uuid4().hex
@@ -137,17 +155,19 @@ class UpdateManifest(Step):
manifest.write(state.test_manifest, state.manifest_path)
class CopyWorkTree(Step):
"""Copy the sync tree over to the destination in the local tree"""
def create(self, state):
copy_wpt_tree(state.sync_tree,
- state.tests_path)
+ state.tests_path,
+ excludes=state.path_excludes,
+ includes=state.path_includes)
class CreateSyncPatch(Step):
"""Add the updated test files to a commit/patch in the local tree."""
def create(self, state):
if state.no_patch:
return
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/tree.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/tree.py
@@ -307,19 +307,18 @@ class GitTree(object):
"""List paths in the tree"""
repo_paths = [self.root] + [os.path.join(self.root, path)
for path in self.submodules()]
rv = []
for repo_path in repo_paths:
paths = vcs.git("ls-tree", "-r", "--name-only", "HEAD", repo=repo_path).split("\n")
- rel_path = os.path.relpath(repo_path, self.root)
- rv.extend(os.path.join(rel_path, item.strip()) for item in paths if item.strip())
-
+ rv.extend(os.path.relpath(os.path.join(repo_path, item), self.root) for item in paths
+ if item.strip())
return rv
def submodules(self):
"""List submodule directories"""
output = self.git("submodule", "status", "--recursive")
rv = []
for line in output.split("\n"):
line = line.strip()
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py
@@ -68,16 +68,18 @@ class SyncFromUpstream(Step):
state.sync_tree = GitTree(root=state.sync["path"])
kwargs = state.kwargs
with state.push(["sync", "paths", "metadata_path", "tests_path", "local_tree",
"sync_tree"]):
state.target_rev = kwargs["rev"]
state.no_patch = kwargs["no_patch"]
state.suite_name = kwargs["suite_name"]
+ state.path_excludes = kwargs["exclude"]
+ state.path_includes = kwargs["include"]
runner = SyncFromUpstreamRunner(self.logger, state)
runner.run()
class UpdateMetadata(Step):
"""Update the expectation metadata from a set of run logs"""
def create(self, state):
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -172,16 +172,18 @@ scheme host and port.""")
help="Path to the folder containing browser prefs")
gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True,
help="Run tests without electrolysis preferences")
gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store",
help="Path to directory containing assertion stack fixing scripts")
gecko_group.add_argument("--setpref", dest="extra_prefs", action='append',
default=[], metavar="PREF=VALUE",
help="Defines an extra user preference (overrides those in prefs_root)")
+ gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true",
+ help="Enable leak checking")
servo_group = parser.add_argument_group("Servo-specific")
servo_group.add_argument("--user-stylesheet",
default=[], action="append", dest="user_stylesheets",
help="Inject a user CSS stylesheet into every test.")
sauce_group = parser.add_argument_group("Sauce Labs-specific")
sauce_group.add_argument("--sauce-browser", dest="sauce_browser",
@@ -419,16 +421,20 @@ def create_parser_update(product_choices
parser.add_argument("--rev", action="store", help="Revision to sync to")
parser.add_argument("--no-patch", action="store_true",
help="Don't create an mq patch or git commit containing the changes.")
parser.add_argument("--sync", dest="sync", action="store_true", default=False,
help="Sync the tests with the latest from upstream")
parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.")
parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script")
parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script")
+ parser.add_argument("--exclude", action="store", nargs="*",
+ help="List of glob-style paths to exclude when syncing tests")
+ parser.add_argument("--include", action="store", nargs="*",
+ help="List of glob-style paths to include which would otherwise be excluded when syncing tests")
# Should make this required iff run=logfile
parser.add_argument("run_log", nargs="*", type=abs_path,
help="Log file from run of tests")
commandline.add_logging_group(parser)
return parser
def create_parser_reduce(product_choices=None):
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -153,18 +153,16 @@ def run_tests(config, test_paths, produc
env_options,
env_extras) as test_environment:
try:
test_environment.ensure_started()
except env.TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" % e.message)
raise
- browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
-
repeat = kwargs["repeat"]
repeat_count = 0
repeat_until_unexpected = kwargs["repeat_until_unexpected"]
while repeat_count < repeat or repeat_until_unexpected:
repeat_count += 1
if repeat_until_unexpected:
logger.info("Repetition %i" % (repeat_count))
@@ -181,32 +179,37 @@ def run_tests(config, test_paths, produc
# processes are managed by a WebDriver server binary. This
# obviates the need for wptrunner to provide a browser, so
# the NullBrowser is used in place of the "target" browser
if test_type == "wdspec":
browser_cls = NullBrowser
else:
browser_cls = target_browser_cls
- for test in test_loader.disabled_tests[test_type]:
- logger.test_start(test.id)
- logger.test_end(test.id, status="SKIP")
+ browser_kwargs = get_browser_kwargs(test_type,
+ run_info,
+ ssl_env=ssl_env,
+ **kwargs)
+
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(test_type,
test_environment.external_config,
test_environment.cache_manager,
run_info,
**kwargs)
if executor_cls is None:
logger.error("Unsupported test type %s for product %s" %
(test_type, product))
continue
+ for test in test_loader.disabled_tests[test_type]:
+ logger.test_start(test.id)
+ logger.test_end(test.id, status="SKIP")
with ManagerGroup("web-platform-tests",
kwargs["processes"],
test_source_cls,
test_source_kwargs,
browser_cls,
browser_kwargs,
executor_cls,
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py
@@ -1,11 +1,8 @@
-DEFAULT_TIMEOUT = 10 # seconds
-LONG_TIMEOUT = 60 # seconds
-
import os
import mozinfo
from wptmanifest.parser import atoms
atom_reset = atoms["Reset"]
enabled_tests = set(["testharness", "reftest", "wdspec"])
@@ -90,36 +87,40 @@ class RunInfo(dict):
break
dirs.add(str(path))
path = os.path.split(path)[0]
mozinfo.find_and_update_from_json(*dirs)
class Test(object):
+
result_cls = None
subtest_result_cls = None
test_type = None
+ default_timeout = 10 # seconds
+ long_timeout = 60 # seconds
+
def __init__(self, tests_root, url, inherit_metadata, test_metadata,
- timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
+ timeout=None, path=None, protocol="http"):
self.tests_root = tests_root
self.url = url
self._inherit_metadata = inherit_metadata
self._test_metadata = test_metadata
- self.timeout = timeout
+ self.timeout = timeout if timeout is not None else self.default_timeout
self.path = path
self.environment = {"protocol": protocol, "prefs": self.prefs}
def __eq__(self, other):
return self.id == other.id
@classmethod
def from_manifest(cls, manifest_item, inherit_metadata, test_metadata):
- timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT
+ timeout = cls.long_timeout if manifest_item.timeout == "long" else cls.default_timeout
protocol = "https" if hasattr(manifest_item, "https") and manifest_item.https else "http"
return cls(manifest_item.source_file.tests_root,
manifest_item.url,
inherit_metadata,
test_metadata,
timeout=timeout,
path=manifest_item.source_file.path,
protocol=protocol)
@@ -164,16 +165,24 @@ class Test(object):
def restart_after(self):
for meta in self.itermeta(None):
restart_after = meta.restart_after
if restart_after is not None:
return True
return False
@property
+ def leaks(self):
+ for meta in self.itermeta(None):
+ leaks = meta.leaks
+ if leaks is not None:
+ return leaks
+ return False
+
+ @property
def tags(self):
tags = set()
for meta in self.itermeta():
meta_tags = meta.tags
if atom_reset in meta_tags:
tags = meta_tags.copy()
tags.remove(atom_reset)
else:
@@ -232,18 +241,17 @@ class ManualTest(Test):
return self.url
class ReftestTest(Test):
result_cls = ReftestResult
test_type = "reftest"
def __init__(self, tests_root, url, inherit_metadata, test_metadata, references,
- timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None,
- dpi=None, protocol="http"):
+ timeout=None, path=None, viewport_size=None, dpi=None, protocol="http"):
Test.__init__(self, tests_root, url, inherit_metadata, test_metadata, timeout,
path, protocol)
for _, ref_type in references:
if ref_type not in ("==", "!="):
raise ValueError
self.references = references
@@ -253,17 +261,17 @@ class ReftestTest(Test):
@classmethod
def from_manifest(cls,
manifest_test,
inherit_metadata,
test_metadata,
nodes=None,
references_seen=None):
- timeout = LONG_TIMEOUT if manifest_test.timeout == "long" else DEFAULT_TIMEOUT
+ timeout = cls.long_timeout if manifest_test.timeout == "long" else cls.default_timeout
if nodes is None:
nodes = {}
if references_seen is None:
references_seen = set()
url = manifest_test.url
@@ -317,20 +325,24 @@ class ReftestTest(Test):
return self.url
@property
def keys(self):
return ("reftype", "refurl")
class WdspecTest(Test):
+
result_cls = WdspecResult
subtest_result_cls = WdspecSubtestResult
test_type = "wdspec"
+ default_timeout = 25
+ long_timeout = 120
+
manifest_test_cls = {"reftest": ReftestTest,
"testharness": TestharnessTest,
"manual": ManualTest,
"wdspec": WdspecTest}
def from_manifest(manifest_test, inherit_metadata, test_metadata):