--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
@@ -13,18 +13,16 @@ import wpttest
from expected import expected_path
from vcs import git
manifest = None # Module that will be imported relative to test_root
manifestitem = None
logger = structuredlog.StructuredLogger("web-platform-tests")
-TestItem = namedtuple("TestItem", ["test_manifest", "expected"])
-
try:
import ujson as json
except ImportError:
import json
def load_test_manifests(serve_root, test_paths):
do_delayed_imports(serve_root)
@@ -39,43 +37,31 @@ def update_expected(test_paths, serve_ro
"""Update the metadata files for web-platform-tests based on
the results obtained in a previous run or runs
If stability is not None, assume log_file_names refers to logs from repeated
test jobs, disable tests that don't behave as expected on all runs"""
manifests = load_test_manifests(serve_root, test_paths)
- id_test_map = update_from_logs(manifests,
- *log_file_names,
- ignore_existing=ignore_existing,
- property_order=property_order,
- boolean_properties=boolean_properties,
- stability=stability)
-
- by_test_manifest = defaultdict(list)
- while id_test_map:
- item = id_test_map.popitem()[1]
- by_test_manifest[item.test_manifest].append(item.expected)
+ for metadata_path, updated_ini in update_from_logs(manifests,
+ *log_file_names,
+ ignore_existing=ignore_existing,
+ property_order=property_order,
+ boolean_properties=boolean_properties,
+ stability=stability):
- for test_manifest, expected in by_test_manifest.iteritems():
- metadata_path = manifests[test_manifest]["metadata_path"]
- write_changes(metadata_path, expected)
- if stability is not None:
- for tree in expected:
- if not tree.modified:
- continue
- for test in expected.iterchildren():
- for subtest in test.iterchildren():
- if subtest.new_disabled:
- print "disabled: %s" % os.path.dirname(subtest.root.test_path) + "/" + subtest.name
- if test.new_disabled:
- print "disabled: %s" % test.root.test_path
-
- return by_test_manifest
+ write_new_expected(metadata_path, updated_ini)
+ if stability:
+ for test in updated_ini.iterchildren():
+ for subtest in test.iterchildren():
+ if subtest.new_disabled:
+ print "disabled: %s" % os.path.dirname(subtest.root.test_path) + "/" + subtest.name
+ if test.new_disabled:
+ print "disabled: %s" % test.root.test_path
def do_delayed_imports(serve_root):
global manifest, manifestitem
from manifest import manifest, item as manifestitem
def files_in_repo(repo_root):
@@ -142,40 +128,34 @@ def update_from_logs(manifests, *log_fil
boolean_properties = kwargs.get("boolean_properties")
stability = kwargs.get("stability")
id_test_map = {}
for test_manifest, paths in manifests.iteritems():
id_test_map.update(create_test_tree(
paths["metadata_path"],
- test_manifest,
- property_order=property_order,
- boolean_properties=boolean_properties))
+ test_manifest))
updater = ExpectedUpdater(manifests,
id_test_map,
ignore_existing=ignore_existing)
for log_filename in log_filenames:
with open(log_filename) as f:
updater.update_from_log(f)
- return coalesce_results(id_test_map, stability)
+ for item in update_results(id_test_map, property_order, boolean_properties, stability):
+ yield item
-def coalesce_results(id_test_map, stability):
- for _, expected in id_test_map.itervalues():
- if not expected.modified:
- continue
- expected.coalesce_properties(stability=stability)
- for test in expected.iterchildren():
- for subtest in test.iterchildren():
- subtest.coalesce_properties(stability=stability)
- test.coalesce_properties(stability=stability)
-
- return id_test_map
+def update_results(id_test_map, property_order, boolean_properties, stability):
+ test_file_items = set(id_test_map.itervalues())
+ for test_file in test_file_items:
+ updated_expected = test_file.update(property_order, boolean_properties, stability)
+ if updated_expected is not None and updated_expected.modified:
+ yield test_file.metadata_path, updated_expected
def directory_manifests(metadata_path):
rv = []
for dirpath, dirname, filenames in os.walk(metadata_path):
if "__dir__.ini" in filenames:
rel_path = os.path.relpath(dirpath, metadata_path)
rv.append(os.path.join(rel_path, "__dir__.ini"))
@@ -204,219 +184,288 @@ def write_changes(metadata_path, expecte
os.rename(metadata_path, temp_path_2)
# Move the new files to the destination location and remove the old files
os.rename(temp_path, metadata_path)
shutil.rmtree(temp_path_2)
def write_new_expected(metadata_path, expected):
# Serialize the data back to a file
- for tree in expected:
- if not tree.is_empty:
- manifest_str = wptmanifest.serialize(tree.node, skip_empty_data=True)
- assert manifest_str != ""
- path = expected_path(metadata_path, tree.test_path)
- dir = os.path.split(path)[0]
- if not os.path.exists(dir):
- os.makedirs(dir)
- with open(path, "wb") as f:
+ path = expected_path(metadata_path, expected.test_path)
+ if not expected.is_empty:
+ manifest_str = wptmanifest.serialize(expected.node, skip_empty_data=True)
+ assert manifest_str != ""
+ dir = os.path.split(path)[0]
+ if not os.path.exists(dir):
+ os.makedirs(dir)
+ tmp_path = path + ".tmp"
+ try:
+ with open(tmp_path, "wb") as f:
f.write(manifest_str)
+ os.rename(tmp_path, path)
+ except (Exception, KeyboardInterrupt):
+ try:
+ os.unlink(tmp_path)
+ except OSError:
+ pass
+ else:
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
class ExpectedUpdater(object):
def __init__(self, test_manifests, id_test_map, ignore_existing=False):
self.id_test_map = id_test_map
self.ignore_existing = ignore_existing
self.run_info = None
self.action_map = {"suite_start": self.suite_start,
"test_start": self.test_start,
"test_status": self.test_status,
"test_end": self.test_end,
"assertion_count": self.assertion_count,
"lsan_leak": self.lsan_leak}
self.tests_visited = {}
- self.test_cache = {}
-
self.types_by_path = {}
for manifest in test_manifests.iterkeys():
for test_type, path, _ in manifest:
if test_type in wpttest.manifest_test_cls:
self.types_by_path[path] = wpttest.manifest_test_cls[test_type]
+ self.run_infos = []
def update_from_log(self, log_file):
self.run_info = None
action_map = self.action_map
for line in log_file:
data = json.loads(line)
action = data["action"]
if action in action_map:
action_map[action](data)
def suite_start(self, data):
self.run_info = data["run_info"]
def test_start(self, data):
test_id = data["test"]
try:
- expected_node = self.id_test_map[test_id].expected.get_test(test_id)
+ test_data = self.id_test_map[test_id]
except KeyError:
print "Test not found %s, skipping" % test_id
return
- self.test_cache[test_id] = expected_node
- if test_id not in self.tests_visited:
- if self.ignore_existing:
- expected_node.clear("expected")
- self.tests_visited[test_id] = set()
+ if self.ignore_existing:
+ test_data.set_requires_update()
+ test_data.clear.append("expected")
+ self.tests_visited[test_id] = set()
def test_status(self, data):
test_id = data["test"]
- test = self.test_cache.get(test_id)
- if test is None:
+ subtest = data["subtest"]
+ test_data = self.id_test_map.get(test_id)
+ if test_data is None:
return
- test_cls = self.types_by_path[test.root.test_path]
- subtest = test.get_subtest(data["subtest"])
+ test_cls = self.types_by_path[test_data.test_path]
- self.tests_visited[test_id].add(data["subtest"])
+ self.tests_visited[test_id].add(subtest)
result = test_cls.subtest_result_cls(
- data["subtest"],
+ subtest,
data["status"],
None)
- subtest.set_result(self.run_info, result)
+ test_data.set(test_id, subtest, "status", self.run_info, result)
+ if data.get("expected") and data["expected"] != data["status"]:
+ test_data.set_requires_update()
def test_end(self, data):
- test_id = data["test"]
- test = self.test_cache.get(test_id)
- if test is None:
- return
- test_cls = self.types_by_path[test.root.test_path]
-
if data["status"] == "SKIP":
return
+ test_id = data["test"]
+ test_data = self.id_test_map.get(test_id)
+ if test_data is None:
+ return
+ test_cls = self.types_by_path[test_data.test_path]
+
+
result = test_cls.result_cls(
data["status"],
None)
- test.set_result(self.run_info, result)
- del self.test_cache[test_id]
+ test_data.set(test_id, None, "status", self.run_info, result)
+ if data.get("expected") and data["status"] != data["expected"]:
+ test_data.set_requires_update()
+ del self.tests_visited[test_id]
def assertion_count(self, data):
test_id = data["test"]
- test = self.test_cache.get(test_id)
- if test is None:
+ test_data = self.id_test_map.get(test_id)
+ if test_data is None:
return
- test.set_asserts(self.run_info, data["count"])
+ test_data.set(test_id, None, "asserts", self.run_info, data["count"])
+ if data["count"] < data["min_expected"] or data["count"] > data["max_expected"]:
+ test_data.set_requires_update()
def lsan_leak(self, data):
dir_path = data.get("scope", "/")
dir_id = os.path.join(dir_path, "__dir__").replace(os.path.sep, "/")
if dir_id.startswith("/"):
dir_id = dir_id[1:]
- expected_node = self.id_test_map[dir_id].expected
-
- expected_node.set_lsan(self.run_info, (data["frames"], data.get("allowed_match")))
+ test_data = self.id_test_map[dir_id]
+ test_data.set(dir_id, None, "lsan", self.run_info, (data["frames"], data.get("allowed_match")))
+ if not data.get("allowed_match"):
+ test_data.set_requires_update()
-def create_test_tree(metadata_path, test_manifest, property_order=None,
- boolean_properties=None):
+def create_test_tree(metadata_path, test_manifest):
"""Create a map of expectation manifests for all tests in test_manifest,
reading existing manifests under manifest_path
:returns: A map of test_id to (manifest, test, expectation_data)
"""
id_test_map = {}
exclude_types = frozenset(["stub", "helper", "manual", "support", "conformancechecker"])
all_types = [item.item_type for item in manifestitem.__dict__.itervalues()
if type(item) == type and
issubclass(item, manifestitem.ManifestItem) and
item.item_type is not None]
include_types = set(all_types) - exclude_types
for _, test_path, tests in test_manifest.itertypes(*include_types):
- expected_data = load_or_create_expected(test_manifest, metadata_path, test_path, tests,
- property_order, boolean_properties)
+ test_file_data = TestFileData(test_manifest,
+ metadata_path,
+ test_path,
+ tests)
for test in tests:
- id_test_map[test.id] = TestItem(test_manifest, expected_data)
+ id_test_map[test.id] = test_file_data
dir_path = os.path.split(test_path)[0].replace(os.path.sep, "/")
while True:
if dir_path:
dir_id = dir_path + "/__dir__"
else:
dir_id = "__dir__"
dir_id = (test_manifest.url_base + dir_id).lstrip("/")
if dir_id not in id_test_map:
dir_object = DirObject(dir_id, dir_path)
- expected_data = load_or_create_expected(test_manifest,
- metadata_path,
- dir_id,
- [],
- property_order,
- boolean_properties)
-
- id_test_map[dir_id] = TestItem(test_manifest, expected_data)
- if not dir_path:
+ test_file_data = TestFileData(test_manifest,
+ metadata_path,
+ dir_id,
+ [])
+ id_test_map[dir_id] = test_file_data
+ if not dir_path or dir_path in id_test_map:
break
dir_path = dir_path.rsplit("/", 1)[0] if "/" in dir_path else ""
return id_test_map
class DirObject(object):
def __init__(self, id, path):
self.id = id
self.path = path
def __hash__(self):
return hash(self.id)
-def load_or_create_expected(test_manifest, metadata_path, test_path, tests, property_order=None,
- boolean_properties=None):
- expected_data = load_expected(test_manifest, metadata_path, test_path, tests,
- property_order=property_order,
- boolean_properties=boolean_properties)
- if expected_data is None:
- expected_data = create_expected(test_manifest,
- test_path,
- tests,
- property_order=property_order,
- boolean_properties=boolean_properties)
- return expected_data
+class TestFileData(object):
+ def __init__(self, test_manifest, metadata_path, test_path, tests):
+ self.test_manifest = test_manifest
+ self.test_path = test_path
+ self.metadata_path = metadata_path
+ self.tests = tests
+ self._expected = None
+ self._requires_update = False
+ self.clear = set()
+ self.data = []
+
+ def set_requires_update(self):
+ self._requires_update = True
+
+ def set(self, test_id, subtest_id, prop, run_info, value):
+ self.data.append((test_id, subtest_id, prop, run_info, value))
+
+ def expected(self, property_order, boolean_properties):
+ if self._expected is None:
+ expected_data = load_expected(self.test_manifest,
+ self.metadata_path,
+ self.test_path,
+ self.tests,
+ property_order,
+ boolean_properties)
+ if expected_data is None:
+ expected_data = create_expected(self.test_manifest,
+ self.test_path,
+ property_order,
+ boolean_properties)
+ self._expected = expected_data
+ return self._expected
+
+ def update(self, property_order, boolean_properties, stability):
+ if not self._requires_update:
+ return
+
+ expected = self.expected(property_order, boolean_properties)
+ expected_by_test = {}
+
+ for test in self.tests:
+ if not expected.has_test(test.id):
+ expected.append(manifestupdate.TestNode.create(test.id))
+ test_expected = expected.get_test(test.id)
+ expected_by_test[test.id] = test_expected
+ for prop in self.clear:
+ test_expected.clear(prop)
+
+ for (test_id, subtest_id, prop, run_info, value) in self.data:
+ # Special case directory metadata
+ if subtest_id is None and test_id.endswith("__dir__"):
+ if prop == "lsan":
+ expected.set_lsan(run_info, value)
+ continue
+
+ test_expected = expected_by_test[test_id]
+ if subtest_id is None:
+ item_expected = test_expected
+ else:
+ item_expected = test_expected.get_subtest(subtest_id)
+ if prop == "status":
+ item_expected.set_result(run_info, value)
+ elif prop == "asserts":
+ item_expected.set_asserts(run_info, value)
+
+ expected.coalesce_properties(stability=stability)
+ for test in expected.iterchildren():
+ for subtest in test.iterchildren():
+ subtest.coalesce_properties(stability=stability)
+ test.coalesce_properties(stability=stability)
+
+ return expected
-def create_expected(test_manifest, test_path, tests, property_order=None,
+def create_expected(test_manifest, test_path, property_order=None,
boolean_properties=None):
expected = manifestupdate.ExpectedManifest(None, test_path, test_manifest.url_base,
property_order=property_order,
boolean_properties=boolean_properties)
- for test in tests:
- expected.append(manifestupdate.TestNode.create(test.id))
return expected
def load_expected(test_manifest, metadata_path, test_path, tests, property_order=None,
boolean_properties=None):
expected_manifest = manifestupdate.get_manifest(metadata_path,
test_path,
test_manifest.url_base,
property_order=property_order,
boolean_properties=boolean_properties)
if expected_manifest is None:
return
- tests_by_id = {item.id: item for item in tests}
+ tests_by_id = {item.id for item in tests}
# Remove expected data for tests that no longer exist
for test in expected_manifest.iterchildren():
if test.id not in tests_by_id:
test.remove()
- # Add tests that don't have expected data
- for test in tests:
- if not expected_manifest.has_test(test.id):
- expected_manifest.append(manifestupdate.TestNode.create(test.id))
-
return expected_manifest
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_update.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_update.py
@@ -33,39 +33,46 @@ item_classes = {"testharness": manifest_
"stub": manifest_item.Stub,
"wdspec": manifest_item.WebdriverSpecTest,
"conformancechecker": manifest_item.ConformanceCheckerTest,
"visual": manifest_item.VisualTest,
"support": manifest_item.SupportFile}
def update(tests, *logs):
- updater = create_updater(tests)
+ id_test_map, updater = create_updater(tests)
for log in logs:
log = create_log(log)
updater.update_from_log(log)
- return metadata.coalesce_results(updater.id_test_map, False)
+ return list(metadata.update_results(id_test_map,
+ ["debug", "os", "version", "processor", "bits"],
+ ["debug"],
+ False))
def create_updater(tests, url_base="/", **kwargs):
id_test_map = {}
m = create_test_manifest(tests, url_base)
test_manifests = {
m: {"url_base": "/",
"tests_path": "."}
}
for test_path, test_ids, test_type, manifest_str in tests:
+ tests = list(m.iterpath(test_path))
if isinstance(test_ids, (str, unicode)):
test_ids = [test_ids]
- expected = manifestupdate.compile(BytesIO(manifest_str), test_path, url_base)
+ test_data = metadata.TestFileData(m, None, test_path, tests)
+ test_data._expected = manifestupdate.compile(BytesIO(manifest_str),
+ test_path,
+ url_base)
for test_id in test_ids:
- id_test_map[test_id] = metadata.TestItem(m, expected)
+ id_test_map[test_id] = test_data
- return metadata.ExpectedUpdater(test_manifests, id_test_map, **kwargs)
+ return id_test_map, metadata.ExpectedUpdater(test_manifests, id_test_map, **kwargs)
def create_log(entries):
logger = structuredlog.StructuredLogger("expected_test")
data = BytesIO()
handler = handlers.StreamHandler(data, formatters.JSONFormatter())
logger.add_handler(handler)
@@ -89,84 +96,103 @@ def create_test_manifest(tests, url_base
if test_type:
source_files.append(SourceFileWithTest(test, str(i) * 40, item_classes[test_type]))
m = manifest.Manifest()
m.update(source_files)
return m
def test_update_0():
- tests = [("path/to/test.htm.ini", ["/path/to/test.htm"], "testharness",
+ tests = [("path/to/test.htm", ["/path/to/test.htm"], "testharness",
"""[test.htm]
[test1]
expected: FAIL""")]
log = suite_log([("test_start", {"test": "/path/to/test.htm"}),
("test_status", {"test": "/path/to/test.htm",
"subtest": "test1",
"status": "PASS",
"expected": "FAIL"}),
("test_end", {"test": "/path/to/test.htm",
"status": "OK"})])
- id_test_map = update(tests, log)
+ updated = update(tests, log)
- assert len(id_test_map) == 1
- assert id_test_map.popitem()[1].expected.is_empty
+ assert len(updated) == 1
+ assert updated[0][1].is_empty
def test_update_1():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness",
+ tests = [("path/to/test.htm", [test_id], "testharness",
"""[test.htm]
[test1]
expected: ERROR""")]
log = suite_log([("test_start", {"test": test_id}),
("test_status", {"test": test_id,
"subtest": "test1",
"status": "FAIL",
"expected": "ERROR"}),
("test_end", {"test": test_id,
"status": "OK"})])
- id_test_map = update(tests, log)
+ updated = update(tests, log)
- new_manifest = id_test_map.popitem()[1].expected
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).children[0].get("expected") == "FAIL"
+def test_skip_0():
+ test_id = "/path/to/test.htm"
+ tests = [("path/to/test.htm", [test_id], "testharness",
+ """[test.htm]
+ [test1]
+ expected: FAIL""")]
+
+ log = suite_log([("test_start", {"test": test_id}),
+ ("test_status", {"test": test_id,
+ "subtest": "test1",
+ "status": "FAIL",
+ "expected": "FAIL"}),
+ ("test_end", {"test": test_id,
+ "status": "OK"})])
+
+ updated = update(tests, log)
+ assert not updated
+
+
def test_new_subtest():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
[test1]
expected: FAIL""")]
log = suite_log([("test_start", {"test": test_id}),
("test_status", {"test": test_id,
"subtest": "test1",
"status": "FAIL",
"expected": "FAIL"}),
("test_status", {"test": test_id,
"subtest": "test2",
"status": "FAIL",
"expected": "PASS"}),
("test_end", {"test": test_id,
"status": "OK"})])
- id_test_map = update(tests, log)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).children[0].get("expected") == "FAIL"
assert new_manifest.get_test(test_id).children[1].get("expected") == "FAIL"
def test_update_multiple_0():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
[test1]
expected: FAIL""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("test_status", {"test": test_id,
"subtest": "test1",
"status": "FAIL",
"expected": "FAIL"}),
@@ -178,29 +204,29 @@ def test_update_multiple_0():
("test_status", {"test": test_id,
"subtest": "test1",
"status": "TIMEOUT",
"expected": "FAIL"}),
("test_end", {"test": test_id,
"status": "OK"})],
run_info={"debug": False, "os": "linux"})
- id_test_map = update(tests, log_0, log_1)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0, log_1)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": False, "os": "osx"}) == "FAIL"
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": False, "os": "linux"}) == "TIMEOUT"
def test_update_multiple_1():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
[test1]
expected: FAIL""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("test_status", {"test": test_id,
"subtest": "test1",
"status": "FAIL",
"expected": "FAIL"}),
@@ -212,31 +238,31 @@ def test_update_multiple_1():
("test_status", {"test": test_id,
"subtest": "test1",
"status": "TIMEOUT",
"expected": "FAIL"}),
("test_end", {"test": test_id,
"status": "OK"})],
run_info={"debug": False, "os": "linux"})
- id_test_map = update(tests, log_0, log_1)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0, log_1)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": False, "os": "osx"}) == "FAIL"
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": False, "os": "linux"}) == "TIMEOUT"
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": False, "os": "windows"}) == "FAIL"
def test_update_multiple_2():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
[test1]
expected: FAIL""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("test_status", {"test": test_id,
"subtest": "test1",
"status": "FAIL",
"expected": "FAIL"}),
@@ -248,29 +274,29 @@ def test_update_multiple_2():
("test_status", {"test": test_id,
"subtest": "test1",
"status": "TIMEOUT",
"expected": "FAIL"}),
("test_end", {"test": test_id,
"status": "OK"})],
run_info={"debug": True, "os": "osx"})
- id_test_map = update(tests, log_0, log_1)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0, log_1)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": False, "os": "osx"}) == "FAIL"
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": True, "os": "osx"}) == "TIMEOUT"
def test_update_multiple_3():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
[test1]
expected:
if debug: FAIL
if not debug and os == "osx": TIMEOUT""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("test_status", {"test": test_id,
"subtest": "test1",
@@ -284,29 +310,29 @@ def test_update_multiple_3():
("test_status", {"test": test_id,
"subtest": "test1",
"status": "TIMEOUT",
"expected": "FAIL"}),
("test_end", {"test": test_id,
"status": "OK"})],
run_info={"debug": True, "os": "osx"})
- id_test_map = update(tests, log_0, log_1)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0, log_1)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": False, "os": "osx"}) == "FAIL"
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": True, "os": "osx"}) == "TIMEOUT"
def test_update_ignore_existing():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
[test1]
expected:
if debug: TIMEOUT
if not debug and os == "osx": NOTRUN""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("test_status", {"test": test_id,
"subtest": "test1",
@@ -320,98 +346,94 @@ def test_update_ignore_existing():
("test_status", {"test": test_id,
"subtest": "test1",
"status": "FAIL",
"expected": "PASS"}),
("test_end", {"test": test_id,
"status": "OK"})],
run_info={"debug": True, "os": "windows"})
- id_test_map = update(tests, log_0, log_1)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0, log_1)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": True, "os": "osx"}) == "FAIL"
assert new_manifest.get_test(test_id).children[0].get(
"expected", {"debug": False, "os": "osx"}) == "NOTRUN"
def test_update_assertion_count_0():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
max-asserts: 4
min-asserts: 2
""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("assertion_count", {"test": test_id,
"count": 6,
"min_expected": 2,
"max_expected": 4}),
("test_end", {"test": test_id,
"status": "OK"})])
- id_test_map = update(tests, log_0)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).get("max-asserts") == 7
assert new_manifest.get_test(test_id).get("min-asserts") == 2
def test_update_assertion_count_1():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
max-asserts: 4
min-asserts: 2
""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("assertion_count", {"test": test_id,
"count": 1,
"min_expected": 2,
"max_expected": 4}),
("test_end", {"test": test_id,
"status": "OK"})])
- id_test_map = update(tests, log_0)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).get("max-asserts") == 4
assert new_manifest.get_test(test_id).has_key("min-asserts") is False
def test_update_assertion_count_2():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
max-asserts: 4
min-asserts: 2
""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("assertion_count", {"test": test_id,
"count": 3,
"min_expected": 2,
"max_expected": 4}),
("test_end", {"test": test_id,
"status": "OK"})])
- id_test_map = update(tests, log_0)
- new_manifest = id_test_map.popitem()[1].expected
-
- assert not new_manifest.is_empty
- assert new_manifest.get_test(test_id).get("max-asserts") == 4
- assert new_manifest.get_test(test_id).get("min-asserts") == 2
+ updated = update(tests, log_0)
+ assert not updated
def test_update_assertion_count_3():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
max-asserts: 4
min-asserts: 2
""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("assertion_count", {"test": test_id,
"count": 6,
"min_expected": 2,
@@ -424,27 +446,27 @@ def test_update_assertion_count_3():
("assertion_count", {"test": test_id,
"count": 7,
"min_expected": 2,
"max_expected": 4}),
("test_end", {"test": test_id,
"status": "OK"})],
run_info={"os": "linux"})
- id_test_map = update(tests, log_0, log_1)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0, log_1)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).get("max-asserts") == 8
assert new_manifest.get_test(test_id).get("min-asserts") == 2
def test_update_assertion_count_4():
test_id = "/path/to/test.htm"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", """[test.htm]""")]
+ tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]""")]
log_0 = suite_log([("test_start", {"test": test_id}),
("assertion_count", {"test": test_id,
"count": 6,
"min_expected": 0,
"max_expected": 0}),
("test_end", {"test": test_id,
"status": "OK"})],
@@ -454,95 +476,95 @@ def test_update_assertion_count_4():
("assertion_count", {"test": test_id,
"count": 7,
"min_expected": 0,
"max_expected": 0}),
("test_end", {"test": test_id,
"status": "OK"})],
run_info={"os": "linux"})
- id_test_map = update(tests, log_0, log_1)
- new_manifest = id_test_map.popitem()[1].expected
+ updated = update(tests, log_0, log_1)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get_test(test_id).get("max-asserts") == "8"
assert new_manifest.get_test(test_id).has_key("min-asserts") is False
def test_update_lsan_0():
test_id = "/path/to/test.htm"
dir_id = "path/to/__dir__"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", ""),
- ("path/to/__dir__.ini", [dir_id], None, "")]
+ tests = [("path/to/test.htm", [test_id], "testharness", ""),
+ ("path/to/__dir__", [dir_id], None, "")]
log_0 = suite_log([("lsan_leak", {"scope": "path/to/",
"frames": ["foo", "bar"]})])
- id_test_map = update(tests, log_0)
- new_manifest = id_test_map[dir_id].expected
+ updated = update(tests, log_0)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get("lsan-allowed") == ["foo"]
def test_update_lsan_1():
test_id = "/path/to/test.htm"
dir_id = "path/to/__dir__"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", ""),
- ("path/to/__dir__.ini", [dir_id], None, """
+ tests = [("path/to/test.htm", [test_id], "testharness", ""),
+ ("path/to/__dir__", [dir_id], None, """
lsan-allowed: [foo]""")]
log_0 = suite_log([("lsan_leak", {"scope": "path/to/",
"frames": ["foo", "bar"]}),
("lsan_leak", {"scope": "path/to/",
"frames": ["baz", "foobar"]})])
- id_test_map = update(tests, log_0)
- new_manifest = id_test_map[dir_id].expected
+ updated = update(tests, log_0)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get("lsan-allowed") == ["baz", "foo"]
def test_update_lsan_2():
test_id = "/path/to/test.htm"
dir_id = "path/to/__dir__"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", ""),
- ("path/__dir__.ini", ["path/__dir__"], None, """
+ tests = [("path/to/test.htm", [test_id], "testharness", ""),
+ ("path/__dir__", ["path/__dir__"], None, """
lsan-allowed: [foo]"""),
- ("path/to/__dir__.ini", [dir_id], None, "")]
+ ("path/to/__dir__", [dir_id], None, "")]
log_0 = suite_log([("lsan_leak", {"scope": "path/to/",
"frames": ["foo", "bar"],
"allowed_match": ["foo"]}),
("lsan_leak", {"scope": "path/to/",
"frames": ["baz", "foobar"]})])
- id_test_map = update(tests, log_0)
- new_manifest = id_test_map[dir_id].expected
+ updated = update(tests, log_0)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get("lsan-allowed") == ["baz"]
def test_update_lsan_3():
test_id = "/path/to/test.htm"
dir_id = "path/to/__dir__"
- tests = [("path/to/test.htm.ini", [test_id], "testharness", ""),
- ("path/to/__dir__.ini", [dir_id], None, "")]
+ tests = [("path/to/test.htm", [test_id], "testharness", ""),
+ ("path/to/__dir__", [dir_id], None, "")]
log_0 = suite_log([("lsan_leak", {"scope": "path/to/",
"frames": ["foo", "bar"]})],
run_info={"os": "win"})
log_1 = suite_log([("lsan_leak", {"scope": "path/to/",
"frames": ["baz", "foobar"]})],
run_info={"os": "linux"})
- id_test_map = update(tests, log_0, log_1)
- new_manifest = id_test_map[dir_id].expected
+ updated = update(tests, log_0, log_1)
+ new_manifest = updated[0][1]
assert not new_manifest.is_empty
assert new_manifest.get("lsan-allowed") == ["baz", "foo"]