diff --git a/scripts/test_windows_acceptance.py b/scripts/test_windows_acceptance.py index bd49b31..d287631 100644 --- a/scripts/test_windows_acceptance.py +++ b/scripts/test_windows_acceptance.py @@ -42,6 +42,7 @@ class WindowsAcceptanceTests(unittest.TestCase): ET.SubElement(ET.SubElement(result, "Output"), "StdOut").text = "fixture reason" result_summary = ET.SubElement(root, "ResultSummary", outcome=summary) ET.SubElement(result_summary, "Counters", total=str(len(rows) if total is None else total), + executed=str(sum(r[1] != "NotExecuted" for r in rows)), passed=str(sum(r[1] == "Passed" for r in rows)), failed=str(sum(r[1] == "Failed" for r in rows)), notExecuted="0") path = self.path / "Example.Tests.trx" @@ -98,6 +99,26 @@ class WindowsAcceptanceTests(unittest.TestCase): path.write_text(path.read_text().replace('passed="2"', 'passed="1"')) self.assertEqual("failed", self.evaluate()["automated_status"]) + def test_contradictory_or_incomplete_counters_fail(self): + changes = [("executed", "999"), ("executed", "-1"), ("executed", "0"), + ("executed", "bad"), ("executed", None), ("notExecuted", "1"), + ("completed", "999")] + changes += [(key, "1") for key in ("aborted", "error", "timeout", "inconclusive", + "passedButRunAborted", "notRunnable", "disconnected", "warning", + "inProgress", "pending", "unknownCounter")] + for key, value in changes: + with self.subTest(counter=key, value=value): + path = self.write_trx() + tree = ET.parse(path) + counters = tree.getroot().find(".//{*}Counters") + assert counters is not None + if value is None: + counters.attrib.pop(key) + else: + counters.set(key, value) + tree.write(path, encoding="utf-8", xml_declaration=True) + self.assertEqual("failed", self.evaluate()["automated_status"]) + def test_aborted_run_fails_even_with_passed_rows(self): self.write_trx(summary="Aborted") self.assertEqual("failed", self.evaluate()["automated_status"]) diff --git a/scripts/windows_acceptance.py b/scripts/windows_acceptance.py index 4b19bae..7f306f9 100644 --- a/scripts/windows_acceptance.py +++ b/scripts/windows_acceptance.py @@ -46,10 +46,24 @@ def read_suite(path): counters = summary.find("{*}Counters") if counters is None or not results or int(counters.attrib["total"]) != len(results): raise ValueError(f"{path.name}: empty results or total does not match enumerated results") + values = {key: int(value) for key, value in counters.attrib.items()} + for key, value in values.items(): + if value < 0 or value > len(results): + raise ValueError(f"{path.name}: impossible {key} counter") + if key not in ("total", "executed", "passed", "failed", "notExecuted", "completed") and value: + raise ValueError(f"{path.name}: nonzero failure/incomplete counter {key}") outcomes = Counter(r["outcome"] for r in results) - for key, outcome in (("passed", "Passed"), ("failed", "Failed")): - if int(counters.attrib[key]) != outcomes[outcome]: + expected = {"passed": outcomes["Passed"], "failed": outcomes["Failed"], + "executed": len(results) - outcomes["NotExecuted"]} + for key, value in expected.items(): + if values[key] != value: raise ValueError(f"{path.name}: {key} counter does not match result rows") + # xUnit can emit skip rows with notExecuted=0. No other contradiction + # gets that exception, and required skips still fail in evaluate(). + if values.get("notExecuted", 0) not in (0, outcomes["NotExecuted"]): + raise ValueError(f"{path.name}: notExecuted counter does not match skip rows") + if values.get("completed", 0) not in (0, values["executed"]): + raise ValueError(f"{path.name}: completed counter does not match executed rows") return { "file": path.name, "sha256": hashlib.sha256(path.read_bytes()).hexdigest(),