diff --git a/scripts/test_windows_acceptance.py b/scripts/test_windows_acceptance.py index d287631..c9d2db9 100644 --- a/scripts/test_windows_acceptance.py +++ b/scripts/test_windows_acceptance.py @@ -123,6 +123,37 @@ class WindowsAcceptanceTests(unittest.TestCase): self.write_trx(summary="Aborted") self.assertEqual("failed", self.evaluate()["automated_status"]) + def test_failed_summary_and_bad_counters_retain_readable_evidence(self): + rows = self.rows + [("Example.Tests.Other.Failure", "Failed"), + ("Example.Tests.Other.Optional", "NotExecuted")] + for mode in ("Failed", "Aborted", "bad-counter", "missing-summary"): + with self.subTest(mode=mode): + path = self.write_trx(rows, summary=mode if mode in ("Failed", "Aborted") else "Completed") + tree = ET.parse(path) + root = tree.getroot() + summary = root.find("{*}ResultSummary") + assert summary is not None + if mode == "bad-counter": + counters = summary.find("{*}Counters") + assert counters is not None + counters.set("total", "999") + ET.SubElement(ET.SubElement(summary, "Output"), "StdOut").text = "runner failure detail" + if mode == "missing-summary": + root.remove(summary) + tree.write(path, encoding="utf-8", xml_declaration=True) + report = self.evaluate() + self.assertEqual("failed", report["automated_status"]) + self.assertIn("Example.Tests", report["suites"]) + suite = report["suites"]["Example.Tests"] + self.assertEqual(4, suite["total"]) + self.assertEqual({"Passed": 2, "Failed": 1, "NotExecuted": 1}, suite["outcomes"]) + self.assertTrue(suite["validation_errors"]) + self.assertEqual("fixture reason", suite["results"][2]["detail"]) + self.assertEqual("fixture reason", suite["results"][3]["detail"]) + if mode != "missing-summary": + self.assertIn("runner failure detail", suite["summary_output"]) + self.assertEqual("failed", report["groups"][0]["automated_status"]) + def test_duplicate_or_unmapped_execution_fails(self): for old, new in (('executionId="1"', 'executionId="0"'), ('testId="1"', 'testId="not-defined"'), ('id="1"', 'id="0"')): diff --git a/scripts/windows_acceptance.py b/scripts/windows_acceptance.py index 7f306f9..d7943c2 100644 --- a/scripts/windows_acceptance.py +++ b/scripts/windows_acceptance.py @@ -41,6 +41,25 @@ def read_suite(path): "detail": "\n".join(output.itertext()).strip() if output is not None else "", }) summary = root.find("{*}ResultSummary") + validation_errors = [] + try: + validate_summary(path, summary, results) + except (ValueError, KeyError) as exc: + validation_errors.append(str(exc)) + output = summary.find("{*}Output") if summary is not None else None + return { + "file": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "total": len(results), + "outcomes": dict(sorted(Counter(r["outcome"] for r in results).items())), + "results": results, + "summary_outcome": summary.attrib.get("outcome") if summary is not None else None, + "summary_output": "\n".join(output.itertext()).strip() if output is not None else "", + "validation_errors": validation_errors, + } + + +def validate_summary(path, summary, results): if summary is None or summary.attrib.get("outcome") not in ("Completed", "Passed"): raise ValueError(f"{path.name}: missing or unsuccessful run summary") counters = summary.find("{*}Counters") @@ -64,13 +83,6 @@ def read_suite(path): raise ValueError(f"{path.name}: notExecuted counter does not match skip rows") if values.get("completed", 0) not in (0, values["executed"]): raise ValueError(f"{path.name}: completed counter does not match executed rows") - return { - "file": path.name, - "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), - "total": len(results), - "outcomes": dict(sorted(outcomes.items())), - "results": results, - } def evaluate(manifest, results_dir): @@ -92,6 +104,7 @@ def evaluate(manifest, results_dir): except (OSError, ET.ParseError, ValueError, KeyError) as exc: errors.append(f"{project}: {exc}") for project, suite in suites.items(): + errors.extend(f"{project}: {error}" for error in suite["validation_errors"]) for result in suite["results"]: if result["outcome"] not in ("Passed", "NotExecuted"): errors.append(f"{project}: {result['name']} => {result['outcome']}") @@ -101,6 +114,9 @@ def evaluate(manifest, results_dir): if not requirements: raise ValueError(f"{group['id']}: empty required test list") failures, matched = [], [] + required_projects = {r["project"] for r in requirements} + for project in sorted(required_projects): + failures.extend(suites.get(project, {}).get("validation_errors", [])) seen = set() for requirement in requirements: project, method, minimum = (