fix(ci): retain readable results from unsuccessful Windows runs

scripts/test_windows_acceptance.py

@@ -123,6 +123,37 @@ class WindowsAcceptanceTests(unittest.TestCase):
         self.write_trx(summary="Aborted")
         self.assertEqual("failed", self.evaluate()["automated_status"])
 
+    def test_failed_summary_and_bad_counters_retain_readable_evidence(self):
+        rows = self.rows + [("Example.Tests.Other.Failure", "Failed"),
+                            ("Example.Tests.Other.Optional", "NotExecuted")]
+        for mode in ("Failed", "Aborted", "bad-counter", "missing-summary"):
+            with self.subTest(mode=mode):
+                path = self.write_trx(rows, summary=mode if mode in ("Failed", "Aborted") else "Completed")
+                tree = ET.parse(path)
+                root = tree.getroot()
+                summary = root.find("{*}ResultSummary")
+                assert summary is not None
+                if mode == "bad-counter":
+                    counters = summary.find("{*}Counters")
+                    assert counters is not None
+                    counters.set("total", "999")
+                ET.SubElement(ET.SubElement(summary, "Output"), "StdOut").text = "runner failure detail"
+                if mode == "missing-summary":
+                    root.remove(summary)
+                tree.write(path, encoding="utf-8", xml_declaration=True)
+                report = self.evaluate()
+                self.assertEqual("failed", report["automated_status"])
+                self.assertIn("Example.Tests", report["suites"])
+                suite = report["suites"]["Example.Tests"]
+                self.assertEqual(4, suite["total"])
+                self.assertEqual({"Passed": 2, "Failed": 1, "NotExecuted": 1}, suite["outcomes"])
+                self.assertTrue(suite["validation_errors"])
+                self.assertEqual("fixture reason", suite["results"][2]["detail"])
+                self.assertEqual("fixture reason", suite["results"][3]["detail"])
+                if mode != "missing-summary":
+                    self.assertIn("runner failure detail", suite["summary_output"])
+                self.assertEqual("failed", report["groups"][0]["automated_status"])
+
     def test_duplicate_or_unmapped_execution_fails(self):
         for old, new in (('executionId="1"', 'executionId="0"'),
                          ('testId="1"', 'testId="not-defined"'), ('id="1"', 'id="0"')):

scripts/windows_acceptance.py

@@ -41,6 +41,25 @@ def read_suite(path):
             "detail": "\n".join(output.itertext()).strip() if output is not None else "",
         })
     summary = root.find("{*}ResultSummary")
+    validation_errors = []
+    try:
+        validate_summary(path, summary, results)
+    except (ValueError, KeyError) as exc:
+        validation_errors.append(str(exc))
+    output = summary.find("{*}Output") if summary is not None else None
+    return {
+        "file": path.name,
+        "sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
+        "total": len(results),
+        "outcomes": dict(sorted(Counter(r["outcome"] for r in results).items())),
+        "results": results,
+        "summary_outcome": summary.attrib.get("outcome") if summary is not None else None,
+        "summary_output": "\n".join(output.itertext()).strip() if output is not None else "",
+        "validation_errors": validation_errors,
+    }
+
+
+def validate_summary(path, summary, results):
     if summary is None or summary.attrib.get("outcome") not in ("Completed", "Passed"):
         raise ValueError(f"{path.name}: missing or unsuccessful run summary")
     counters = summary.find("{*}Counters")
@@ -64,13 +83,6 @@ def read_suite(path):
         raise ValueError(f"{path.name}: notExecuted counter does not match skip rows")
     if values.get("completed", 0) not in (0, values["executed"]):
         raise ValueError(f"{path.name}: completed counter does not match executed rows")
-    return {
-        "file": path.name,
-        "sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
-        "total": len(results),
-        "outcomes": dict(sorted(outcomes.items())),
-        "results": results,
-    }
 
 
 def evaluate(manifest, results_dir):
@@ -92,6 +104,7 @@ def evaluate(manifest, results_dir):
         except (OSError, ET.ParseError, ValueError, KeyError) as exc:
             errors.append(f"{project}: {exc}")
     for project, suite in suites.items():
+        errors.extend(f"{project}: {error}" for error in suite["validation_errors"])
         for result in suite["results"]:
             if result["outcome"] not in ("Passed", "NotExecuted"):
                 errors.append(f"{project}: {result['name']} => {result['outcome']}")
@@ -101,6 +114,9 @@ def evaluate(manifest, results_dir):
         if not requirements:
             raise ValueError(f"{group['id']}: empty required test list")
         failures, matched = [], []
+        required_projects = {r["project"] for r in requirements}
+        for project in sorted(required_projects):
+            failures.extend(suites.get(project, {}).get("validation_errors", []))
         seen = set()
         for requirement in requirements:
             project, method, minimum = (