fix(ci): reject contradictory Windows test counters

scripts/test_windows_acceptance.py

@@ -42,6 +42,7 @@ class WindowsAcceptanceTests(unittest.TestCase):
                 ET.SubElement(ET.SubElement(result, "Output"), "StdOut").text = "fixture reason"
         result_summary = ET.SubElement(root, "ResultSummary", outcome=summary)
         ET.SubElement(result_summary, "Counters", total=str(len(rows) if total is None else total),
+                      executed=str(sum(r[1] != "NotExecuted" for r in rows)),
                       passed=str(sum(r[1] == "Passed" for r in rows)),
                       failed=str(sum(r[1] == "Failed" for r in rows)), notExecuted="0")
         path = self.path / "Example.Tests.trx"
@@ -98,6 +99,26 @@ class WindowsAcceptanceTests(unittest.TestCase):
         path.write_text(path.read_text().replace('passed="2"', 'passed="1"'))
         self.assertEqual("failed", self.evaluate()["automated_status"])
 
+    def test_contradictory_or_incomplete_counters_fail(self):
+        changes = [("executed", "999"), ("executed", "-1"), ("executed", "0"),
+                   ("executed", "bad"), ("executed", None), ("notExecuted", "1"),
+                   ("completed", "999")]
+        changes += [(key, "1") for key in ("aborted", "error", "timeout", "inconclusive",
+                    "passedButRunAborted", "notRunnable", "disconnected", "warning",
+                    "inProgress", "pending", "unknownCounter")]
+        for key, value in changes:
+            with self.subTest(counter=key, value=value):
+                path = self.write_trx()
+                tree = ET.parse(path)
+                counters = tree.getroot().find(".//{*}Counters")
+                assert counters is not None
+                if value is None:
+                    counters.attrib.pop(key)
+                else:
+                    counters.set(key, value)
+                tree.write(path, encoding="utf-8", xml_declaration=True)
+                self.assertEqual("failed", self.evaluate()["automated_status"])
+
     def test_aborted_run_fails_even_with_passed_rows(self):
         self.write_trx(summary="Aborted")
         self.assertEqual("failed", self.evaluate()["automated_status"])

scripts/windows_acceptance.py

@@ -46,10 +46,24 @@ def read_suite(path):
     counters = summary.find("{*}Counters")
     if counters is None or not results or int(counters.attrib["total"]) != len(results):
         raise ValueError(f"{path.name}: empty results or total does not match enumerated results")
+    values = {key: int(value) for key, value in counters.attrib.items()}
+    for key, value in values.items():
+        if value < 0 or value > len(results):
+            raise ValueError(f"{path.name}: impossible {key} counter")
+        if key not in ("total", "executed", "passed", "failed", "notExecuted", "completed") and value:
+            raise ValueError(f"{path.name}: nonzero failure/incomplete counter {key}")
     outcomes = Counter(r["outcome"] for r in results)
-    for key, outcome in (("passed", "Passed"), ("failed", "Failed")):
-        if int(counters.attrib[key]) != outcomes[outcome]:
+    expected = {"passed": outcomes["Passed"], "failed": outcomes["Failed"],
+                "executed": len(results) - outcomes["NotExecuted"]}
+    for key, value in expected.items():
+        if values[key] != value:
             raise ValueError(f"{path.name}: {key} counter does not match result rows")
+    # xUnit can emit skip rows with notExecuted=0. No other contradiction
+    # gets that exception, and required skips still fail in evaluate().
+    if values.get("notExecuted", 0) not in (0, outcomes["NotExecuted"]):
+        raise ValueError(f"{path.name}: notExecuted counter does not match skip rows")
+    if values.get("completed", 0) not in (0, values["executed"]):
+        raise ValueError(f"{path.name}: completed counter does not match executed rows")
     return {
         "file": path.name,
         "sha256": hashlib.sha256(path.read_bytes()).hexdigest(),