From b591528aa26ac460fede64ef01f1fd19aa3a4d3a Mon Sep 17 00:00:00 2001 From: AJ Isaacs Date: Tue, 6 Oct 2026 15:56:27 -0400 Subject: [PATCH] ci(windows): retain fail-closed desktop acceptance evidence --- .github/workflows/ci.yml | 5 + docs/windows-acceptance.md | 80 +++ scripts/test_windows_acceptance.py | 146 +++++ scripts/windows-acceptance.json | 986 +++++++++++++++++++++++++++++ scripts/windows_acceptance.py | 174 +++++ 5 files changed, 1391 insertions(+) create mode 100644 docs/windows-acceptance.md create mode 100644 scripts/test_windows_acceptance.py create mode 100644 scripts/windows-acceptance.json create mode 100644 scripts/windows_acceptance.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adac11a..a651365 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,11 @@ jobs: if ($LASTEXITCODE -ne 0) { $failed += $project } } if ($failed.Count -gt 0) { throw "Failed: $($failed -join ', ')" } + - name: Test acceptance evidence checker + run: python scripts/test_windows_acceptance.py -v + - name: Verify named Windows acceptance coverage + if: always() + run: python scripts/windows_acceptance.py --results TestResults - name: Upload test results if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/docs/windows-acceptance.md b/docs/windows-acceptance.md new file mode 100644 index 0000000..7379406 --- /dev/null +++ b/docs/windows-acceptance.md @@ -0,0 +1,80 @@ +# Automated Windows acceptance + +The `windows-desktop` job in `.github/workflows/ci.yml` runs on GitHub-hosted +`windows-2022` for every pull request and `master` push. It executes the full +WinForms and FrontEnd test projects in Release, including shown-form STA tests +and Windows child-process lifetime tests. No production service, customer +fixture, physical GPU, or CNC controller is required. + +Each test host has a five-minute hang watchdog with a mini dump; the job has a +20-minute limit. A failure in either suite fails the step, without suppressing +the other suite. This job is separate from the existing Linux `tests` aggregate; +require `windows-desktop` as well when assessing acceptance. + +## Results and coverage contract + +Download the `windows-desktop-test-results` artifact from the workflow run. +It contains both TRX files, any hang diagnostics, and: + +- `windows-acceptance.json`: commit/tree, runner platform, workflow identifiers, + TRX hashes, enumerated results, failures/skips and per-area test evidence. +- `windows-acceptance.md`: a readable summary, also shown in the Actions job + summary, with remaining acceptance beside the automated result. + +Artifacts are retained for 14 days. Save evidence needed for longer-lived task +verification before expiration. On pull requests the checked-out commit can be +GitHub's test merge commit; use the report's commit/tree and workflow link, not +just the branch name. + +`scripts/windows-acceptance.json` is the committed coverage contract. It names +required test methods and the minimum number of executed cases for each +(including theory rows). The report fails if either TRX is absent/malformed, +empty or inconsistent; if a required method/case is missing, failed or skipped; +if any other test fails; or if a run was aborted. Skips outside the named contract +are retained explicitly, never counted as passes. Added cases are allowed; +removing or renaming required coverage requires an intentional manifest change. +The whole suites still run, not just this manifest's subset. + +The areas are BOM import, Nest Info material preservation, saved-nest browsing, +nesting/commit policy, PDF export, the cutting dialog, automatic cutoffs, +overlap/posting consent, UI lifetime, import/export failures, and MCP processes. +The manifest is public test coverage only. Private tracker IDs, plans, customer +data, and acceptance notes stay out of the repository and CI artifacts. + +The report is evidence, not automatic task closure. A passing area does not +establish absent implementation, real-service/two-PC behavior, DPI or physical +printing quality, packaged-build behavior, or machine safety. In particular, +fake-repository browser tests do not verify the HTTP/SQLite save/paging lifecycle. +Keep those remaining gates in the tracker; link exact test names and the workflow +run when updating an individual task. + +## Repeat locally on Windows + +Use a clean, committed checkout, .NET 8 and Python 3. Run from the repository root +in PowerShell. Always use a new results directory so stale TRX files cannot stand +in for a failed build. Do not reuse downloaded CI results as a new local run. + +```powershell +$results = Join-Path 'TestResults' ([guid]::NewGuid().ToString()) +$failed = @() +foreach ($project in @('OpenNest.WinForms.Tests', 'OpenNest.FrontEnd.Tests')) { + dotnet test "$project/$project.csproj" -c Release --blame-hang-timeout 5m --blame-hang-dump-type mini --logger "trx;LogFileName=$project.trx" --results-directory $results + if ($LASTEXITCODE -ne 0) { $failed += $project } +} +python scripts/windows_acceptance.py --results $results +if ($LASTEXITCODE -ne 0 -or $failed.Count -gt 0) { throw 'Windows acceptance failed; inspect results.' } +``` + +The CLI refuses Linux/macOS as Windows runtime evidence and refuses tracked +source edits. The source revision is provenance of the checkout running the +command, not a cryptographic attestation of arbitrary supplied TRX files; the +fresh-checkout CI run and the fresh-directory local procedure establish that +relationship. The JSON retains hashes so the saved files can be matched later. + +## Maintain the gate + +Run `python scripts/test_windows_acceptance.py -v` on any platform after changing +the parser or manifest. Its generated TRX fixtures test the reporter only, not +the application. Exercise a draft PR on Windows before integrating new form +cases. Inspect actual TRX failures and skip reasons; a Linux cross-build is not +a Windows pass. Keep visual/operator limitations alongside the named cases. diff --git a/scripts/test_windows_acceptance.py b/scripts/test_windows_acceptance.py new file mode 100644 index 0000000..bd49b31 --- /dev/null +++ b/scripts/test_windows_acceptance.py @@ -0,0 +1,146 @@ +"""Synthetic parser controls, not Windows application acceptance evidence.""" + +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import xml.etree.ElementTree as ET + +import windows_acceptance as acceptance + + +class WindowsAcceptanceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.path = Path(self.temp.name) + self.manifest = { + "schema_version": 1, + "projects": ["Example.Tests"], + "groups": [{ + "id": "example", "title": "Example", + "required_tests": [{"project": "Example.Tests", "method": "Example.Tests.Form.Accept", "minimum_cases": 2}], + "remaining_acceptance": ["Visual/operator check"], + }], + } + self.rows = [("Example.Tests.Form.Accept", "Passed"), ("Example.Tests.Form.Accept", "Passed")] + + def write_trx(self, rows=None, total=None, summary="Completed"): + rows = self.rows if rows is None else rows + root = ET.Element("TestRun", xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010") + definitions = ET.SubElement(root, "TestDefinitions") + results = ET.SubElement(root, "Results") + for index, (method, outcome) in enumerate(rows): + test = ET.SubElement(definitions, "UnitTest", id=str(index)) + class_name, name = method.rsplit(".", 1) + ET.SubElement(test, "TestMethod", className=class_name, name=name) + result = ET.SubElement(results, "UnitTestResult", testId=str(index), executionId=str(index), + testName=f"{method}(case: {index})", outcome=outcome) + if outcome != "Passed": + ET.SubElement(ET.SubElement(result, "Output"), "StdOut").text = "fixture reason" + result_summary = ET.SubElement(root, "ResultSummary", outcome=summary) + ET.SubElement(result_summary, "Counters", total=str(len(rows) if total is None else total), + passed=str(sum(r[1] == "Passed" for r in rows)), + failed=str(sum(r[1] == "Failed" for r in rows)), notExecuted="0") + path = self.path / "Example.Tests.trx" + ET.ElementTree(root).write(path, encoding="utf-8", xml_declaration=True) + return path + + def evaluate(self): + return acceptance.evaluate(self.manifest, self.path) + + def test_complete_theory_passes_and_retains_manual_gate(self): + self.write_trx() + report = self.evaluate() + self.assertEqual("passed", report["automated_status"]) + self.assertEqual(2, report["groups"][0]["result_count"]) + self.assertEqual(["Visual/operator check"], report["groups"][0]["remaining_acceptance"]) + self.assertEqual(2, report["suites"]["Example.Tests"]["total"]) + + def test_missing_or_malformed_file_fails(self): + self.assertEqual("failed", self.evaluate()["automated_status"]) + (self.path / "Example.Tests.trx").write_text(" {result['outcome']}") + reports = [] + for group in groups: + requirements = group["required_tests"] + if not requirements: + raise ValueError(f"{group['id']}: empty required test list") + failures, matched = [], [] + seen = set() + for requirement in requirements: + project, method, minimum = ( + requirement["project"], requirement["method"], requirement["minimum_cases"] + ) + if project not in projects or not method.startswith(project + "."): + raise ValueError(f"{group['id']}: invalid project/method") + if type(minimum) is not int or minimum < 1 or (project, method) in seen: + raise ValueError(f"{group['id']}: duplicate test or invalid minimum_cases") + seen.add((project, method)) + rows = [r for r in suites.get(project, {}).get("results", []) if r["method"] == method] + if len(rows) < minimum: + failures.append(f"{method}: expected at least {minimum} cases, found {len(rows)}") + for row in rows: + if row["outcome"] != "Passed": + failures.append(f"{row['name']}: {row['outcome']}") + matched.extend(rows) + reports.append({ + "id": group["id"], + "title": group["title"], + "automated_status": "failed" if failures else "passed", + "result_count": len(matched), + "failures": failures, + "remaining_acceptance": group["remaining_acceptance"], + "tests": matched, + }) + return { + "schema_version": 1, + "automated_status": "failed" if errors or any(g["failures"] for g in reports) else "passed", + "errors": errors, + "suites": suites, + "groups": reports, + "notice": "Automated evidence only. Remaining acceptance is not waived; no tracker status is changed.", + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results", type=Path, default=Path("TestResults")) + parser.add_argument("--manifest", type=Path, default=Path(__file__).with_name("windows-acceptance.json")) + args = parser.parse_args(argv) + try: + report = evaluate(json.loads(args.manifest.read_text(encoding="utf-8")), args.results) + except (OSError, ValueError, KeyError, TypeError) as exc: + report = {"schema_version": 1, "automated_status": "failed", "errors": [str(exc)], "suites": {}, "groups": []} + source_root = Path(__file__).resolve().parent.parent + try: + def git(*arguments): + return subprocess.check_output(["git", *arguments], cwd=source_root, text=True).strip() + report["source"] = {"commit": git("rev-parse", "HEAD"), "tree": git("rev-parse", "HEAD^{tree}")} + if git("status", "--porcelain", "--untracked-files=no"): + raise ValueError("Tracked source differs from the committed tree") + except (OSError, subprocess.CalledProcessError, ValueError) as exc: + report["errors"].append(str(exc)) + report["automated_status"] = "failed" + report["runtime"] = platform.platform() + report["created_at"] = datetime.now(timezone.utc).isoformat() + report["workflow"] = {key: os.environ.get(key) for key in ( + "GITHUB_REPOSITORY", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "GITHUB_SHA" + )} + if platform.system() != "Windows": + report["errors"].append("Windows acceptance must execute on Windows; cross-builds are not runtime evidence") + report["automated_status"] = "failed" + args.results.mkdir(parents=True, exist_ok=True) + (args.results / "windows-acceptance.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + lines = ["# Windows automated acceptance", "", report["automated_status"].upper(), "", + "Automated evidence only; this does not close tracker tasks or waive remaining acceptance.", ""] + lines.extend(f"- {error}" for error in report["errors"]) + for project, suite in report["suites"].items(): + lines.append(f"- {project}: {suite['total']} results, {suite['outcomes']}") + for group in report["groups"]: + lines.extend(["", f"## {group['title']}: {group['automated_status']} ({group['result_count']} results)"]) + lines.extend(f"- FAIL: {f}" for f in group["failures"]) + lines.extend(f"- Remaining: {r}" for r in group["remaining_acceptance"]) + summary = "\n".join(lines) + "\n" + (args.results / "windows-acceptance.md").write_text(summary, encoding="utf-8") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as output: + output.write(summary) + print(summary) + return 0 if report["automated_status"] == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main())