ci(windows): retain fail-closed desktop acceptance evidence

This commit is contained in:
aj committed 2026-10-06 15:56:27 -04:00
1 parent bb43118451
commit b591528aa2
5 files changed
+1391

No files matched your search

+146
View File
@@ -0,0 +1,146 @@
"""Synthetic parser controls, not Windows application acceptance evidence."""
import copy
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import xml.etree.ElementTree as ET
import windows_acceptance as acceptance
class WindowsAcceptanceTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.path = Path(self.temp.name)
self.manifest = {
"schema_version": 1,
"projects": ["Example.Tests"],
"groups": [{
"id": "example", "title": "Example",
"required_tests": [{"project": "Example.Tests", "method": "Example.Tests.Form.Accept", "minimum_cases": 2}],
"remaining_acceptance": ["Visual/operator check"],
}],
}
self.rows = [("Example.Tests.Form.Accept", "Passed"), ("Example.Tests.Form.Accept", "Passed")]
def write_trx(self, rows=None, total=None, summary="Completed"):
rows = self.rows if rows is None else rows
root = ET.Element("TestRun", xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010")
definitions = ET.SubElement(root, "TestDefinitions")
results = ET.SubElement(root, "Results")
for index, (method, outcome) in enumerate(rows):
test = ET.SubElement(definitions, "UnitTest", id=str(index))
class_name, name = method.rsplit(".", 1)
ET.SubElement(test, "TestMethod", className=class_name, name=name)
result = ET.SubElement(results, "UnitTestResult", testId=str(index), executionId=str(index),
testName=f"{method}(case: {index})", outcome=outcome)
if outcome != "Passed":
ET.SubElement(ET.SubElement(result, "Output"), "StdOut").text = "fixture reason"
result_summary = ET.SubElement(root, "ResultSummary", outcome=summary)
ET.SubElement(result_summary, "Counters", total=str(len(rows) if total is None else total),
passed=str(sum(r[1] == "Passed" for r in rows)),
failed=str(sum(r[1] == "Failed" for r in rows)), notExecuted="0")
path = self.path / "Example.Tests.trx"
ET.ElementTree(root).write(path, encoding="utf-8", xml_declaration=True)
return path
def evaluate(self):
return acceptance.evaluate(self.manifest, self.path)
def test_complete_theory_passes_and_retains_manual_gate(self):
self.write_trx()
report = self.evaluate()
self.assertEqual("passed", report["automated_status"])
self.assertEqual(2, report["groups"][0]["result_count"])
self.assertEqual(["Visual/operator check"], report["groups"][0]["remaining_acceptance"])
self.assertEqual(2, report["suites"]["Example.Tests"]["total"])
def test_missing_or_malformed_file_fails(self):
self.assertEqual("failed", self.evaluate()["automated_status"])
(self.path / "Example.Tests.trx").write_text("<broken", encoding="utf-8")
self.assertEqual("failed", self.evaluate()["automated_status"])
def test_missing_method_or_theory_row_fails(self):
for rows in ([self.rows[0]], [("Example.Tests.Other.Accept", "Passed")] * 2):
with self.subTest(rows=rows):
self.write_trx(rows)
self.assertEqual("failed", self.evaluate()["automated_status"])
def test_skip_failed_aborted_and_unknown_required_results_fail(self):
for outcome in ("NotExecuted", "Failed", "Aborted", "TimedOut", "Unknown"):
with self.subTest(outcome=outcome):
self.write_trx([self.rows[0], (self.rows[1][0], outcome)])
report = self.evaluate()
self.assertEqual("failed", report["automated_status"])
self.assertEqual("fixture reason", report["groups"][0]["tests"][1]["detail"])
def test_unmapped_failed_test_still_fails_gate(self):
self.write_trx(self.rows + [("Example.Tests.Other.Failure", "Failed")])
self.assertEqual("failed", self.evaluate()["automated_status"])
def test_unmapped_skip_is_explicit_not_a_pass(self):
self.write_trx(self.rows + [("Example.Tests.Other.Optional", "NotExecuted")])
report = self.evaluate()
self.assertEqual("passed", report["automated_status"])
suite = report["suites"]["Example.Tests"]
self.assertEqual({"NotExecuted": 1, "Passed": 2}, suite["outcomes"])
self.assertEqual("fixture reason", suite["results"][2]["detail"])
def test_empty_or_inconsistent_counters_fail(self):
for rows, total in (([], 0), (self.rows, 3)):
self.write_trx(rows, total)
self.assertEqual("failed", self.evaluate()["automated_status"])
path = self.write_trx()
path.write_text(path.read_text().replace('passed="2"', 'passed="1"'))
self.assertEqual("failed", self.evaluate()["automated_status"])
def test_aborted_run_fails_even_with_passed_rows(self):
self.write_trx(summary="Aborted")
self.assertEqual("failed", self.evaluate()["automated_status"])
def test_duplicate_or_unmapped_execution_fails(self):
for old, new in (('executionId="1"', 'executionId="0"'),
('testId="1"', 'testId="not-defined"'), ('id="1"', 'id="0"')):
path = self.write_trx()
path.write_text(path.read_text().replace(old, new))
self.assertEqual("failed", self.evaluate()["automated_status"])
def test_invalid_manifests_are_rejected(self):
original = copy.deepcopy(self.manifest)
for change in (
lambda m: m.update(projects=[]),
lambda m: m.update(groups=[]),
lambda m: m["groups"].append(copy.deepcopy(m["groups"][0])),
lambda m: m["groups"][0].update(required_tests=[]),
lambda m: m["groups"][0]["required_tests"][0].update(minimum_cases=0),
lambda m: m["groups"][0]["required_tests"][0].update(project="Unknown.Tests"),
):
self.manifest = copy.deepcopy(original)
change(self.manifest)
with self.assertRaises(ValueError):
self.evaluate()
def test_cli_refuses_linux_runtime_and_dirty_source(self):
self.write_trx()
manifest = self.path / "manifest.json"
manifest.write_text(json.dumps(self.manifest), encoding="utf-8")
args = ["--results", str(self.path), "--manifest", str(manifest)]
for system, dirty in (("Linux", ""), ("Windows", " M source.cs")):
with self.subTest(system=system, dirty=dirty), \
patch.object(acceptance.platform, "system", return_value=system), \
patch.object(acceptance.platform, "platform", return_value="Synthetic parser fixture"), \
patch.object(acceptance.subprocess, "check_output", side_effect=["a" * 40, "b" * 40, dirty]), \
patch.dict(acceptance.os.environ, {"GITHUB_STEP_SUMMARY": ""}), \
patch("builtins.print"):
self.assertEqual(1, acceptance.main(args))
report = json.loads((self.path / "windows-acceptance.json").read_text())
self.assertEqual("failed", report["automated_status"])
self.assertTrue(report["errors"])
if __name__ == "__main__":
unittest.main()
+986
View File
@@ -0,0 +1,986 @@
{
"schema_version": 1,
"projects": [
"OpenNest.FrontEnd.Tests",
"OpenNest.WinForms.Tests"
],
"groups": [
{
"id": "bom-import",
"title": "BOM import and editable row identity",
"required_tests": [
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BomImportFormTests.BlankingMaterialOrThickness_MarksTheRowAndDropsItFromTheGroups",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BomImportFormTests.EditingQty_ChangesThatPartAndTheGroupTotal",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BomImportFormTests.Edits_ChangeThePartShownInThatRow_AfterTheRowsAreReloadedInAnotherOrder",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BomImportFormTests.InvalidQty_IsRefusedAndTheOldValueKept",
"minimum_cases": 5
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BomImportFormTests.PartColumns_KeepBomOrderAndOnlyMaterialThicknessAndQtyAreEditable",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BomImportFormTests.ProgrammaticCommitOfInvalidText_KeepsTheRowsValue",
"minimum_cases": 4
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BomImportFormTests.RowsWithoutADrawing_CannotEnterEditMode",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"Visual appearance of disabled rows; additional material-dropdown behavior is not covered."
]
},
{
"id": "nest-material",
"title": "Nest Info material preservation",
"required_tests": [
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.EditNestInfoFormTests.SaveWithChangedMaterial_GetsNameOnly",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.EditNestInfoFormTests.SaveWithUnchangedMaterial_KeepsGradeAndDensity",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"Shared-material selection requires its separate implementation."
]
},
{
"id": "database-browser",
"title": "Saved-nest browser",
"required_tests": [
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.SavedNestsFormTests.HighlightedNestShowsItsPlatesAndDrawings_AndEnterOpensIt",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.SavedNestsFormTests.MissingArchive_ShowsTheErrorInTheDetailsLine_AndEscapeCancels",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.SavedNestsFormTests.OpensAsALargeNormalWindowInsideTheOwnersScreen",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.SavedNestsFormTests.PlatePreview_StepsThroughThePlates_AndFollowsThePlatesTable",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"Real HTTP/SQLite save/update/copy, server switching, outage/recovery, paging/sort/search/debounce, empty-last-page deletion, and two-PC checks are not established by the fake-repository form tests.",
"Multi-monitor and DPI appearance need operator acceptance."
]
},
{
"id": "nesting-pipeline",
"title": "Nesting controls and front-end commit policy",
"required_tests": [
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.DefaultEngineSelectionTests.ConsoleAutonestWithoutAnEngineUsesDefault",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.DefaultEngineSelectionTests.McpAutonestEngineDescriptionNamesEveryBuiltInEngine",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.DefaultEngineSelectionTests.McpAutonestWithoutAnEngineUsesDefault",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.DefaultEngineSelectionTests.McpFillToolsWithoutAnEngineUseFill",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Api_PluginSelectionDerivesCountsPreservesIdsAndReportsValidation",
"minimum_cases": 5
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Api_UnknownEngineAndCancellationPropagate",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Console_CancelledOrdinaryFillDoesNotOverwriteOutput",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Console_CancelledSolveDoesNotSave",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Console_EmptyResultDoesNotOverwriteOutputOrOriginal",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Console_KeepPartsOnOccupiedSheetRejectsBeforeSolve",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Console_RefusesInvalidWithoutOverwritingUnlessFaithfullyKeepable",
"minimum_cases": 5
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Console_ValidCommitPreservesSettingsAndUsesOnePhysicalSheet",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Mcp_CancellationDoesNotCommit",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Mcp_OccupiedSheetRejectsBeforeSolve",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Mcp_OnlyCommitsFaithfulSingleSheet",
"minimum_cases": 6
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.Mcp_SessionGateSerializesCallsBeforeCheckingOccupiedTarget",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.NestingPipelineTests.UnknownEngineDoesNotWriteOrCommit",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.ActionSelectAreaCutOffTests.SelectedAreaAndFillStayOnSelectedSideOfCutOff",
"minimum_cases": 16
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutoNestFormStockOptionsTests.BlankAndCancelledRowsDoNotBecomeStockOptions",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutoNestFormStockOptionsTests.FractionalStockSizesRoundTripUnderCommaDecimalCulture",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutoNestFormStockOptionsTests.InvalidPendingEditCannotPublishOldBoundValue",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutoNestFormStockOptionsTests.NonblankInvalidRowRejectsWholeCollection",
"minimum_cases": 5
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutoNestFormStockOptionsTests.StockOptionsKeepAnAddRowAfterAddingAPlate",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutoNestFormStockOptionsTests.StockSizeRegexHasExplicitTimeout",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutoNestFormStockOptionsTests.TimeoutAfterValidRowPublishesNoPrefixAndDoesNotAcceptDialog",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestValidationFormTests.ClosingWholeJobProgressWaitsForWorkerAndRequestsCancellation",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestValidationFormTests.CompletionClosesWithoutCancellingSuccessfulJob",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestValidationFormTests.DiscardIsDefaultAndMalformedOutputCannotBeKept",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateViewFillLifetimeTests.ClosedProgressWindowStaysBusyThroughCommitAndCleanup",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateViewFillLifetimeTests.QueuedProgressCannotOverwriteNewPreviewEvenDuringAnotherFill",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateViewFillLifetimeTests.SetupFailureReleasesGuardAndClosesProgressWindow",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateViewFillLifetimeTests.WorkerFailureReleasesGuardAndDiscardsPreview",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"Interactive engine preference/fallback and every group/area-fill path still need their specific operator checks."
]
},
{
"id": "pdf-report",
"title": "Desktop PDF export",
"required_tests": [
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.BusyAfterDialogRejectsWriteWithoutTouchingDestination",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.BusyPlateActionOnAnyViewOfTheNestRejects",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.CancelWritesNothingAndNotifiesNothing",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.InteractiveFillOnAnyViewOfTheNestRejects",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.InvalidNestNameRejectsBeforeAnyDialog",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.JobBusyRejectsBeforeAnyDialog",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.MenuFollowsActiveDocument",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.RenderFailureLeavesExistingDestinationUnchanged",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.SuccessfulExportWritesPdfAndLeavesJobStateUntouched",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.NestReportExportTests.WriteRejectsAStaleTargetFromAnotherDocument",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"Packaged-build fonts/notices, physical print output, and report visual readability are not verified by these development-build tests."
]
},
{
"id": "cutting-planner",
"title": "Current/all-plate cutting dialog",
"required_tests": [
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.PlateViewReorderTests.DisposedViewIgnoresItsRetainedPlate",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.PlateViewReorderTests.FollowsAnInPlaceReorderOfItsCurrentPlateOnly",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.PlateViewReorderTests.KeepsOneLayoutPerOccurrenceOfARepeatedPart",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CuttingPlanFormTests.AllPlates_SkipsEmptyPlatesAndRefusesTheBatchWhenOnePlateIsBlocked",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CuttingPlanFormTests.ApplyAfterALiveEdit_ChangesNothingAndReplanRecovers",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CuttingPlanFormTests.BlockedPlate_KeepsApplyDisabledAndShowsTheCurrentParts",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CuttingPlanFormTests.BusyEditor_DoesNotStartPlanningUntilItIsFree",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CuttingPlanFormTests.ClosingWhilePlanning_CancelsAndWaitsForTheWorker",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CuttingPlanFormTests.Constructor_PlansWithItsOwnCopyOfTheSettings",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CuttingPlanFormTests.PlanThenApply_InstallsThePlanAndRedrawsTheEditorInPlateOrder",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"Selected-part side-panel migration and legacy automatic-path retirement are separate unfinished work.",
"Actual job and NC/controller safety are not certified."
]
},
{
"id": "automatic-cutoffs",
"title": "Automatic cutoff dialogs",
"required_tests": [
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.ApplyRecomputesWithCurrentPartsAndSettingsAndRegeneratesOnce",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.ApplyRefusesEmptyPlate",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.ApplyRefusesNewBlockingConflictSincePreview",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.CloseCallbackClearsPreviewWithoutApplying",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.FailedRegenerationRestoresDefinitionsProgramsAndPartSequence",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.InitialSpacingUsesNestUnitsAndActualSheetWidth",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.InvalidMinimumTailBlocksApply",
"minimum_cases": 3
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.MinimumTailChangeClearsPreviewAndApplyUsesCurrentValue",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.NestApply_LaterConflictBlocksEntireNest",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.NestApply_PreviewsWithoutMutationAndAppliesEveryPlate",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.NestApply_RechecksOperationGuardAfterPreview",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.NestPreview_DisposeLeavesEveryPlateUntouched",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.PreviewIsDetachedAndDisposeClearsIt",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.AutomaticCutOffFormTests.SpacingChangeAndInvalidPreviewClearOldOverlay",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"Full saved-job cutting-state persistence and operator corner-lead checks remain separate."
]
},
{
"id": "overlap-posting",
"title": "Overlap display and posting consent",
"required_tests": [
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.LeadInSidePanelTests.PanelClosesWithoutEscapeReopeningIt",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PostVerificationFormTests.ClearReportIsDisplayedAndCancelIsDefault",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PostVerificationFormTests.ClosingDuringVerificationWaitsForWorkerAndDiscardsLateResult",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PostVerificationFormTests.DisposalIsIdempotent",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PostVerificationFormTests.RealModalCancellationKeepsOwnerDisabledUntilWorkerStops",
"minimum_cases": 3
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PostVerificationFormTests.UnexpectedFailureCannotBeOverridden",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PostVerificationFormTests.WarningsRequireFreshAcknowledgmentAndUncheckingRevokesPermission",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.AppendingPartRetainsTheExistingPairsPathButInvalidatesTheFullReport",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.AutoCheckKeepsDisplayOffButManualCheckStillShowsAreasAndReportsToStatusBar",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.AutoCheckRunsOnceTheLayoutSettlesAndRechecksAfterADrag",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.AutoCheckSupersedesARunningCheckWhenTheLayoutChanges",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.AutoCheckWaitsForInteractionsAndRespectsCancel",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.BothRetainsAreaPathAndCoincidentHoverListsEveryPairWithoutChangingSelection",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.CentroidMarkerUsesGraphCoordinatesAndFixedScreenSizeAcrossZoom",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.CheckCommandCapturesUnitsAndNamesUntilTheNextCheck",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.CleanupReleasesDequeuedWorkersEvenWithoutCompleteOrFail",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.CompletedPathIsReleasedOnHandleRecreationAndCannotReappear",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.CrowdedHoverPagesStayInViewportAndReachLastPairWithoutSelectionOrWheelCapture",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.DefaultAnalyzerRechecksIncrementallyAndEditorsDropTheCache",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.DisplayPanZoomAndPreviewDoNotAnalyzeOrChangeCommittedParts",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.EditNestFormEnablesAutoCheck",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.EditOrPlateSwitchRejectsPendingReport",
"minimum_cases": 7
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.FailureAndIncompleteAreVisibleAndNeverReportedAsClear",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.HandleDestructionOrDisposalNeverResurrectsRequests",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.HoverClearsBeforeStaleDetailsCanBeShown",
"minimum_cases": 11
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.LongNameAndTinyViewNeverSilentlyElideTheOnlyPair",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.MarkerAndHoverUsePairMaterialCentroidRatherThanBoundsCenter",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.MenusFollowMdiActivationDocumentDisplayAndRunningState",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.MovingDirectlyToAnotherCoincidentGroupStartsItsFirstPage",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.PagingResetsWithHoverLifetimeAndInactiveKeysAreNotConsumed",
"minimum_cases": 9
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.PaintChecksFreshnessEvenWhenDisplayIsOff",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.PairDetailsOnlySuppressNormalTooltipWhileVisible",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.RecheckPreservesEachVisibleModeAndDisplayOnlyRepaints",
"minimum_cases": 3
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.RestartAndCancelRejectLateCompletionsOnTheUiThread",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.RetainedPathClearsOnHardInvalidationOrRequestFailure",
"minimum_cases": 3
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.SlotChangingCollectionEventsImmediatelyDiscardCurrentReportAndPath",
"minimum_cases": 3
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.SnapshotKeepsOriginalLabelsWhenLiveNamesChange",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.TripleOverlapUsesOneFillAndGraphCoordinatesDoNotDoublePan",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.PlateOverlapOverlayTests.UnchangedPairStaysVisibleDuringMotionAndPendingRecheck",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"DPI/visual appearance, full lead/tab/lock save-reopen-edit-post lifecycle, and actual machine qualification remain separate."
]
},
{
"id": "ui-lifetime",
"title": "UI lifetime, focus and disposal",
"required_tests": [
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.SeparatorPenLifetimeTests.DisposeReleasesBothPensAndDisposesTheControlOnlyOnce",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.SeparatorPenLifetimeTests.PaintingKeepsOrientationColorsStylesAndResizeBehavior",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BestFitViewerOperationLifetimeTests.CloseOrDisposalRejectsLateResultsAndErrorsWithoutTouchingControls",
"minimum_cases": 4
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BestFitViewerOperationLifetimeTests.CurrentComputeOrRenderFailureReportsOnceAndClearsPartialAndPreviousResults",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BestFitViewerOperationLifetimeTests.DispatcherDrainsPendingCallbacksWhenCompletionAlreadyHolds",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BestFitViewerOperationLifetimeTests.DispatcherObservesAsyncVoidFailureAfterOperationCompletes",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BestFitViewerOperationLifetimeTests.OldCompletionCannotPublishReportOrResetNewerOperation",
"minimum_cases": 4
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BestFitViewerOperationLifetimeTests.OnlyRequestedCancellationIsExpectedAndNonmodal",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.BestFitViewerOperationLifetimeTests.ShownAndSelectionPublishCompleteResultsWithPagingAndSourceSelection",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.EditNestFocusTraversalTests.NestedActiveContainersReturnTheirLeaf",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.EditNestFocusTraversalTests.NestedContainerWithoutActiveControlIsReturned",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.EditNestFocusTraversalTests.NoActiveControlReturnsTheForm",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.EditNestFocusTraversalTests.OrdinaryActiveControlIsReturnedUnchanged",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.EditNestFocusTraversalTests.PlateListRefreshKeepsFocusOnThePlateList",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.EditNestFocusTraversalTests.PlateListRefreshRestoresTheOriginalNestedLeaf",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.StaTestThreadTests.QueuedCallbackExceptionFailsTheTestInsteadOfOpeningADialog",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.StaTestThreadTests.WithoutThrowExceptionModeTheCallbackExceptionNeverReachesTheTestBody",
"minimum_cases": 1
}
],
"remaining_acceptance": []
},
{
"id": "import-export",
"title": "Import/export failures and highlighting",
"required_tests": [
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.RegexImportFailureTests.Api_TimeoutRetainsInnerExceptionAndNeverSolvesOrReturnsResponse",
"minimum_cases": 2
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.RegexImportFailureTests.Console_TimeoutExitsOneWithoutOutputEvenAfterEarlierImport",
"minimum_cases": 2
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.RegexImportFailureTests.Mcp_TimeoutReturnsErrorWithoutAddingDrawingOrChangingEarlierDrawing",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.ArchUnitsTests.ArchitecturalRegexHasExplicitTimeout",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.ArchUnitsTests.InvalidFieldIsRedAndNaNAndCorrectionRestoresIt",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.ArchUnitsTests.OrdinaryArchitecturalInputsKeepTheirMeaning",
"minimum_cases": 5
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.ProgramEditorControlTests.OrdinaryHighlightingPreservesAllColorsAndUnmatchedCoordinateText",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.ProgramEditorControlTests.ReverseAndReorderFinishModelPreviewAndEventsWithOrWithoutTimeout",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.ProgramEditorControlTests.SuccessfulRetryAndClearRemoveTheNonmodalTimeoutIndication",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.ProgramEditorControlTests.TimeoutDuringInitialLoadKeepsCompleteTextAndBaseColorWithoutAParentHandle",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.ProgramEditorControlTests.UnexpectedApplicationFailureStillRestoresSelectionAndResumesLayout",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Controls.ProgramEditorControlTests.UnexpectedMatchingFailureIsNotSwallowed",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CadConverterImportFailureTests.ExtractionTimeoutPublishesNeitherRawTextNorPartialFile",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CadConverterImportFailureTests.MTextRegexHasExplicitTimeout",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CadConverterImportFailureTests.MTextStrippingKeepsItsExistingRules",
"minimum_cases": 4
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.CadConverterImportFailureTests.SuccessfulExtractionPreservesTextAndControlCodeSemantics",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.CancelWritesNothingAndNotifiesNothing",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.CoordinateExportLeavesDocumentAndJobStateUntouched",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.CoordinatesPreserveFormatAndReleaseFile",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.ExportAllStopsAtFirstWriteFailure",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.ExportAllStopsWhenSecondDialogIsCanceled",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.ExportAllWritesEveryPlateOnceAndAdvancesToLast",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.FailureAfterPartialWriteIsNotSuccess",
"minimum_cases": 3
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.FlushFailureDuringWriterDisposalIsNotSuccess",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.JpegRendersShownPlateAndReleasesFile",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.MissingParentDirectoryReportsOneFailure",
"minimum_cases": 3
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.PlateExportTests.UnsupportedFilterReportsFailureWithoutCreatingFile",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.ShapeLibraryFormTests.FriendlyNameRegexHasExplicitTimeout",
"minimum_cases": 1
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.ShapeLibraryFormTests.FriendlyNameTimeoutDiscardsPartialFormatting",
"minimum_cases": 2
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.ShapeLibraryFormTests.FriendlyNamesKeepSuffixAndWordBoundaryBehavior",
"minimum_cases": 4
},
{
"project": "OpenNest.WinForms.Tests",
"method": "OpenNest.WinForms.Tests.Forms.ShapeLibraryFormTests.InvalidShapeInputClearsOldPreviewAndBlocksAddUntilCorrected",
"minimum_cases": 1
}
],
"remaining_acceptance": [
"Atomic replacement of existing plate-export destinations is a separate hardening task, not a guarantee of truthful failure reporting."
]
},
{
"id": "mcp-process",
"title": "Windows child-process lifetime",
"required_tests": [
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.AlreadyCancelled_DoesNotStartChild",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.Arguments_PreserveSpacesAndQuotes",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.Cancellation_StopsOwnedLiveTreeWithoutKillingUnrelatedProcess",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.Completed_PreservesStderrAndNonzeroStatus",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.ConcurrentDrains_PreserveFullOutputBeyondPipeCapacity",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.Deadline_BoundsHungChildAndReapsIt",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.Deadline_BoundsPipeDrainAfterParentExits",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.InvalidConfiguration_FailsClearlyWithoutStartingChild",
"minimum_cases": 5
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.MissingNest_FailsClearly",
"minimum_cases": 1
},
{
"project": "OpenNest.FrontEnd.Tests",
"method": "OpenNest.FrontEnd.Tests.McpEngineHarnessProcessTests.Normal_ReachesPrivateChild",
"minimum_cases": 1
}
],
"remaining_acceptance": []
}
]
}
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""Fail-closed evidence for named Windows tests, not automatic feature sign-off."""
import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
import xml.etree.ElementTree as ET
def read_suite(path):
"""Read actual result rows; TRX notExecuted counters can omit xUnit skips."""
root = ET.parse(path).getroot()
if root.tag != "{http://microsoft.com/schemas/VisualStudio/TeamTest/2010}TestRun":
raise ValueError(f"{path.name}: not a TRX TestRun")
definitions = {}
for test in root.findall("./{*}TestDefinitions/{*}UnitTest"):
method = test.find("{*}TestMethod")
identity = test.attrib["id"]
if identity in definitions or method is None:
raise ValueError(f"{path.name}: duplicate or incomplete test definition")
definitions[identity] = method.attrib["className"] + "." + method.attrib["name"]
results = []
executions = set()
for result in root.findall("./{*}Results/{*}UnitTestResult"):
execution = result.attrib["executionId"]
if execution in executions:
raise ValueError(f"{path.name}: duplicate test execution")
executions.add(execution)
output = result.find("{*}Output")
results.append({
"method": definitions[result.attrib["testId"]],
"name": result.attrib["testName"],
"outcome": result.attrib["outcome"],
"detail": "\n".join(output.itertext()).strip() if output is not None else "",
})
summary = root.find("{*}ResultSummary")
if summary is None or summary.attrib.get("outcome") not in ("Completed", "Passed"):
raise ValueError(f"{path.name}: missing or unsuccessful run summary")
counters = summary.find("{*}Counters")
if counters is None or not results or int(counters.attrib["total"]) != len(results):
raise ValueError(f"{path.name}: empty results or total does not match enumerated results")
outcomes = Counter(r["outcome"] for r in results)
for key, outcome in (("passed", "Passed"), ("failed", "Failed")):
if int(counters.attrib[key]) != outcomes[outcome]:
raise ValueError(f"{path.name}: {key} counter does not match result rows")
return {
"file": path.name,
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"total": len(results),
"outcomes": dict(sorted(outcomes.items())),
"results": results,
}
def evaluate(manifest, results_dir):
if manifest.get("schema_version") != 1:
raise ValueError("Unsupported acceptance manifest schema")
projects = manifest["projects"]
groups = manifest["groups"]
if not projects or len(projects) != len(set(projects)) or not groups:
raise ValueError("Manifest must contain unique projects and nonempty groups")
if any(Path(p).name != p or not p.endswith(".Tests") for p in projects):
raise ValueError("Invalid test project name")
ids = [g["id"] for g in groups]
if len(ids) != len(set(ids)):
raise ValueError("Duplicate acceptance group")
suites, errors = {}, []
for project in projects:
try:
suites[project] = read_suite(results_dir / f"{project}.trx")
except (OSError, ET.ParseError, ValueError, KeyError) as exc:
errors.append(f"{project}: {exc}")
for project, suite in suites.items():
for result in suite["results"]:
if result["outcome"] not in ("Passed", "NotExecuted"):
errors.append(f"{project}: {result['name']} => {result['outcome']}")
reports = []
for group in groups:
requirements = group["required_tests"]
if not requirements:
raise ValueError(f"{group['id']}: empty required test list")
failures, matched = [], []
seen = set()
for requirement in requirements:
project, method, minimum = (
requirement["project"], requirement["method"], requirement["minimum_cases"]
)
if project not in projects or not method.startswith(project + "."):
raise ValueError(f"{group['id']}: invalid project/method")
if type(minimum) is not int or minimum < 1 or (project, method) in seen:
raise ValueError(f"{group['id']}: duplicate test or invalid minimum_cases")
seen.add((project, method))
rows = [r for r in suites.get(project, {}).get("results", []) if r["method"] == method]
if len(rows) < minimum:
failures.append(f"{method}: expected at least {minimum} cases, found {len(rows)}")
for row in rows:
if row["outcome"] != "Passed":
failures.append(f"{row['name']}: {row['outcome']}")
matched.extend(rows)
reports.append({
"id": group["id"],
"title": group["title"],
"automated_status": "failed" if failures else "passed",
"result_count": len(matched),
"failures": failures,
"remaining_acceptance": group["remaining_acceptance"],
"tests": matched,
})
return {
"schema_version": 1,
"automated_status": "failed" if errors or any(g["failures"] for g in reports) else "passed",
"errors": errors,
"suites": suites,
"groups": reports,
"notice": "Automated evidence only. Remaining acceptance is not waived; no tracker status is changed.",
}
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--results", type=Path, default=Path("TestResults"))
parser.add_argument("--manifest", type=Path, default=Path(__file__).with_name("windows-acceptance.json"))
args = parser.parse_args(argv)
try:
report = evaluate(json.loads(args.manifest.read_text(encoding="utf-8")), args.results)
except (OSError, ValueError, KeyError, TypeError) as exc:
report = {"schema_version": 1, "automated_status": "failed", "errors": [str(exc)], "suites": {}, "groups": []}
source_root = Path(__file__).resolve().parent.parent
try:
def git(*arguments):
return subprocess.check_output(["git", *arguments], cwd=source_root, text=True).strip()
report["source"] = {"commit": git("rev-parse", "HEAD"), "tree": git("rev-parse", "HEAD^{tree}")}
if git("status", "--porcelain", "--untracked-files=no"):
raise ValueError("Tracked source differs from the committed tree")
except (OSError, subprocess.CalledProcessError, ValueError) as exc:
report["errors"].append(str(exc))
report["automated_status"] = "failed"
report["runtime"] = platform.platform()
report["created_at"] = datetime.now(timezone.utc).isoformat()
report["workflow"] = {key: os.environ.get(key) for key in (
"GITHUB_REPOSITORY", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "GITHUB_SHA"
)}
if platform.system() != "Windows":
report["errors"].append("Windows acceptance must execute on Windows; cross-builds are not runtime evidence")
report["automated_status"] = "failed"
args.results.mkdir(parents=True, exist_ok=True)
(args.results / "windows-acceptance.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
lines = ["# Windows automated acceptance", "", report["automated_status"].upper(), "",
"Automated evidence only; this does not close tracker tasks or waive remaining acceptance.", ""]
lines.extend(f"- {error}" for error in report["errors"])
for project, suite in report["suites"].items():
lines.append(f"- {project}: {suite['total']} results, {suite['outcomes']}")
for group in report["groups"]:
lines.extend(["", f"## {group['title']}: {group['automated_status']} ({group['result_count']} results)"])
lines.extend(f"- FAIL: {f}" for f in group["failures"])
lines.extend(f"- Remaining: {r}" for r in group["remaining_acceptance"])
summary = "\n".join(lines) + "\n"
(args.results / "windows-acceptance.md").write_text(summary, encoding="utf-8")
if os.environ.get("GITHUB_STEP_SUMMARY"):
with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as output:
output.write(summary)
print(summary)
return 0 if report["automated_status"] == "passed" else 1
if __name__ == "__main__":
sys.exit(main())