test(irregular): add six synthetic nests and bounded validity gate

scripts/check-synthetic-nests.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+"""Bounded validity/fulfillment gate; parallel elapsed times are not speed evidence."""
+
+import argparse
+import csv
+import hashlib
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import zipfile
+
+ROOT = Path(__file__).resolve().parents[1]
+# Exact demand and sheet count: empty, skipped, and partial jobs must never pass.
+EXPECTED = {
+    "single-triangle": (1, 1),
+    "paired-wedges": (2, 1),
+    "repeated-ell-fill": (7, 1),
+    "mixed-irregular": (6, 1),
+    "rotation-edge-fit": (1, 1),
+    "multi-sheet": (3, 3),
+}
+
+
+class GateError(Exception):
+    pass
+
+
+def check_fixtures(directory):
+    files = list(directory.rglob("*.nest"))
+    names = [path.stem for path in files]
+    if len(names) != len(EXPECTED) or set(names) != set(EXPECTED):
+        raise GateError(f"Expected exactly six fixtures {sorted(EXPECTED)}; found {sorted(names)}")
+    if list(directory.rglob("*.manifest.json")):
+        raise GateError("Unexpected manifest in synthetic fixture directory")
+    hashes = {}
+    for path in sorted(files):
+        with zipfile.ZipFile(path) as archive:
+            metadata = json.loads(archive.read("nest.json").decode("utf-8-sig"))
+            drawings = metadata["drawings"]
+            demand = sum(drawing["quantity"]["required"] for drawing in drawings)
+            if demand != EXPECTED[path.stem][0] or not drawings:
+                raise GateError(f"Unexpected demand in {path.name}: {demand}")
+            if any(metadata.get(key) for key in ("customer", "madeBy")):
+                raise GateError(f"Non-synthetic identity metadata in {path.name}")
+            for drawing in drawings:
+                if drawing.get("customer") or drawing.get("source", {}).get("path"):
+                    raise GateError(f"Non-synthetic drawing metadata in {path.name}")
+            if archive.comment or any(entry.comment for entry in archive.infolist()):
+                raise GateError(f"Unexpected ZIP comments in {path.name}")
+        hashes[path.name] = hashlib.sha256(path.read_bytes()).hexdigest()
+    return hashes
+
+
+def check_csv(path):
+    seen = set()
+    outcomes = []
+    with path.open(newline="", encoding="utf-8-sig") as stream:
+        reader = csv.DictReader(stream)
+        required = {"Job", "Engine", "Valid", "Crashed", "FullyPlaced", "PartsPlaced",
+                    "PartsRequested", "PlatesUsed", "Notes"}
+        if not required.issubset(reader.fieldnames or []):
+            raise GateError("Missing benchmark CSV columns")
+        for row in reader:
+            if row["Engine"] == "Baseline":
+                continue
+            if row["Engine"] != "Irregular":
+                raise GateError(f"Unexpected engine: {row['Engine']}")
+            name = row["Job"]
+            if name not in EXPECTED or name in seen:
+                raise GateError(f"Unknown or duplicate fixture result: {name}")
+            seen.add(name)
+            demand, sheets = EXPECTED[name]
+            if (row["Valid"] != "True" or row["Crashed"] != "False"
+                    or row["FullyPlaced"] != "True" or row["Notes"].strip()):
+                raise GateError(f"Invalid, crashed, timed-out or incomplete result: {name}: {row}")
+            if (int(row["PartsRequested"]) != demand or int(row["PartsPlaced"]) != demand
+                    or int(row["PlatesUsed"]) != sheets):
+                raise GateError(f"Unexpected fulfillment or sheet count: {name}: {row}")
+            outcomes.append(row)
+    if seen != set(EXPECTED):
+        raise GateError(f"Missing fixture results: {sorted(set(EXPECTED) - seen)}")
+    return outcomes
+
+
+def terminate_tree(process):
+    if os.name == "nt":
+        if process.poll() is None:
+            subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"],
+                           check=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT,
+                           timeout=10)
+    else:
+        # Also remove any descendants left after their parent exits. The session is ours alone.
+        try:
+            os.killpg(process.pid, signal.SIGKILL)
+        except ProcessLookupError:
+            pass
+    process.wait(timeout=10)
+
+
+def run_process(command, deadline, log):
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        raise GateError("Whole-check watchdog expired")
+    print("Running: " + " ".join(map(str, command)), flush=True)
+    with log.open("w", encoding="utf-8") as output:
+        process = subprocess.Popen(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT,
+                                   start_new_session=os.name != "nt",
+                                   creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0)
+        try:
+            code = process.wait(timeout=remaining)
+            if code:
+                raise GateError(f"Subprocess exited {code}; see {log}")
+        except subprocess.TimeoutExpired as error:
+            raise GateError(f"Whole-check watchdog expired; see {log}") from error
+        finally:
+            terminate_tree(process)
+
+
+def bounded_int(low, high):
+    def parse(value):
+        number = int(value)
+        if not low <= number <= high:
+            raise argparse.ArgumentTypeError(f"must be between {low} and {high}")
+        return number
+    return parse
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--parallel", type=bounded_int(1, 4), default=2)
+    parser.add_argument("--timeout", type=bounded_int(1, 300), default=300,
+                        help="whole build + solve budget, seconds (maximum 300)")
+    parser.add_argument("--fixtures", type=Path, default=ROOT / "test-data/synthetic-nests")
+    parser.add_argument("--output", type=Path, help="retain logs/CSV/provenance in this directory")
+    parser.add_argument("--check-csv", type=Path, help="check an existing CSV without solving")
+    args = parser.parse_args(argv)
+    try:
+        if args.check_csv:
+            outcomes = check_csv(args.check_csv)
+        else:
+            deadline = time.monotonic() + args.timeout
+            hashes = check_fixtures(args.fixtures)
+            output = args.output or Path(tempfile.mkdtemp(prefix="opennest-synthetic-"))
+            output = output.resolve()
+            output.mkdir(parents=True, exist_ok=True)
+            (output / "accepted.json").unlink(missing_ok=True)
+            print(f"Evidence directory: {output}", flush=True)
+            project = ROOT / "OpenNest.Benchmark/OpenNest.Benchmark.csproj"
+            run_process(["dotnet", "build", str(project), "-c", "Release",
+                         "--disable-build-servers"], deadline, output / "build.log")
+            assembly = ROOT / "OpenNest.Benchmark/bin/Release/net8.0/OpenNest.Benchmark.dll"
+            command = ["dotnet", str(assembly), str(args.fixtures.resolve()),
+                       "--engines", "Irregular", "--parallel", str(args.parallel),
+                       "--progress", "--csv", str(output / "results.csv")]
+            run_process(command, deadline, output / "benchmark.log")
+            outcomes = check_csv(output / "results.csv")
+            if check_fixtures(args.fixtures) != hashes:
+                raise GateError("Fixture inputs changed during the check")
+            (output / "accepted.json").write_text(json.dumps({
+                "fixtures": hashes, "parallel": args.parallel,
+                "benchmark_command": list(map(str, command)), "outcomes": outcomes,
+                "benchmark_assembly_sha256": hashlib.sha256(assembly.read_bytes()).hexdigest(),
+                "checker_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
+                "timing": "diagnostic only; not speed evidence",
+            }, indent=2) + "\n", encoding="utf-8")
+        for row in sorted(outcomes, key=lambda row: row["Job"]):
+            print(f"PASS {row['Job']}: {row['PartsPlaced']}/{row['PartsRequested']} parts, "
+                  f"{row['PlatesUsed']} sheet(s)")
+        print("PASS: exactly six Irregular results, all valid and fully placed")
+        return 0
+    except KeyboardInterrupt:
+        print("FAIL: cancelled; owned subprocess tree terminated", file=sys.stderr)
+        return 130
+    except (GateError, OSError, ValueError, KeyError, zipfile.BadZipFile,
+            subprocess.SubprocessError) as error:
+        print(f"FAIL: {error}", file=sys.stderr)
+        return 1
+
+
+def cancel(_signum, _frame):
+    raise KeyboardInterrupt()
+
+
+if __name__ == "__main__":
+    # Treat CI/terminal termination the same as Ctrl-C; run_process's finally owns cleanup.
+    signal.signal(signal.SIGTERM, cancel)
+    sys.exit(main())

scripts/test_check_synthetic_nests.py

@@ -0,0 +1,171 @@
+"""Unit data below is intentionally synthetic CSV, not recorded benchmark evidence."""
+
+import csv
+import importlib.util
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+sys.dont_write_bytecode = True
+spec = importlib.util.spec_from_file_location("synthetic_gate", Path(__file__).with_name("check-synthetic-nests.py"))
+assert spec is not None and spec.loader is not None
+gate = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(gate)
+
+
+class SyntheticGateTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory(prefix="opennest-gate-test-")
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name)
+        self.csv = self.root / "results.csv"
+        self.rows = [dict(Job=name, Engine="Irregular", Valid="True", Crashed="False",
+                          FullyPlaced="True", PartsPlaced=str(demand), PartsRequested=str(demand),
+                          PlatesUsed=str(sheets), Notes="")
+                     for name, (demand, sheets) in gate.EXPECTED.items()]
+
+    def write_csv(self, rows):
+        with self.csv.open("w", newline="", encoding="utf-8") as stream:
+            writer = csv.DictWriter(stream, fieldnames=list(self.rows[0]))
+            writer.writeheader()
+            writer.writerows(rows)
+
+    def test_exact_six_and_baseline_filter(self):
+        self.write_csv(self.rows + [dict(self.rows[0], Engine="Baseline", Valid="False")])
+        self.assertEqual(6, len(gate.check_csv(self.csv)))
+
+    def test_fail_closed_result_cases(self):
+        cases = {
+            "invalid": dict(Valid="False"),
+            "crashed": dict(Crashed="True", Notes="exception"),
+            "timeout": dict(Valid="False", Notes="Timed out after 5 minutes"),
+            "incomplete": dict(FullyPlaced="False", PartsPlaced="0"),
+            "empty": dict(PartsPlaced="0", PartsRequested="0", PlatesUsed="0"),
+            "unknown": dict(Job="unknown"),
+            "engine": dict(Engine="Unknown"),
+            "demand": dict(PartsRequested="99"),
+            "sheets": dict(PlatesUsed="0"),
+            "notes": dict(Notes="validation warning"),
+        }
+        for label, change in cases.items():
+            with self.subTest(label=label):
+                self.write_csv([dict(self.rows[0], **change)] + self.rows[1:])
+                with self.assertRaises(gate.GateError):
+                    gate.check_csv(self.csv)
+        for rows in ([], self.rows[:-1], self.rows + [self.rows[0]]):
+            self.write_csv(rows)
+            with self.assertRaises(gate.GateError):
+                gate.check_csv(self.csv)
+
+    def test_malformed_csv_returns_nonzero(self):
+        self.write_csv([dict(self.rows[0], PartsPlaced="not-a-number")] + self.rows[1:])
+        self.assertEqual(1, gate.main(["--check-csv", str(self.csv)]))
+        self.csv.write_text("Job,Engine\n", encoding="utf-8")
+        self.assertEqual(1, gate.main(["--check-csv", str(self.csv)]))
+
+    def test_missing_fixture_actual_entrypoint(self):
+        self.assertEqual(1, gate.main(["--fixtures", str(self.root)]))
+
+    def test_committed_fixtures_are_complete_and_synthetic(self):
+        self.assertEqual(6, len(gate.check_fixtures(gate.ROOT / "test-data/synthetic-nests")))
+
+    def test_concurrency_and_watchdog_are_bounded(self):
+        for args in (["--parallel", "0"], ["--parallel", "5"], ["--timeout", "301"]):
+            with self.assertRaises(SystemExit) as error:
+                gate.main(args)
+            self.assertEqual(2, error.exception.code)
+
+    def test_nonzero_child_fails(self):
+        script = self.root / "exit.py"
+        script.write_text("raise SystemExit(7)\n", encoding="utf-8")
+        with self.assertRaisesRegex(gate.GateError, "exited 7"):
+            gate.run_process([sys.executable, str(script)], time.monotonic() + 10,
+                             self.root / "exit.log")
+
+    @unittest.skipIf(os.name == "nt", "POSIX SIGTERM probe; Windows requires runtime acceptance")
+    def test_cancellation_kills_ready_owned_tree(self):
+        parent = self.root / "cancel-parent.py"
+        pidfile = self.root / "cancel-pids.txt"
+        ready = self.root / "cancel-ready"
+        parent.write_text(
+            "import os, subprocess, sys, time\n"
+            "from pathlib import Path\n"
+            "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
+            f"Path({str(pidfile)!r}).write_text(str(os.getpid()) + ' ' + str(child.pid))\n"
+            f"Path({str(ready)!r}).touch()\n"
+            "time.sleep(60)\n", encoding="utf-8")
+        wrapper = self.root / "cancel-wrapper.py"
+        wrapper.write_text(
+            "import importlib.util, signal, sys, time\n"
+            "from pathlib import Path\n"
+            "sys.dont_write_bytecode = True\n"
+            f"spec = importlib.util.spec_from_file_location('gate', {str(Path(__file__).with_name('check-synthetic-nests.py'))!r})\n"
+            "gate = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(gate)\n"
+            "signal.signal(signal.SIGTERM, gate.cancel)\n"
+            "try:\n"
+            f"    gate.run_process([sys.executable, {str(parent)!r}], time.monotonic() + 15, "
+            f"Path({str(self.root / 'cancel.log')!r}))\n"
+            "except KeyboardInterrupt:\n    sys.exit(130)\n", encoding="utf-8")
+        process = subprocess.Popen([sys.executable, str(wrapper)], start_new_session=True)
+        try:
+            end = time.monotonic() + 10
+            while not ready.exists() and time.monotonic() < end:
+                time.sleep(0.01)
+            self.assertTrue(ready.exists(), "cancellation tree never became ready")
+            pids = [int(value) for value in pidfile.read_text().split()]
+            process.send_signal(signal.SIGTERM)
+            self.assertEqual(130, process.wait(timeout=10))
+            self.assert_quiescent(pids)
+        finally:
+            gate.terminate_tree(process)
+
+    def assert_quiescent(self, pids):
+        for pid in pids:
+            end = time.monotonic() + 5
+            while True:
+                status = Path(f"/proc/{pid}/stat")
+                if status.exists() and status.read_text().split()[2] == "Z":
+                    break  # Linux init may not yet have reaped a killed orphan.
+                try:
+                    os.kill(pid, 0)
+                except ProcessLookupError:
+                    break
+                self.assertLess(time.monotonic(), end, f"owned process {pid} remains alive")
+                time.sleep(0.01)
+
+    @unittest.skipIf(os.name == "nt", "POSIX session-tree probe; Windows uses taskkill /T /F")
+    def test_watchdog_kills_ready_parent_and_grandchild(self):
+        # A real owned process tree, synchronized before the watchdog. No benchmark results mocked.
+        child = self.root / "child.py"
+        child.write_text("import time\ntime.sleep(60)\n", encoding="utf-8")
+        parent = self.root / "parent.py"
+        pidfile = self.root / "pids.txt"
+        ready = self.root / "ready"
+        parent.write_text(
+            "import os, subprocess, sys, time\n"
+            "from pathlib import Path\n"
+            f"child = subprocess.Popen([sys.executable, {str(child)!r}])\n"
+            f"Path({str(pidfile)!r}).write_text(str(os.getpid()) + ' ' + str(child.pid))\n"
+            f"Path({str(ready)!r}).touch()\n"
+            "time.sleep(60)\n", encoding="utf-8")
+        from concurrent.futures import ThreadPoolExecutor
+        with ThreadPoolExecutor(max_workers=1) as executor:
+            future = executor.submit(gate.run_process, [sys.executable, str(parent)],
+                                     time.monotonic() + 5, self.root / "timeout.log")
+            end = time.monotonic() + 4
+            while not ready.exists() and time.monotonic() < end:
+                time.sleep(0.01)
+            self.assertTrue(ready.exists(), "child tree never became ready")
+            pids = [int(value) for value in pidfile.read_text().split()]
+            with self.assertRaisesRegex(gate.GateError, "watchdog expired"):
+                future.result(timeout=10)
+        self.assert_quiescent(pids)
+
+
+if __name__ == "__main__":
+    unittest.main()