diff --git a/AGENTS.md b/AGENTS.md index af2fc746..ada49cdd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ dotnet test OpenNest.Server.Tests/OpenNest.Server.Tests.csproj -c Release dotnet test OpenNest.WinForms.Tests/OpenNest.WinForms.Tests.csproj ``` -Keep desktop-dependent tests in `OpenNest.WinForms.Tests`, never add a WinForms reference to `OpenNest.Tests`. Optional CHR fixtures use local `OpenNest.Tests/test-config.json` and skip when absent. On Linux, build Windows projects with `-p:EnableWindowsTargeting=true`; this is not Windows runtime verification. The headless console builds independently with `dotnet build OpenNest.Console/OpenNest.Console.csproj`. +After changing planning, contour preparation or geometry checks, run the [problem-nest corpus](docs/problem-nests.md) (`Category=ProblemNests`, opt-in via `OPENNEST_PROBLEM_NESTS_SERVER`); no plate may rank below its baseline. Keep desktop-dependent tests in `OpenNest.WinForms.Tests`, never add a WinForms reference to `OpenNest.Tests`. Optional CHR fixtures use local `OpenNest.Tests/test-config.json` and skip when absent. On Linux, build Windows projects with `-p:EnableWindowsTargeting=true`; this is not Windows runtime verification. The headless console builds independently with `dotnet build OpenNest.Console/OpenNest.Console.csproj`. Releases: follow [the release procedure](docs/releasing.md) and `scripts/Publish-Windows.ps1`; workflow artifacts are candidates, not published releases. GitHub (`ajisaacs/OpenNest`) is the primary repository; Gitea is a read-only backup mirror. diff --git a/OpenNest.Tests/CuttingPlanning/ProblemNestRegressionTests.cs b/OpenNest.Tests/CuttingPlanning/ProblemNestRegressionTests.cs new file mode 100644 index 00000000..4fb7ac39 --- /dev/null +++ b/OpenNest.Tests/CuttingPlanning/ProblemNestRegressionTests.cs @@ -0,0 +1,160 @@ +using System.Diagnostics; +using System.Text.RegularExpressions; +using OpenNest.CNC.CuttingStrategy; +using OpenNest.Data; +using OpenNest.Engine.CuttingPlanning; +using OpenNest.IO; +using Xunit.Abstractions; + +namespace OpenNest.Tests.CuttingPlanning; + +/// +/// Plans every nest on a dedicated problem-nest OpenNest server and fails when a plate's outcome is +/// worse than its recorded baseline. Opt-in and private: the nests and their baselines stay on the +/// server, never in the repository. See docs/problem-nests.md. +/// +[Trait("Category", "ProblemNests")] +public partial class ProblemNestRegressionTests(ITestOutputHelper output) +{ + internal const string ServerVariable = "OPENNEST_PROBLEM_NESTS_SERVER"; + internal const string TimeoutVariable = "OPENNEST_PROBLEM_NESTS_TIMEOUT"; + internal const string Timeout = "Timeout"; + // A corpus is a handful of nests; more means the variable names a production archive. + internal const int MaximumNests = 200; + + [SkippableFact] + public async Task ProblemNests_PlanNoWorseThanTheirBaselines() + { + var server = Environment.GetEnvironmentVariable(ServerVariable); + Skip.If(string.IsNullOrWhiteSpace(server), $"Set {ServerVariable} to the OpenNest server URL to plan its problem nests."); + using var repository = new RemoteNestRepository(server!); + var records = await ProblemNests(repository); + var limit = TimeSpan.FromSeconds(int.TryParse(Environment.GetEnvironmentVariable(TimeoutVariable), out var seconds) + && seconds > 0 ? seconds : 120); + Assert.True(records.Count > 0, $"No nests on {server}."); + + var failures = new List(); + foreach (var record in records) + { + var bytes = await repository.GetFileAsync(record.Id) + ?? throw new InvalidOperationException($"'{record.Name}' has no file."); + using var stream = new MemoryStream(bytes); + var nest = new NestReader(stream).Read(); + var baseline = Baseline(record.Comments); + var observed = new SortedDictionary(); + for (var index = 0; index < nest.Plates.Count; index++) + { + var plate = nest.Plates[index]; + if (plate.Parts.Count == 0) + continue; + var number = index + 1; + var watch = Stopwatch.StartNew(); + var (outcome, planned) = Plan(plate, number, limit); + observed[number] = outcome; + var expected = baseline.GetValueOrDefault(number); + output.WriteLine($"{record.Name} plate {number}: {outcome} (baseline {expected ?? "none"}), " + + $"{planned?.Result.Expansions ?? 0} expansions, {watch.Elapsed.TotalSeconds:0.0} s"); + if (expected == null) + continue; + if (Rank(outcome) < Rank(expected)) + failures.Add($"{record.Name} plate {number}: {outcome}, worse than baseline {expected}. " + + string.Join(" | ", planned?.Result.Findings.Take(3).Select(f => f.Message) ?? [])); + else if (outcome != expected) + output.WriteLine($" changed from {expected}; update the baseline if this is intended."); + } + var line = "planning: " + string.Join(" ", observed.Select(p => $"{p.Key}={p.Value}")); + if (baseline.Count == 0) + failures.Add($"{record.Name} has no baseline. Add this line to its comments: {line}"); + else if (baseline.Keys.Except(observed.Keys).Any() || observed.Keys.Except(baseline.Keys).Any()) + failures.Add($"{record.Name}: baseline plates do not match the nest's plates. Observed: {line}"); + } + Assert.True(failures.Count == 0, string.Join(Environment.NewLine, failures)); + } + + // Capture and planning share the limit; a plate that never finishes must not hang the run. + private static (string Outcome, CuttingPlanPlateResult Planned) Plan(Plate plate, int number, TimeSpan limit) + { + using var cancellation = new CancellationTokenSource(limit); + try + { + var planned = CuttingPlanBatch.Capture([plate], plate.CuttingParameters ?? DefaultParameters(), + preservePartOrder: false, [number], cancellation.Token).Plan(token: cancellation.Token).Plates[0]; + return (planned.Result.Status == CuttingPlanStatus.Cancelled ? Timeout : Outcome(planned), planned); + } + catch (OperationCanceledException) + { + return (Timeout, null); + } + } + + internal static async Task> ProblemNests(RemoteNestRepository repository) + { + var page = await repository.QueryAsync(new NestQuery + { + Sort = NestSortField.Name, + Descending = false, + Limit = MaximumNests, + }); + if (page.Total > MaximumNests) + throw new InvalidOperationException($"The server holds {page.Total} nests, more than a problem-nest " + + $"corpus of at most {MaximumNests}. Point {ServerVariable} at the dedicated corpus server."); + return page.Items; + } + + /// Plate number to outcome, from a "planning: 1=Ready 2=BestEffort" comment line. + internal static Dictionary Baseline(string comments) + { + var line = (comments ?? "").Split('\n').Select(l => l.Trim()) + .FirstOrDefault(l => l.StartsWith("planning:", StringComparison.OrdinalIgnoreCase)); + return line == null ? [] + : BaselineEntry().Matches(line).ToDictionary(m => int.Parse(m.Groups[1].Value), m => m.Groups[2].Value); + } + + // Ready means replayed and overlap-clear; a ready route with overlapping parts is still blocked. + internal static string Outcome(CuttingPlanPlateResult planned) => + planned.IsReady ? "Ready" : planned.IsRouteReady ? "Overlap" : planned.Result.Status.ToString(); + + internal static int Rank(string outcome) => outcome switch + { + "Ready" => 3, + nameof(CuttingPlanStatus.BestEffort) => 2, + nameof(CuttingPlanStatus.NoSolutionWithinBudget) => 1, + Timeout => -1, + _ => 0, + }; + + // A plate without its own settings uses the lead-ins problem nests were reported with. + internal static CuttingParameters DefaultParameters() + { + var parameters = new CuttingParameters(); + parameters.ExternalLeadIn = new LineLeadIn { Length = 0.15, ApproachAngle = 90 }; + parameters.InternalLeadIn = new LineLeadIn { Length = 0.15, ApproachAngle = 90 }; + parameters.ArcCircleLeadIn = new LineLeadIn { Length = 0.15, ApproachAngle = 90 }; + return parameters; + } + + [GeneratedRegex(@"(\d+)\s*=\s*(\w+)")] + private static partial Regex BaselineEntry(); +} + +public class ProblemNestBaselineTests +{ + [Fact] + public void Baseline_ReadsThePlanningLineAmongOtherComments() + { + var baseline = ProblemNestRegressionTests.Baseline( + "Plan blocked on unsupported geometry.\r\nplanning: 1=NoSolutionWithinBudget 2 = BestEffort\nnotes"); + + Assert.Equal(new Dictionary { [1] = "NoSolutionWithinBudget", [2] = "BestEffort" }, baseline); + Assert.Empty(ProblemNestRegressionTests.Baseline("no baseline yet")); + } + + [Fact] + public void Rank_OrdersReadyAboveBestEffortAboveBudgetAboveBlockedAboveTimeout() + { + var ranks = new[] { "Ready", "BestEffort", "NoSolutionWithinBudget", "UnsupportedGeometry", "Overlap", "Timeout" } + .Select(ProblemNestRegressionTests.Rank).ToArray(); + + Assert.Equal([3, 2, 1, 0, 0, -1], ranks); + } +} diff --git a/docs/problem-nests.md b/docs/problem-nests.md new file mode 100644 index 00000000..ffe5f135 --- /dev/null +++ b/docs/problem-nests.md @@ -0,0 +1,47 @@ +# Problem-nest regression corpus + +Real nests that once failed to plan are kept on a dedicated OpenNest server that holds nothing +else, not in this repository: they carry customer geometry and names. After a change to planning, contour preparation or +the geometry checks, plan them all and confirm that no plate got worse. + +## Running + +```sh +# Windows PowerShell: $env:OPENNEST_PROBLEM_NESTS_SERVER = "http://:" +OPENNEST_PROBLEM_NESTS_SERVER=http://: \ + dotnet test OpenNest.Tests/OpenNest.Tests.csproj -c Release \ + --filter "Category=ProblemNests" --logger "console;verbosity=detailed" +``` + +Without the variable the test skips, so routine runs never reach the network. With it, an +unreachable server or an empty corpus fails instead of passing silently. +`ProblemNestRegressionTests` downloads every nest on the server. A server holding more than 200 +nests is refused, since it is not a corpus. It plans each plate that has parts on its own, the way Plan Cutting +does (whole-part reorder, falling back to the current order). It prints each plate's outcome, +search expansions and time. Each plate has a time limit covering both capture and planning: +120 s by default, or `OPENNEST_PROBLEM_NESTS_TIMEOUT` seconds. A plate that hits it is recorded +as `Timeout`, so a plate that never finishes cannot hang the run. + +Each plate is ranked: `Ready` (replayed with no overlaps) above `BestEffort` above +`NoSolutionWithinBudget` above every blocked outcome (`UnsupportedGeometry`, +`ConstraintConflict`, `InvalidInput`, `Overlap`) above `Timeout`. The test fails when a plate ranks below its +baseline, when a nest has no baseline, or when the baseline's plate numbers differ from the +nest's. A different outcome at the same or a better rank is only reported. When it is +intended, update the baseline. + +Plates use their own cutting parameters. A plate without them uses line lead-ins of 0.15 at +90 degrees (external, internal and arc/circle) and no lead-outs. + +## Adding a nest + +1. In the desktop app, point **File > Storage Mode...** at the corpus server, then drag the + `.nest` file onto the window. Switch the server back afterwards. +2. In **Nest > Nest Info**, name it after the problem, and in the comments describe what failed + and which commit fixed it, if any. +3. Add a baseline line to the comments, with plate numbers counted from the first plate: + `planning: 1=NoSolutionWithinBudget 2=BestEffort`. + To get the line, save without one and run the test: the failure prints the observed line. +4. Save. Every save keeps the earlier versions, so a baseline change can be reviewed later. + +Record the outcome the code produces today, not the outcome you want. A nest that is still +blocked guards against getting worse, and its baseline moves up once a fix lands.