using System.Diagnostics; using System.Text.RegularExpressions; using OpenNest.CNC.CuttingStrategy; using OpenNest.Data; using OpenNest.Engine.CuttingPlanning; using OpenNest.Engine.Sequencing; using OpenNest.IO; using Xunit.Abstractions; namespace OpenNest.Tests.CuttingPlanning; /// /// Plans every nest on a dedicated problem-nest OpenNest server and fails when a plate's outcome is /// worse than its recorded baseline. A nest with a cut-off baseline is planned a second time with /// its applied lead-ins and cut-offs cleared and default scrap cut-offs added. A "checks:" comment /// line names the planner areas a nest guards; a run limited to some areas skips nests that guard /// none of them, and a nest without the line is always planned. A run touching the part order or /// entries plans every nest, since those change the search on every plate. Each plate's share of /// its expansion budget is reported, and plates near it are listed last: they are the ones a change /// to the search order tips into . Opt-in and private: /// the nests and their baselines stay on the server, never in the repository. See docs/problem-nests.md. /// [Trait("Category", "ProblemNests")] public partial class ProblemNestRegressionTests(ITestOutputHelper output) { internal const string ServerVariable = "OPENNEST_PROBLEM_NESTS_SERVER"; internal const string TimeoutVariable = "OPENNEST_PROBLEM_NESTS_TIMEOUT"; internal const string CutOffsVariable = "OPENNEST_PROBLEM_NESTS_CUTOFFS"; internal const string ChecksVariable = "OPENNEST_PROBLEM_NESTS_CHECKS"; internal const string Timeout = "Timeout"; internal const string PlanningLine = "planning:"; internal const string CutOffPlanningLine = "planning with cut-offs:"; internal const string ChecksLine = "checks:"; internal const string AutoNestLine = "auto-nest:"; /// The area of a nest whose problem was Auto Nest's layout, not its cutting plan. internal const string AutoNestArea = "auto-nest"; /// The planner areas a nest can guard; docs/problem-nests.md says which changes touch each. internal static readonly string[] PlannerAreas = ["part-order", "entries", "contours", "geometry-checks", "cut-offs", "grid"]; /// Every area a nest can guard: the planner's and Auto Nest's. internal static readonly string[] Areas = [.. PlannerAreas, AutoNestArea]; /// /// Areas whose changes reorder the search on every plate, not just the nests tagged with them: /// a run naming one plans the whole corpus. /// internal static readonly string[] WholeCorpusAreas = ["part-order", "entries"]; /// A plate whose strict search spent at least this share of its budget is near it. internal const double NearBudget = 0.8; // A corpus is a handful of nests; more means the variable names a production archive. internal const int MaximumNests = 200; [SkippableFact] public async Task ProblemNests_PlanNoWorseThanTheirBaselines() { var server = Environment.GetEnvironmentVariable(ServerVariable); Skip.If(string.IsNullOrWhiteSpace(server), $"Set {ServerVariable} to the OpenNest server URL to plan its problem nests."); using var repository = new RemoteNestRepository(server!); var records = await ProblemNests(repository); var limit = TimeSpan.FromSeconds(int.TryParse(Environment.GetEnvironmentVariable(TimeoutVariable), out var seconds) && seconds > 0 ? seconds : 120); Assert.True(records.Count > 0, $"No nests on {server}."); var named = Checks(Environment.GetEnvironmentVariable(ChecksVariable)); Skip.If(named.Length != 0 && !named.Intersect(PlannerAreas).Any(), $"{string.Join(", ", named)} names no planner area, so no nest is planned."); var wanted = Wanted(named); if (wanted.Length != named.Length) output.WriteLine($"{string.Join(", ", named.Intersect(WholeCorpusAreas))} change the search on every plate, " + "so every nest is planned."); var failures = new List(); var usage = new List<(string Plate, double Share)>(); foreach (var record in records) { var guards = Checks(CommentLine(record.Comments, ChecksLine)); if (!Selected(guards, wanted) || (guards.Length != 0 && !guards.Intersect(PlannerAreas).Any())) { output.WriteLine($"{record.Name}: skipped, checks {string.Join(", ", guards)}"); continue; } if (guards.Length == 0) output.WriteLine($"{record.Name}: no {ChecksLine} line, so always planned; add one naming the areas it guards."); var bytes = await repository.GetFileAsync(record.Id) ?? throw new InvalidOperationException($"'{record.Name}' has no file."); var baseline = Baseline(record.Comments); var observed = PlanAll(record.Name, bytes, false, baseline, limit, failures, usage); if (baseline.Count == 0) failures.Add($"{record.Name} has no baseline. Add this line to its comments: {Line(PlanningLine, observed)}"); if (Environment.GetEnvironmentVariable(CutOffsVariable) == "1") PlanAll(record.Name, bytes, true, [], limit, failures, usage); else if (Baseline(record.Comments, CutOffPlanningLine) is { Count: > 0 } withCutOffs) PlanAll(record.Name, bytes, true, withCutOffs, limit, failures, usage); } ReportNearBudget(usage); Assert.True(failures.Count == 0, string.Join(Environment.NewLine, failures)); } /// /// Runs Auto Nest (the Default engine, the nest's own sheet sizes, spacing and clamps, as the /// benchmark builds the job) on every nest that guards , and fails /// when a layout is invalid, leaves parts unplaced or needs more sheets than its /// "auto-nest: 2 sheets" baseline. /// [SkippableFact] public async Task ProblemNests_AutoNestNoWorseThanTheirBaselines() { var server = Environment.GetEnvironmentVariable(ServerVariable); Skip.If(string.IsNullOrWhiteSpace(server), $"Set {ServerVariable} to the OpenNest server URL to nest its problem nests."); var named = Checks(Environment.GetEnvironmentVariable(ChecksVariable)); Skip.If(named.Length != 0 && !named.Contains(AutoNestArea), $"{string.Join(", ", named)} does not name {AutoNestArea}."); using var repository = new RemoteNestRepository(server!); var records = await ProblemNests(repository); var limit = TimeSpan.FromSeconds(int.TryParse(Environment.GetEnvironmentVariable(TimeoutVariable), out var seconds) && seconds > 0 ? seconds : 120); var failures = new List(); var nested = 0; foreach (var record in records) { if (!Checks(CommentLine(record.Comments, ChecksLine)).Contains(AutoNestArea)) continue; nested++; var bytes = await repository.GetFileAsync(record.Id) ?? throw new InvalidOperationException($"'{record.Name}' has no file."); var baseline = SheetBaseline(record.Comments); var watch = Stopwatch.StartNew(); var (sheets, problem) = AutoNest(bytes, limit); output.WriteLine($"{record.Name}: {(problem ?? $"{sheets} sheets")} (baseline {baseline?.ToString() ?? "none"}), " + $"{watch.Elapsed.TotalSeconds:0.0} s"); if (problem != null) failures.Add($"{record.Name}: {problem}"); else if (baseline == null) failures.Add($"{record.Name} has no Auto Nest baseline. Add this line to its comments: {AutoNestLine} {sheets} sheets"); else if (sheets > baseline) failures.Add($"{record.Name}: {sheets} sheets, worse than baseline {baseline}."); else if (sheets < baseline) output.WriteLine($" better than {baseline}; update the baseline."); } Skip.If(nested == 0, $"No nest on {server} checks {AutoNestArea}."); Assert.True(failures.Count == 0, string.Join(Environment.NewLine, failures)); } // Auto Nest on the nest's demand: sheet count, or why the layout cannot be kept. private static (int Sheets, string Problem) AutoNest(byte[] bytes, TimeSpan limit) { var path = Path.Combine(Path.GetTempPath(), $"opennest-problem-{Guid.NewGuid():N}.nest"); try { File.WriteAllBytes(path, bytes); var job = OpenNest.Benchmark.JobLoader.Load(path).Single().BuildNestJob(maxPlates: 100); using var cancellation = new CancellationTokenSource(limit); var result = OpenNest.Engine.Jobs.NestingEngineRegistry.Create("Default").Solve(job, token: cancellation.Token); var violations = OpenNest.Engine.Jobs.NestLayoutCheck.Violations(job, result); if (violations.Count != 0) return (result.Plates.Count, "invalid layout: " + string.Join(" | ", violations.Take(3))); var unplaced = result.Fulfillment.Sum(f => f.Unplaced); return unplaced != 0 ? (result.Plates.Count, $"{unplaced} parts unplaced") : (result.Plates.Count, null); } catch (OperationCanceledException) { return (0, Timeout); } finally { File.Delete(path); } } /// The sheet count of an "auto-nest: 2 sheets" comment line; null when there is none. internal static int? SheetBaseline(string comments) => CommentLine(comments, AutoNestLine) is { } line && Regex.Match(line, @"\d+") is { Success: true } count ? int.Parse(count.Value) : null; // Plates near their budget pass today but are the first a reordered search pushes over it. private void ReportNearBudget(List<(string Plate, double Share)> usage) { var near = usage.Where(u => u.Share >= NearBudget).OrderByDescending(u => u.Share).ToList(); output.WriteLine(near.Count == 0 ? $"No plate's search used {NearBudget:P0} of its budget." : $"Near the budget ({near.Count}; a change that adds plates here or raises their share is a regression):"); foreach (var (plate, share) in near) output.WriteLine($" {plate}: {share:P0}"); } // Plans each plate with parts and compares it with the baseline; plate number to outcome. private SortedDictionary PlanAll(string name, byte[] bytes, bool cutOffs, Dictionary baseline, TimeSpan limit, List failures, List<(string Plate, double Share)> usage) { using var stream = new MemoryStream(bytes); var nest = new NestReader(stream).Read(); var label = cutOffs ? $"{name} with cut-offs" : name; var observed = new SortedDictionary(); for (var index = 0; index < nest.Plates.Count; index++) { var plate = nest.Plates[index]; if (plate.Parts.Count == 0) continue; var number = index + 1; var watch = Stopwatch.StartNew(); if (cutOffs) AddScrapCutOffs(plate, number, nest.Units); var budget = CuttingPlanBatch.PlateBudget(plate.Parts.Count); var (outcome, planned) = Plan(plate, number, limit); observed[number] = outcome; var expected = baseline.GetValueOrDefault(number); var share = BudgetShare(planned, budget); if (share is { } used) usage.Add(($"{label} plate {number}", used)); output.WriteLine($"{label} plate {number}: {outcome} (baseline {expected ?? "none"}), " + $"{planned?.Result.Expansions ?? 0} of {budget} expansions" + (share is { } shown ? $" ({shown:P0})" : "") + (planned?.KeptCurrentOrder == true ? ", new order ran out" : "") + $", {watch.Elapsed.TotalSeconds:0.0} s"); if (expected == null) continue; if (Rank(outcome) < Rank(expected)) failures.Add($"{label} plate {number}: {outcome}, worse than baseline {expected}. " + string.Join(" | ", planned?.Result.Findings.Take(3).Select(f => f.Message) ?? [])); else if (outcome != expected) output.WriteLine($" changed from {expected}; update the baseline if this is intended."); } if (baseline.Count != 0 && (baseline.Keys.Except(observed.Keys).Any() || observed.Keys.Except(baseline.Keys).Any())) failures.Add($"{label}: baseline plates do not match the nest's plates. Observed: " + Line(cutOffs ? CutOffPlanningLine : PlanningLine, observed)); return observed; } private static string Line(string prefix, SortedDictionary observed) => prefix + " " + string.Join(" ", observed.Select(p => $"{p.Key}={p.Value}")); // As Plan Cutting after "Clear Lead-ins & Cut-offs", with "Add scrap cut-offs" at its defaults. private static void AddScrapCutOffs(Plate plate, int number, Units units) { AppliedCuttingWork.Clear([plate], [number]); var spacing = units == Units.Millimeters ? 889 : 35; var staging = AutomaticCutOffBatch.Stage([plate], new AutomaticCutOffOptions { Spacing = spacing, MinimumStripLength = spacing / 2.0, MinimumTailLength = units == Units.Millimeters ? 304.8 : 12, KeepHorizontalRemnant = true, // As both desktop entry points set it. }, new CutOffSettings()); Assert.False(staging.IsBlocked, $"Plate {number}'s scrap cut-offs are blocked."); PlateSequencing.PlaceCutOffs(plate, staging.Plans[0].Definitions); } // Capture and planning share the limit; a plate that never finishes must not hang the run. private static (string Outcome, CuttingPlanPlateResult Planned) Plan(Plate plate, int number, TimeSpan limit) { using var cancellation = new CancellationTokenSource(limit); try { var planned = CuttingPlanBatch.Capture([plate], plate.CuttingParameters ?? DefaultParameters(), preservePartOrder: false, [number], cancellation.Token).Plan(token: cancellation.Token).Plates[0]; return (planned.Result.Status == CuttingPlanStatus.Canceled ? Timeout : Outcome(planned), planned); } catch (OperationCanceledException) { return (Timeout, null); } } internal static async Task> ProblemNests(RemoteNestRepository repository) { var page = await repository.QueryAsync(new NestQuery { Sort = NestSortField.Name, Descending = false, Limit = MaximumNests, }); if (page.Total > MaximumNests) throw new InvalidOperationException($"The server holds {page.Total} nests, more than a problem-nest " + $"corpus of at most {MaximumNests}. Point {ServerVariable} at the dedicated corpus server."); return page.Items; } /// /// Plate number to outcome, from a "planning: 1=Ready 2=BestEffort" comment line, or from the /// line starting with . /// internal static Dictionary Baseline(string comments, string prefix = PlanningLine) => CommentLine(comments, prefix) is not { } line ? [] : BaselineEntry().Matches(line).ToDictionary(m => int.Parse(m.Groups[1].Value), m => m.Groups[2].Value); // The rest of the first comment line starting with the prefix; null when there is none. private static string CommentLine(string comments, string prefix) => (comments ?? "").Split('\n').Select(l => l.Trim()) .FirstOrDefault(l => l.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))?[prefix.Length..]; /// /// The areas in a comma- or space-separated list, such as "part-order, entries". An area not in /// throws, so a misspelt line or filter cannot quietly skip a nest. /// internal static string[] Checks(string list) { var areas = (list ?? "").Split([',', ' '], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Select(a => a.ToLowerInvariant()).Distinct().ToArray(); var unknown = areas.Except(Areas).ToArray(); if (unknown.Length != 0) throw new ArgumentException( $"Unknown problem-nest area {string.Join(", ", unknown)}; the areas are {string.Join(", ", Areas)}."); return areas; } /// The areas a run plans: none, meaning every nest, when one changes the search on every plate. internal static string[] Wanted(string[] named) => named.Intersect(WholeCorpusAreas).Any() ? [] : named; // Every nest when no area is wanted; otherwise one guarding a wanted area, or one naming none. internal static bool Selected(IReadOnlyCollection guards, IReadOnlyCollection wanted) => wanted.Count == 0 || guards.Count == 0 || guards.Intersect(wanted).Any(); /// /// The share of the plate's budget its applied strict search spent; null when no strict search /// found the route (a refusal, the safest-route fallback or the best-effort plan), since then /// there is no margin left to measure. /// internal static double? BudgetShare(CuttingPlanPlateResult planned, int budget) => planned?.Result is { IsSafestRouteFallback: false, Expansions: > 0 } result && budget > 0 && result.Status is CuttingPlanStatus.Ready or CuttingPlanStatus.BestEffort ? (double)result.Expansions / budget : null; // Ready means replayed and overlap-clear; a ready route with overlapping parts is still blocked. internal static string Outcome(CuttingPlanPlateResult planned) => planned.IsReady ? "Ready" : planned.IsRouteReady ? "Overlap" : planned.Result.Status.ToString(); internal static int Rank(string outcome) => outcome switch { "Ready" => 3, nameof(CuttingPlanStatus.BestEffort) => 2, nameof(CuttingPlanStatus.NoSolutionWithinBudget) => 1, Timeout => -1, _ => 0, }; // A plate without its own settings uses the lead-ins problem nests were reported with. internal static CuttingParameters DefaultParameters() { var parameters = new CuttingParameters(); parameters.ExternalLeadIn = new LineLeadIn { Length = 0.15, ApproachAngle = 90 }; parameters.InternalLeadIn = new LineLeadIn { Length = 0.15, ApproachAngle = 90 }; return parameters; } [GeneratedRegex(@"(\d+)\s*=\s*(\w+)")] private static partial Regex BaselineEntry(); } public class ProblemNestBaselineTests { [Fact] public void Baseline_ReadsThePlanningLineAmongOtherComments() { var baseline = ProblemNestRegressionTests.Baseline( "Plan blocked on unsupported geometry.\r\nplanning: 1=NoSolutionWithinBudget 2 = BestEffort\nnotes"); Assert.Equal(new Dictionary { [1] = "NoSolutionWithinBudget", [2] = "BestEffort" }, baseline); Assert.Empty(ProblemNestRegressionTests.Baseline("no baseline yet")); } [Fact] public void Baseline_ReadsTheCutOffLineApartFromThePlanningLine() { const string comments = "planning: 1=Ready 2=BestEffort\nplanning with cut-offs: 1=Ready 2=Ready"; Assert.Equal(new Dictionary { [1] = "Ready", [2] = "BestEffort" }, ProblemNestRegressionTests.Baseline(comments)); Assert.Equal(new Dictionary { [1] = "Ready", [2] = "Ready" }, ProblemNestRegressionTests.Baseline(comments, ProblemNestRegressionTests.CutOffPlanningLine)); } [Fact] public void Checks_SelectNestsGuardingAWantedAreaAndThoseNamingNone() { var guards = ProblemNestRegressionTests.Checks(" Part-Order, entries "); Assert.Equal(["part-order", "entries"], guards); Assert.True(ProblemNestRegressionTests.Selected(guards, [])); Assert.True(ProblemNestRegressionTests.Selected(guards, ["entries", "grid"])); Assert.False(ProblemNestRegressionTests.Selected(guards, ["grid"])); Assert.True(ProblemNestRegressionTests.Selected([], ["grid"])); Assert.Throws(() => ProblemNestRegressionTests.Checks("part-order, lead-ins")); } [Fact] public void SheetBaseline_ReadsTheAutoNestLine() { Assert.Equal(2, ProblemNestRegressionTests.SheetBaseline("Three sheets before the fix.\nchecks: auto-nest\nauto-nest: 2 sheets")); Assert.Null(ProblemNestRegressionTests.SheetBaseline("planning: 1=Ready")); Assert.Equal(["auto-nest"], ProblemNestRegressionTests.Checks("auto-nest")); } [Fact] public void Wanted_PlansEveryNestForAreasThatReorderEveryPlate() { Assert.Equal(["cut-offs", "grid"], ProblemNestRegressionTests.Wanted(["cut-offs", "grid"])); Assert.Empty(ProblemNestRegressionTests.Wanted(["cut-offs", "part-order"])); Assert.Empty(ProblemNestRegressionTests.Wanted(["entries"])); } [Fact] public void Rank_OrdersReadyAboveBestEffortAboveBudgetAboveBlockedAboveTimeout() { var ranks = new[] { "Ready", "BestEffort", "NoSolutionWithinBudget", "UnsupportedGeometry", "Overlap", "Timeout" } .Select(ProblemNestRegressionTests.Rank).ToArray(); Assert.Equal([3, 2, 1, 0, 0, -1], ranks); } }