The benchmark is about to be used as the objective for LLM-designed engines, and several gaps would have rewarded the wrong behavior: - Ranking was utilization-first, so dropping awkward parts raised the score. Rank valid > fully placed > cost > plates, where cost is salvage-credited sheet area plus a largest-sheet penalty per unplaced part; placing a part is never scored worse than omitting it. - Salvage rate was ignored in scoring; cost now uses EstimateNetArea, recomputed from job geometry rather than trusted from the engine. - Rotation constraints were never validated. Add RotationPolicy.Allows (shared with NestJobPlacementValidator) and check every placement. - Returned sheets were trusted, so an engine could loosen spacing or invent a size. Sheets must now match offered stock. - Part-in-part placements were flagged as overlaps; spacing now accounts for cutouts, with an X-sorted sweep to prune distant pairs. - Summary averaged per-job percentages; it now sums areas and cost. - --spacing and sheet sizes parsed with the current culture. - Warn when .nest jobs offer only their original sheet sizes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
297 lines
10 KiB
C#
297 lines
10 KiB
C#
using OpenNest.Benchmark;
|
|
using OpenNest.CNC;
|
|
using OpenNest.Engine.Jobs;
|
|
using OpenNest.Geometry;
|
|
using OpenNest.Math;
|
|
|
|
namespace OpenNest.Tests.Benchmark;
|
|
|
|
public class BenchmarkScoringTests
|
|
{
|
|
// ── ranking ──────────────────────────────────────────────────────────────
|
|
|
|
[Fact]
|
|
public void Compare_CompleteRunBeatsIncompleteRunWithHigherUtilization()
|
|
{
|
|
var cherryPicked = Result(placed: 150, requested: 219, placedArea: 90, plateArea: 100);
|
|
var complete = Result(placed: 219, requested: 219, placedArea: 120, plateArea: 143);
|
|
|
|
Assert.True(cherryPicked.Utilization > complete.Utilization);
|
|
Assert.True(Report.Compare(complete, cherryPicked) < 0);
|
|
}
|
|
|
|
[Fact]
|
|
public void Compare_AmongIncompleteRuns_PlacingMorePartsWinsEvenOnMoreSheet()
|
|
{
|
|
var more = Result(placed: 9, requested: 10, placedArea: 90, plateArea: 200);
|
|
var fewer = Result(placed: 5, requested: 10, placedArea: 50, plateArea: 100);
|
|
|
|
Assert.True(Report.Compare(more, fewer) < 0);
|
|
}
|
|
|
|
[Fact]
|
|
public void Compare_CompleteRuns_SalvageCreditBreaksEqualSheetUsage()
|
|
{
|
|
var cleanRemnant = Result(10, 10, placedArea: 60, plateArea: 100, netSheetArea: 80);
|
|
var scattered = Result(10, 10, placedArea: 60, plateArea: 100, netSheetArea: 100);
|
|
|
|
Assert.True(Report.Compare(cleanRemnant, scattered) < 0);
|
|
}
|
|
|
|
[Fact]
|
|
public void Cost_InvalidRunPaysPenaltyOnEveryRequestedPart()
|
|
{
|
|
var invalid = new JobResult
|
|
{
|
|
Valid = false,
|
|
PartsPlaced = 10,
|
|
PartsRequested = 10,
|
|
UnplacedPartPenalty = 100,
|
|
Violations = { "overlap" },
|
|
};
|
|
|
|
Assert.Equal(1000, invalid.Cost);
|
|
}
|
|
|
|
// ── validation through the runner ────────────────────────────────────────
|
|
|
|
[Fact]
|
|
public void Run_RotationOutsideConstraint_IsInvalid()
|
|
{
|
|
var part = Rect(10, 5);
|
|
LockRotation(part);
|
|
var job = Job(0.5, (part, 1));
|
|
|
|
var rotated = RunSingle(job, j => Sheet(j, (j.Parts[0].Id, 20, 20, Angle.HalfPI)));
|
|
var upright = RunSingle(job, j => Sheet(j, (j.Parts[0].Id, 20, 20, 0)));
|
|
|
|
Assert.False(rotated.Valid);
|
|
Assert.Contains(rotated.Violations, v => v.Contains("rotation constraint"));
|
|
Assert.True(upright.Valid, string.Join("; ", upright.Violations));
|
|
}
|
|
|
|
[Fact]
|
|
public void Run_StockWithLoosenedSpacing_IsInvalid()
|
|
{
|
|
var job = Job(0.5, (Rect(10, 5), 1));
|
|
|
|
var result = RunSingle(
|
|
job,
|
|
j =>
|
|
{
|
|
var real = j.Plates[0];
|
|
var forged = new NestPlateStock(real.Id, real.Size, null, 0, real.EdgeSpacing, real.Quadrant);
|
|
return new[]
|
|
{
|
|
new NestJobPlateResult(0, forged, new[] { new NestJobPlacement(j.Parts[0].Id, 0, 1, 1, 0) }),
|
|
};
|
|
}
|
|
);
|
|
|
|
Assert.False(result.Valid);
|
|
Assert.Contains(result.Violations, v => v.Contains("does not match any stock"));
|
|
}
|
|
|
|
[Fact]
|
|
public void Run_PartInsideAnotherPartsCutout_IsValidWhenItClearsTheEdge()
|
|
{
|
|
var job = Job(0.5, (Frame(), 1), (Rect(4, 4), 1));
|
|
|
|
var clear = RunSingle(
|
|
job,
|
|
j => Sheet(j, (j.Parts[0].Id, 10, 10, 0), (j.Parts[1].Id, 18, 18, 0))
|
|
);
|
|
var tooClose = RunSingle(
|
|
job,
|
|
j => Sheet(j, (j.Parts[0].Id, 10, 10, 0), (j.Parts[1].Id, 15.2, 18, 0))
|
|
);
|
|
|
|
Assert.True(clear.Valid, string.Join("; ", clear.Violations));
|
|
Assert.True(clear.FullyPlaced);
|
|
Assert.False(tooClose.Valid);
|
|
Assert.Contains(tooClose.Violations, v => v.Contains("closer than the required spacing"));
|
|
}
|
|
|
|
[Fact]
|
|
public void Run_OverlappingNeighbours_AreStillCaughtBySweep()
|
|
{
|
|
var job = Job(0.5, (Rect(10, 5), 3));
|
|
|
|
var result = RunSingle(
|
|
job,
|
|
j => Sheet(
|
|
j,
|
|
(j.Parts[0].Id, 60, 1, 0),
|
|
(j.Parts[0].Id, 1, 1, 0),
|
|
(j.Parts[0].Id, 10.2, 1, 0)
|
|
)
|
|
);
|
|
|
|
Assert.False(result.Valid);
|
|
Assert.Single(result.Violations);
|
|
}
|
|
|
|
[Fact]
|
|
public void Run_WithSalvageCredit_CompactLayoutOutscoresScatteredLayout()
|
|
{
|
|
var job = Job(0.5, (Rect(10, 5), 2));
|
|
var engines = new List<NestingEngineInfo>
|
|
{
|
|
Engine("Compact", j => Sheet(j, (j.Parts[0].Id, 0, 0, 0), (j.Parts[0].Id, 0, 10, 0))),
|
|
Engine("Scattered", j => Sheet(j, (j.Parts[0].Id, 0, 0, 0), (j.Parts[0].Id, 86, 43, 0))),
|
|
};
|
|
|
|
var results = BenchmarkRunner.Run(
|
|
new List<BenchmarkJob> { job },
|
|
engines,
|
|
salvageRate: 0.5,
|
|
minimumSalvageDimension: 10
|
|
);
|
|
var compact = results.Single(r => r.EngineName == "Compact");
|
|
var scattered = results.Single(r => r.EngineName == "Scattered");
|
|
|
|
Assert.True(compact.FullyPlaced && scattered.FullyPlaced);
|
|
Assert.Equal(compact.PlateArea, scattered.PlateArea);
|
|
Assert.True(compact.NetSheetArea < scattered.NetSheetArea);
|
|
Assert.True(Report.Compare(compact, scattered) < 0);
|
|
}
|
|
|
|
[Fact]
|
|
public void TryParseSheetSize_UsesInvariantCulture()
|
|
{
|
|
var previous = Thread.CurrentThread.CurrentCulture;
|
|
Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("de-DE");
|
|
|
|
try
|
|
{
|
|
Assert.True(JobLoader.TryParseSheetSize("48.5x96", out var size));
|
|
Assert.Equal(48.5, size.Width);
|
|
Assert.Equal(96, size.Length);
|
|
}
|
|
finally
|
|
{
|
|
Thread.CurrentThread.CurrentCulture = previous;
|
|
}
|
|
}
|
|
|
|
// ── helpers ──────────────────────────────────────────────────────────────
|
|
|
|
private static JobResult Result(
|
|
int placed,
|
|
int requested,
|
|
double placedArea,
|
|
double plateArea,
|
|
double? netSheetArea = null
|
|
) =>
|
|
new()
|
|
{
|
|
Valid = true,
|
|
PartsPlaced = placed,
|
|
PartsRequested = requested,
|
|
PlacedArea = placedArea,
|
|
PlateArea = plateArea,
|
|
NetSheetArea = netSheetArea ?? plateArea,
|
|
UnplacedPartPenalty = 100,
|
|
PlatesUsed = 1,
|
|
};
|
|
|
|
private static Drawing Rect(double w, double h)
|
|
{
|
|
var pgm = new OpenNest.CNC.Program();
|
|
pgm.Codes.Add(new RapidMove(new Vector(0, 0)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(w, 0)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(w, h)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(0, h)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(0, 0)));
|
|
return new Drawing("rect", pgm);
|
|
}
|
|
|
|
/// <summary>20x20 square with a 10x10 cutout centred in it.</summary>
|
|
private static Drawing Frame()
|
|
{
|
|
var pgm = new OpenNest.CNC.Program();
|
|
pgm.Codes.Add(new RapidMove(new Vector(5, 5)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(5, 15)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(15, 15)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(15, 5)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(5, 5)));
|
|
pgm.Codes.Add(new RapidMove(new Vector(0, 0)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(20, 0)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(20, 20)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(0, 20)));
|
|
pgm.Codes.Add(new LinearMove(new Vector(0, 0)));
|
|
return new Drawing("frame", pgm);
|
|
}
|
|
|
|
private static void LockRotation(Drawing drawing)
|
|
{
|
|
drawing.Constraints ??= new NestConstraints();
|
|
drawing.Constraints.StepAngle = Angle.TwoPI;
|
|
drawing.Constraints.StartAngle = 0;
|
|
drawing.Constraints.EndAngle = 0;
|
|
}
|
|
|
|
private static BenchmarkJob Job(double spacing, params (Drawing Drawing, int Quantity)[] parts) =>
|
|
new()
|
|
{
|
|
SourceFile = "test.manifest.json",
|
|
CandidateSizes = new List<Size> { new(48, 96) },
|
|
EdgeSpacing = new Spacing(0, 0),
|
|
PartSpacing = spacing,
|
|
Quadrant = 1,
|
|
Requests = parts
|
|
.Select(p => new DrawingRequest { Drawing = p.Drawing, Quantity = p.Quantity })
|
|
.ToList(),
|
|
};
|
|
|
|
private static NestJobPlateResult[] Sheet(
|
|
NestJob job,
|
|
params (string PartId, double X, double Y, double Rotation)[] placements
|
|
)
|
|
{
|
|
var counts = new Dictionary<string, int>();
|
|
return new[]
|
|
{
|
|
new NestJobPlateResult(
|
|
0,
|
|
job.Plates[0],
|
|
placements.Select(p =>
|
|
{
|
|
var index = counts.GetValueOrDefault(p.PartId);
|
|
counts[p.PartId] = index + 1;
|
|
return new NestJobPlacement(p.PartId, index, p.X, p.Y, p.Rotation);
|
|
})
|
|
),
|
|
};
|
|
}
|
|
|
|
private static JobResult RunSingle(
|
|
BenchmarkJob job,
|
|
Func<NestJob, IEnumerable<NestJobPlateResult>> layout
|
|
) => Assert.Single(BenchmarkRunner.Run(new List<BenchmarkJob> { job }, new[] { Engine("Scripted", layout) }));
|
|
|
|
private static NestingEngineInfo Engine(
|
|
string name,
|
|
Func<NestJob, IEnumerable<NestJobPlateResult>> layout
|
|
) => new(name, "test double", () => new ScriptedEngine(layout));
|
|
|
|
/// <summary>Returns a fixed layout without going through NestJobRunner, the way a
|
|
/// hand-written engine could, so only the benchmark's own validator judges it.</summary>
|
|
private sealed class ScriptedEngine(Func<NestJob, IEnumerable<NestJobPlateResult>> layout)
|
|
: INestingEngine
|
|
{
|
|
public NestJobResult Solve(
|
|
NestJob job,
|
|
IProgress<NestJobProgress>? progress = null,
|
|
CancellationToken token = default
|
|
) =>
|
|
new(
|
|
NestJobStatus.Complete,
|
|
NestJobStopReason.Completed,
|
|
layout(job),
|
|
Array.Empty<PartFulfillment>(),
|
|
Array.Empty<StockUsage>()
|
|
);
|
|
}
|
|
}
|