aj/OpenNest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(benchmark): rank by sheet cost so engines can't game the score

Browse Source 
The benchmark is about to be used as the objective for LLM-designed
engines, and several gaps would have rewarded the wrong behavior:

- Ranking was utilization-first, so dropping awkward parts raised the
  score. Rank valid > fully placed > cost > plates, where cost is
  salvage-credited sheet area plus a largest-sheet penalty per unplaced
  part; placing a part is never scored worse than omitting it.
- Salvage rate was ignored in scoring; cost now uses EstimateNetArea,
  recomputed from job geometry rather than trusted from the engine.
- Rotation constraints were never validated. Add RotationPolicy.Allows
  (shared with NestJobPlacementValidator) and check every placement.
- Returned sheets were trusted, so an engine could loosen spacing or
  invent a size. Sheets must now match offered stock.
- Part-in-part placements were flagged as overlaps; spacing now accounts
  for cutouts, with an X-sorted sweep to prune distant pairs.
- Summary averaged per-job percentages; it now sums areas and cost.
- --spacing and sheet sizes parsed with the current culture.
- Warn when .nest jobs offer only their original sheet sizes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aj
2026-09-22 15:05:43 -04:00
co-authored by Claude Opus 5.5
parent 98c4929d14
commit 57e9f625b6
13 changed files with 669 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
OpenNest.Benchmark/Report.cs
+39 -19
View File
@@ -9,10 +9,12 @@ namespace OpenNest.Benchmark
{
/// <summary>
/// Console + CSV reporting for benchmark results. Ranking rule per job:
/// valid beats invalid; higher aggregate utilization wins; if utilization
/// ties and both engines fully placed every requested part, fewer plates
/// used wins (the multi-plate analogue of "smaller remnant" - both are
/// proxies for wasting less material). Ties beyond that are a shared win.
/// valid beats invalid; placing every requested part beats not; then lower
/// JobResult.Cost wins - salvage-credited sheet area consumed plus a
/// largest-sheet penalty per unplaced part, so dropping awkward parts can
/// never buy a better score; then fewer plates. Ties beyond that are a
/// shared win. Across jobs, costs and areas are summed (not averaged), so
/// a big job weighs more than a three-part one.
/// </summary>
public static class Report
{
@@ -29,7 +31,7 @@ namespace OpenNest.Benchmark
var best = ranked.Count > 0 ? ranked[0] : null;
Console.WriteLine(
$"{"Engine", -16} {"Result", -9} {"Parts", -10} {"Util%", -8} {"Plates", -18} {"Time(ms)", -9} Notes"
$"{"Engine", -16} {"Result", -9} {"Parts", -10} {"Util%", -7} {"Net%", -7} {"Cost", -12} {"Plates", -18} {"Time(ms)", -9} Notes"
);
foreach (var r in ranked)
@@ -42,12 +44,13 @@ namespace OpenNest.Benchmark
: "INVALID";
var partsCol = $"{r.PartsPlaced}/{r.PartsRequested}";
var utilCol = r.Valid ? $"{r.Utilization * 100:F1}" : "-";
var netCol = r.Valid ? $"{r.NetUtilization * 100:F1}" : "-";
var platesCol =
r.PlatesUsed > 0 ? $"{r.PlatesUsed} ({SizeSummary(r.SizeBreakdown)})" : "-";
var notes = r.Crashed ? r.Error : string.Join("; ", r.Violations.Take(2));
Console.WriteLine(
$"{marker}{r.EngineName, -15} {status, -9} {partsCol, -10} {utilCol, -8} {platesCol, -18} {r.ElapsedMs, -9} {notes}"
$"{marker}{r.EngineName, -15} {status, -9} {partsCol, -10} {utilCol, -7} {netCol, -7} {r.Cost, -12:F1} {platesCol, -18} {r.ElapsedMs, -9} {notes}"
);
}
}
@@ -67,25 +70,33 @@ namespace OpenNest.Benchmark
Valid = g.Count(r => r.Valid),
Crashed = g.Count(r => r.Crashed),
FullyPlaced = g.Count(r => r.FullyPlaced),
TotalUtilization = g.Sum(r => r.Utilization),
Unplaced = g.Sum(r => r.PartsUnplaced),
PlacedArea = g.Where(r => r.Valid).Sum(r => r.PlacedArea),
PlateArea = g.Where(r => r.Valid).Sum(r => r.PlateArea),
NetSheetArea = g.Where(r => r.Valid).Sum(r => r.NetSheetArea),
TotalCost = g.Sum(r => r.Cost),
TotalPlates = g.Sum(r => r.PlatesUsed),
TotalTimeMs = g.Sum(r => r.ElapsedMs),
})
.OrderByDescending(e => e.TotalUtilization)
.OrderBy(e => e.TotalCost)
.ToList();
var wins = CountWins(results);
// Util% and Net% are area-weighted over valid runs (sum placed / sum
// sheet), not a mean of per-job percentages. TotalCost sums across
// jobs, so it is only meaningful when every job uses the same units.
Console.WriteLine(
$"{"Engine", -16} {"Jobs", -6} {"Valid", -7} {"Complete", -9} {"Wins", -6} {"AvgUtil%", -10} {"Plates", -8} {"TotalTime(ms)", -14}"
$"{"Engine", -16} {"Jobs", -6} {"Valid", -7} {"Complete", -9} {"Unplaced", -9} {"Wins", -6} {"Util%", -7} {"Net%", -7} {"TotalCost", -14} {"Plates", -8} {"TotalTime(ms)", -14}"
);
foreach (var e in byEngine)
{
var avgUtil = e.Jobs > 0 ? e.TotalUtilization / e.Jobs * 100 : 0;
var util = e.PlateArea > 0 ? e.PlacedArea / e.PlateArea * 100 : 0;
var netUtil = e.NetSheetArea > 0 ? e.PlacedArea / e.NetSheetArea * 100 : 0;
var winCount = wins.TryGetValue(e.Engine, out var w) ? w : 0;
Console.WriteLine(
$"{e.Engine, -16} {e.Jobs, -6} {e.Valid, -7} {e.FullyPlaced, -9} {winCount, -6} {avgUtil, -10:F1} {e.TotalPlates, -8} {e.TotalTimeMs, -14}"
$"{e.Engine, -16} {e.Jobs, -6} {e.Valid, -7} {e.FullyPlaced, -9} {e.Unplaced, -9} {winCount, -6} {util, -7:F1} {netUtil, -7:F1} {e.TotalCost, -14:F1} {e.TotalPlates, -8} {e.TotalTimeMs, -14}"
);
}
}
@@ -94,7 +105,7 @@ namespace OpenNest.Benchmark
{
var sb = new StringBuilder();
sb.AppendLine(
"Job,Engine,Valid,Crashed,FullyPlaced,PartsPlaced,PartsRequested,Utilization,PlatesUsed,SizeBreakdown,ElapsedMs,Notes"
"Job,Engine,Valid,Crashed,FullyPlaced,PartsPlaced,PartsRequested,Utilization,NetUtilization,PlateArea,NetSheetArea,Cost,PlatesUsed,SizeBreakdown,ElapsedMs,Notes"
);
foreach (var r in results)
@@ -111,6 +122,10 @@ namespace OpenNest.Benchmark
r.PartsPlaced,
r.PartsRequested,
r.Utilization.ToString("F4", CultureInfo.InvariantCulture),
r.NetUtilization.ToString("F4", CultureInfo.InvariantCulture),
r.PlateArea.ToString("F2", CultureInfo.InvariantCulture),
r.NetSheetArea.ToString("F2", CultureInfo.InvariantCulture),
r.Cost.ToString("F2", CultureInfo.InvariantCulture),
r.PlatesUsed,
Csv(SizeSummary(r.SizeBreakdown)),
r.ElapsedMs,
@@ -159,9 +174,10 @@ namespace OpenNest.Benchmark
return wins;
}
/// <summary>Lower sorts first (better). Valid beats invalid, then higher
/// aggregate utilization, then (if both fully placed) fewer plates used.</summary>
private static int Compare(JobResult a, JobResult b)
/// <summary>Lower sorts first (better). Valid beats invalid, complete beats
/// incomplete, then lower cost (relative tolerance, since costs are areas
/// in whatever units the job uses), then fewer plates.</summary>
public static int Compare(JobResult a, JobResult b)
{
if (a.Valid != b.Valid)
return a.Valid ? -1 : 1;
@@ -169,12 +185,16 @@ namespace OpenNest.Benchmark
if (!a.Valid)
return 0;
var utilDiff = b.Utilization - a.Utilization;
if (a.FullyPlaced != b.FullyPlaced)
return a.FullyPlaced ? -1 : 1;
if (System.Math.Abs(utilDiff) > Epsilon)
return utilDiff > 0 ? 1 : -1;
var costDiff = a.Cost - b.Cost;
var scale = System.Math.Max(1, System.Math.Max(a.Cost, b.Cost));
if (a.FullyPlaced && b.FullyPlaced && a.PlatesUsed != b.PlatesUsed)
if (System.Math.Abs(costDiff) > Epsilon * scale)
return costDiff > 0 ? 1 : -1;
if (a.PlatesUsed != b.PlatesUsed)
return a.PlatesUsed > b.PlatesUsed ? 1 : -1;
return 0;