fix(benchmark): rank by sheet cost so engines can't game the score
The benchmark is about to be used as the objective for LLM-designed engines, and several gaps would have rewarded the wrong behavior: - Ranking was utilization-first, so dropping awkward parts raised the score. Rank valid > fully placed > cost > plates, where cost is salvage-credited sheet area plus a largest-sheet penalty per unplaced part; placing a part is never scored worse than omitting it. - Salvage rate was ignored in scoring; cost now uses EstimateNetArea, recomputed from job geometry rather than trusted from the engine. - Rotation constraints were never validated. Add RotationPolicy.Allows (shared with NestJobPlacementValidator) and check every placement. - Returned sheets were trusted, so an engine could loosen spacing or invent a size. Sheets must now match offered stock. - Part-in-part placements were flagged as overlaps; spacing now accounts for cutouts, with an X-sorted sweep to prune distant pairs. - Summary averaged per-job percentages; it now sums areas and cost. - --spacing and sheet sizes parsed with the current culture. - Warn when .nest jobs offer only their original sheet sizes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -42,6 +42,12 @@ namespace OpenNest.Benchmark
|
||||
|
||||
public int TotalRequestedQuantity => Requests.Sum(r => r.Quantity);
|
||||
|
||||
/// <summary>Sheet area charged per unplaced part: the largest candidate
|
||||
/// sheet. Any single part that fits the stock at all fits on one such
|
||||
/// sheet, so placing a part is never scored worse than leaving it out.</summary>
|
||||
public double UnplacedPartPenalty =>
|
||||
CandidateSizes.Count == 0 ? 0 : CandidateSizes.Max(s => s.Width * s.Length);
|
||||
|
||||
/// <summary>
|
||||
/// Builds the whole-job request this job represents: one NestJobPart per
|
||||
/// requested drawing, and one NestPlateStock per candidate sheet size
|
||||
|
||||
@@ -100,11 +100,21 @@ namespace OpenNest.Benchmark
|
||||
);
|
||||
|
||||
var validation = NestValidator.Validate(plateRuns, requirements);
|
||||
NestValidator.ValidateAgainstJob(
|
||||
nestJob,
|
||||
jobResult,
|
||||
job.Requests.ToDictionary(r => r.Drawing.Id.ToString(), r => r.Drawing.Name),
|
||||
validation
|
||||
);
|
||||
var totalPlaced = plateRuns.Sum(pr => pr.Parts.Count);
|
||||
var placedArea = validation.Valid
|
||||
? plateRuns.Sum(pr => pr.Parts.Sum(p => p.BaseDrawing.Area))
|
||||
: 0;
|
||||
var plateArea = plateRuns.Sum(pr => pr.Plate.Area());
|
||||
// Salvage credit is recomputed from the job's own geometry, never taken from the engine.
|
||||
var netSheetArea = validation.Valid
|
||||
? jobResult.Plates.Sum(p => StockLadderNestingEngine.EstimateNetArea(nestJob, p))
|
||||
: 0;
|
||||
|
||||
var sizeBreakdown = plateRuns
|
||||
.GroupBy(pr => pr.Plate.Size.ToString(1))
|
||||
@@ -157,9 +167,7 @@ namespace OpenNest.Benchmark
|
||||
PlacedArea = placedArea,
|
||||
SalvageRate = salvageRate,
|
||||
MinimumSalvageDimension = minimumSalvageDimension,
|
||||
EstimatedNetArea = jobResult.Plates.Sum(p =>
|
||||
StockLadderNestingEngine.EstimateNetArea(nestJob, p)
|
||||
),
|
||||
EstimatedNetArea = netSheetArea,
|
||||
Fulfillment = jobResult.Fulfillment,
|
||||
StockUsage = jobResult.StockUsage,
|
||||
Plates = jobResult.Plates,
|
||||
@@ -185,6 +193,8 @@ namespace OpenNest.Benchmark
|
||||
PartsRequested = requested,
|
||||
PlacedArea = placedArea,
|
||||
PlateArea = plateArea,
|
||||
NetSheetArea = netSheetArea,
|
||||
UnplacedPartPenalty = job.UnplacedPartPenalty,
|
||||
PlatesUsed = plateRuns.Count,
|
||||
SizeBreakdown = sizeBreakdown,
|
||||
ElapsedMs = sw.ElapsedMilliseconds,
|
||||
@@ -199,6 +209,7 @@ namespace OpenNest.Benchmark
|
||||
JobName = job.Name,
|
||||
Valid = false,
|
||||
PartsRequested = requested,
|
||||
UnplacedPartPenalty = job.UnplacedPartPenalty,
|
||||
ElapsedMs = sw.ElapsedMilliseconds,
|
||||
Error = $"Timed out after {SolveTimeout.TotalMinutes:F0} minute(s)",
|
||||
};
|
||||
@@ -212,6 +223,7 @@ namespace OpenNest.Benchmark
|
||||
JobName = job.Name,
|
||||
Valid = false,
|
||||
PartsRequested = requested,
|
||||
UnplacedPartPenalty = job.UnplacedPartPenalty,
|
||||
ElapsedMs = sw.ElapsedMilliseconds,
|
||||
Error = $"{ex.GetType().Name}: {ex.Message}",
|
||||
};
|
||||
|
||||
@@ -86,7 +86,7 @@ namespace OpenNest.Benchmark
|
||||
|
||||
foreach (var text in manifest.SheetSizes ?? new List<string>())
|
||||
{
|
||||
if (!Size.TryParse(text, out var size))
|
||||
if (!JobLoader.TryParseSheetSize(text, out var size))
|
||||
throw new InvalidOperationException(
|
||||
$"Manifest '{manifestPath}': could not parse sheet size '{text}' (expected e.g. \"48x96\")."
|
||||
);
|
||||
|
||||
@@ -81,6 +81,29 @@ namespace OpenNest.Benchmark
|
||||
return jobs;
|
||||
}
|
||||
|
||||
/// <summary>Parses "WxL" with invariant-culture numbers, so "48.5x96" means the
|
||||
/// same thing on every machine (Size.Parse follows the current culture).</summary>
|
||||
public static bool TryParseSheetSize(string text, out Size size)
|
||||
{
|
||||
size = default;
|
||||
var dims = text?.Split('x', 'X');
|
||||
|
||||
if (dims == null || dims.Length != 2)
|
||||
return false;
|
||||
|
||||
var style = System.Globalization.NumberStyles.Float;
|
||||
var culture = System.Globalization.CultureInfo.InvariantCulture;
|
||||
|
||||
if (
|
||||
!double.TryParse(dims[0].Trim(), style, culture, out var width)
|
||||
|| !double.TryParse(dims[1].Trim(), style, culture, out var length)
|
||||
)
|
||||
return false;
|
||||
|
||||
size = new Size(width, length);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static List<string> ResolveFiles(string inputPath)
|
||||
{
|
||||
if (Directory.Exists(inputPath))
|
||||
|
||||
@@ -6,8 +6,9 @@ namespace OpenNest.Benchmark
|
||||
/// Outcome of running one engine against one job. A job may span several
|
||||
/// plates (PlatesUsed, SizeBreakdown), since the engine may need more than
|
||||
/// one sheet - possibly of different sizes - to place everything asked of
|
||||
/// it. An invalid or crashed run always scores zero utilization, per the
|
||||
/// benchmark rules.
|
||||
/// it. An invalid or crashed run places nothing as far as scoring is
|
||||
/// concerned: it earns no area and pays the unplaced penalty on every
|
||||
/// requested part.
|
||||
/// </summary>
|
||||
public class JobResult
|
||||
{
|
||||
@@ -20,6 +21,17 @@ namespace OpenNest.Benchmark
|
||||
public int PartsRequested { get; init; }
|
||||
public double PlacedArea { get; init; }
|
||||
public double PlateArea { get; init; }
|
||||
|
||||
/// <summary>Sheet area consumed after crediting salvageable offcuts
|
||||
/// (StockLadderNestingEngine.EstimateNetArea summed over every plate).
|
||||
/// Equals PlateArea when salvage credit is disabled.</summary>
|
||||
public double NetSheetArea { get; init; }
|
||||
|
||||
/// <summary>Sheet area charged for each requested part that was not
|
||||
/// placed: the largest candidate sheet's area, so leaving a part out
|
||||
/// always costs at least as much as the extra sheet it would need.</summary>
|
||||
public double UnplacedPartPenalty { get; init; }
|
||||
|
||||
public int PlatesUsed { get; init; }
|
||||
public Dictionary<string, int> SizeBreakdown { get; init; } = new();
|
||||
public long ElapsedMs { get; init; }
|
||||
@@ -31,5 +43,21 @@ namespace OpenNest.Benchmark
|
||||
/// total placed drawing area over total plate area, matching
|
||||
/// Plate.Utilization()'s per-plate definition summed across the job.</summary>
|
||||
public double Utilization => Valid && PlateArea > 0 ? PlacedArea / PlateArea : 0;
|
||||
|
||||
/// <summary>Placed area over salvage-credited sheet area.</summary>
|
||||
public double NetUtilization =>
|
||||
Valid && NetSheetArea > 0 ? PlacedArea / NetSheetArea : 0;
|
||||
|
||||
public int PartsUnplaced =>
|
||||
Valid ? System.Math.Max(0, PartsRequested - PartsPlaced) : PartsRequested;
|
||||
|
||||
/// <summary>
|
||||
/// The ranking score, in sheet area (lower is better): net sheet area
|
||||
/// consumed plus the unplaced penalty. An engine cannot improve it by
|
||||
/// dropping awkward parts, and it sums honestly across jobs of
|
||||
/// different sizes. Invalid runs consume no sheet but pay the penalty
|
||||
/// on every requested part.
|
||||
/// </summary>
|
||||
public double Cost => (Valid ? NetSheetArea : 0) + PartsUnplaced * UnplacedPartPenalty;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using OpenNest.Converters;
|
||||
using OpenNest.Engine.Jobs;
|
||||
using OpenNest.Geometry;
|
||||
using OpenNest.Math;
|
||||
|
||||
@@ -57,6 +58,92 @@ namespace OpenNest.Benchmark
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks what the materialized layout cannot show: every sheet must be
|
||||
/// one of the job's own stock entries (an engine may not invent a sheet
|
||||
/// size or loosen its spacing/edge settings, which the layout checks
|
||||
/// would otherwise trust), finite stock may not be overdrawn, and every
|
||||
/// placement's rotation must satisfy its part's RotationPolicy.
|
||||
/// </summary>
|
||||
public static void ValidateAgainstJob(
|
||||
NestJob job,
|
||||
NestJobResult jobResult,
|
||||
IReadOnlyDictionary<string, string> displayNames,
|
||||
ValidationResult result
|
||||
)
|
||||
{
|
||||
var stockById = job.Plates.ToDictionary(s => s.Id);
|
||||
var partsById = job.Parts.ToDictionary(p => p.Id);
|
||||
var sheetsUsed = new Dictionary<string, int>();
|
||||
|
||||
foreach (var sheet in jobResult.Plates)
|
||||
{
|
||||
if (
|
||||
!stockById.TryGetValue(sheet.Stock.Id, out var stock)
|
||||
|| !SameSettings(stock, sheet.Stock)
|
||||
)
|
||||
{
|
||||
result.Violations.Add(
|
||||
$"Plate {sheet.PlateIndex} uses stock '{sheet.Stock.Id}' ({sheet.Stock.Size}) that does not match any stock offered by the job"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
sheetsUsed[stock.Id] = sheetsUsed.GetValueOrDefault(stock.Id) + 1;
|
||||
}
|
||||
|
||||
foreach (var (stockId, used) in sheetsUsed)
|
||||
{
|
||||
var available = stockById[stockId].Quantity;
|
||||
|
||||
if (available.HasValue && used > available.Value)
|
||||
{
|
||||
result.Violations.Add(
|
||||
$"Used {used} sheet(s) of stock '{stockId}' but only {available.Value} are available"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var sheet in jobResult.Plates)
|
||||
{
|
||||
foreach (var placement in sheet.Placements)
|
||||
{
|
||||
if (!partsById.TryGetValue(placement.PartId, out var part))
|
||||
continue; // reported by ValidateQuantities
|
||||
|
||||
if (!part.Rotation.Allows(placement.Rotation))
|
||||
{
|
||||
var name = displayNames.TryGetValue(part.Id, out var n) ? n : part.Id;
|
||||
result.Violations.Add(
|
||||
$"'{name}' placed at {Angle.ToDegrees(placement.Rotation):F3}° on plate {sheet.PlateIndex}, "
|
||||
+ $"outside its rotation constraint ({Describe(part.Rotation)})"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool SameSettings(NestPlateStock expected, NestPlateStock actual) =>
|
||||
ReferenceEquals(expected, actual)
|
||||
|| (
|
||||
expected.Size.Equals(actual.Size)
|
||||
&& expected.PartSpacing.IsEqualTo(actual.PartSpacing)
|
||||
&& expected.EdgeSpacing.Left.IsEqualTo(actual.EdgeSpacing.Left)
|
||||
&& expected.EdgeSpacing.Right.IsEqualTo(actual.EdgeSpacing.Right)
|
||||
&& expected.EdgeSpacing.Top.IsEqualTo(actual.EdgeSpacing.Top)
|
||||
&& expected.EdgeSpacing.Bottom.IsEqualTo(actual.EdgeSpacing.Bottom)
|
||||
&& expected.Quadrant == actual.Quadrant
|
||||
);
|
||||
|
||||
private static string Describe(RotationPolicy policy) =>
|
||||
policy.Kind switch
|
||||
{
|
||||
RotationPolicyKind.Fixed => $"fixed at {Angle.ToDegrees(policy.Start):F3}°",
|
||||
RotationPolicyKind.BoundedSweep =>
|
||||
$"{Angle.ToDegrees(policy.Start):F3}° to {Angle.ToDegrees(policy.End):F3}° in {Angle.ToDegrees(policy.Step):F3}° steps",
|
||||
_ => "any",
|
||||
};
|
||||
|
||||
private static void ValidateQuantities(
|
||||
List<Part> parts,
|
||||
IReadOnlyDictionary<Drawing, (string Name, int Quantity)> requirements,
|
||||
@@ -142,6 +229,16 @@ namespace OpenNest.Benchmark
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every pair of parts must be at least <paramref name="spacing"/> apart.
|
||||
/// Each part's material is inflated by the spacing (perimeter offset
|
||||
/// outward, holes shrunk inward) and tested against the other part's raw
|
||||
/// material, with holes subtracted on both sides - so a small part
|
||||
/// nested inside another part's cutout (part-in-part) is legal as long as
|
||||
/// it clears the cutout's edge by the spacing. Pairs are pruned with an
|
||||
/// X-sorted sweep over bounding boxes so only neighbours reach the
|
||||
/// polygon clipper.
|
||||
/// </summary>
|
||||
private static void ValidateSpacing(
|
||||
List<Part> parts,
|
||||
double spacing,
|
||||
@@ -149,29 +246,49 @@ namespace OpenNest.Benchmark
|
||||
ValidationResult result
|
||||
)
|
||||
{
|
||||
var worldPolygons = new Polygon[parts.Count];
|
||||
var inflatedPolygons = new Polygon[parts.Count];
|
||||
var raw = new PartOutline[parts.Count];
|
||||
var inflated = new PartOutline[parts.Count];
|
||||
|
||||
for (var i = 0; i < parts.Count; i++)
|
||||
{
|
||||
worldPolygons[i] = WorldPolygon(parts[i], 0);
|
||||
inflatedPolygons[i] =
|
||||
spacing > Tolerance.Epsilon
|
||||
? WorldPolygon(parts[i], spacing)
|
||||
: worldPolygons[i];
|
||||
raw[i] = Outline(parts[i], 0);
|
||||
inflated[i] = spacing > Tolerance.Epsilon ? Outline(parts[i], spacing) : raw[i];
|
||||
}
|
||||
|
||||
for (var i = 0; i < parts.Count; i++)
|
||||
{
|
||||
if (worldPolygons[i] == null || inflatedPolygons[i] == null)
|
||||
continue;
|
||||
var order = Enumerable
|
||||
.Range(0, parts.Count)
|
||||
.Where(i => raw[i] != null && inflated[i] != null)
|
||||
.OrderBy(i => raw[i].Perimeter.BoundingBox.Left)
|
||||
.ToList();
|
||||
|
||||
for (var j = i + 1; j < parts.Count; j++)
|
||||
for (var a = 0; a < order.Count; a++)
|
||||
{
|
||||
var i = order[a];
|
||||
var reach = inflated[i].Perimeter.BoundingBox;
|
||||
|
||||
for (var b = a + 1; b < order.Count; b++)
|
||||
{
|
||||
if (worldPolygons[j] == null)
|
||||
var j = order[b];
|
||||
var other = raw[j].Perimeter.BoundingBox;
|
||||
|
||||
// Sorted by Left, so nothing further along can reach part i either.
|
||||
if (other.Left > reach.Right + Tolerance.Epsilon)
|
||||
break;
|
||||
|
||||
if (!BoxesTouch(reach, other))
|
||||
continue;
|
||||
|
||||
if (Collision.HasOverlap(inflatedPolygons[i], worldPolygons[j]))
|
||||
// Inflating one side by the full spacing covers both cases: part j
|
||||
// inside part i's (shrunk) cutout, or part i's inflated outline
|
||||
// inside part j's raw cutout.
|
||||
if (
|
||||
Collision.HasOverlap(
|
||||
inflated[i].Perimeter,
|
||||
raw[j].Perimeter,
|
||||
inflated[i].Holes,
|
||||
raw[j].Holes
|
||||
)
|
||||
)
|
||||
{
|
||||
result.Violations.Add(
|
||||
$"'{DisplayName(parts[i], requirements)}' and '{DisplayName(parts[j], requirements)}' are closer than the required spacing ({spacing:F3})"
|
||||
@@ -181,6 +298,12 @@ namespace OpenNest.Benchmark
|
||||
}
|
||||
}
|
||||
|
||||
private static bool BoxesTouch(Box a, Box b) =>
|
||||
a.Left <= b.Right + Tolerance.Epsilon
|
||||
&& b.Left <= a.Right + Tolerance.Epsilon
|
||||
&& a.Bottom <= b.Top + Tolerance.Epsilon
|
||||
&& b.Bottom <= a.Top + Tolerance.Epsilon;
|
||||
|
||||
/// <summary>Friendly name for a violation message, falling back to the materialized
|
||||
/// Drawing's own Name (the raw partId string) if this part wasn't in requirements at all -
|
||||
/// that mismatch is already reported by ValidateQuantities, so this is display-only.</summary>
|
||||
@@ -192,12 +315,21 @@ namespace OpenNest.Benchmark
|
||||
? requirement.Name
|
||||
: part.BaseDrawing.Name;
|
||||
|
||||
private sealed class PartOutline
|
||||
{
|
||||
public Polygon Perimeter { get; init; }
|
||||
public List<Polygon> Holes { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts a part's perimeter as a world-space polygon, optionally inflated
|
||||
/// outward by the given spacing, mirroring Part.Intersects' own geometry
|
||||
/// extraction (part.Program is already rotated; only a Location offset is needed).
|
||||
/// Extracts a part's material as world-space polygons - the perimeter and
|
||||
/// its cutouts - grown by <paramref name="inflateBy"/> (perimeter offset
|
||||
/// outward, cutouts offset inward). A cutout that closes up under the
|
||||
/// offset is dropped, which treats it as solid: conservative, since it
|
||||
/// has no room for another part at the required spacing anyway.
|
||||
/// part.Program is already rotated; only a Location offset is needed.
|
||||
/// </summary>
|
||||
private static Polygon WorldPolygon(Part part, double inflateBy)
|
||||
private static PartOutline Outline(Part part, double inflateBy)
|
||||
{
|
||||
var entities = ConvertProgram
|
||||
.ToGeometry(part.Program)
|
||||
@@ -207,23 +339,61 @@ namespace OpenNest.Benchmark
|
||||
if (entities.Count == 0)
|
||||
return null;
|
||||
|
||||
var perimeter = new ShapeProfile(entities).Perimeter;
|
||||
var profile = new ShapeProfile(entities);
|
||||
|
||||
if (perimeter == null)
|
||||
if (profile.Perimeter == null)
|
||||
return null;
|
||||
|
||||
var perimeter = profile.Perimeter;
|
||||
|
||||
if (inflateBy > Tolerance.Epsilon)
|
||||
perimeter = perimeter.OffsetOutward(inflateBy) ?? perimeter;
|
||||
|
||||
// Adaptive tolerance instead of Shape.ToPolygon()'s default (up to 1000
|
||||
// segments per arc) - arc-heavy real parts otherwise produce thousands
|
||||
// of vertices, which is needlessly slow for a spacing check.
|
||||
var polygon = perimeter.ToPolygonWithTolerance(0.01, circumscribe: true);
|
||||
var polygon = ToWorldPolygon(perimeter, part.Location);
|
||||
|
||||
if (polygon == null)
|
||||
return null;
|
||||
|
||||
polygon.Offset(part.Location);
|
||||
var holes = new List<Polygon>();
|
||||
|
||||
foreach (var cutout in profile.Cutouts)
|
||||
{
|
||||
var hole = cutout;
|
||||
|
||||
if (inflateBy > Tolerance.Epsilon)
|
||||
{
|
||||
hole = cutout.OffsetInward(inflateBy);
|
||||
|
||||
// An offset that collapsed or flipped inside-out leaves no usable room.
|
||||
if (
|
||||
hole == null
|
||||
|| hole.Area() <= Tolerance.Epsilon
|
||||
|| hole.Area() >= cutout.Area()
|
||||
)
|
||||
continue;
|
||||
}
|
||||
|
||||
var holePolygon = ToWorldPolygon(hole, part.Location);
|
||||
|
||||
if (holePolygon != null)
|
||||
holes.Add(holePolygon);
|
||||
}
|
||||
|
||||
return new PartOutline { Perimeter = polygon, Holes = holes };
|
||||
}
|
||||
|
||||
private static Polygon ToWorldPolygon(Shape shape, Vector location)
|
||||
{
|
||||
// Adaptive tolerance instead of Shape.ToPolygon()'s default (up to 1000
|
||||
// segments per arc) - arc-heavy real parts otherwise produce thousands
|
||||
// of vertices, which is needlessly slow for a spacing check.
|
||||
var polygon = shape.ToPolygonWithTolerance(0.01, circumscribe: true);
|
||||
|
||||
if (polygon == null)
|
||||
return null;
|
||||
|
||||
polygon.Offset(location);
|
||||
polygon.UpdateBounds();
|
||||
return polygon;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,6 +82,18 @@ static class BenchmarkConsole
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
options.SheetSizes.Count == 0
|
||||
&& jobs.Any(j => j.SourceFile.EndsWith(".nest", StringComparison.OrdinalIgnoreCase))
|
||||
)
|
||||
{
|
||||
Console.Error.WriteLine(
|
||||
"Warning: no --sheet-sizes given, so each .nest job only offers the sheet sizes its "
|
||||
+ "original layout used - a hint toward that answer. Pass --sheet-sizes with the "
|
||||
+ "sizes you actually stock for an unbiased comparison."
|
||||
);
|
||||
}
|
||||
|
||||
Console.WriteLine($"Engines: {string.Join(", ", engines.Select(e => e.Name))}");
|
||||
|
||||
var solves = jobs.Count * engines.Count;
|
||||
@@ -129,7 +141,10 @@ static class BenchmarkConsole
|
||||
break;
|
||||
|
||||
case "--spacing" when i + 1 < args.Length:
|
||||
o.PartSpacing = double.Parse(args[++i]);
|
||||
o.PartSpacing = double.Parse(
|
||||
args[++i],
|
||||
System.Globalization.CultureInfo.InvariantCulture
|
||||
);
|
||||
break;
|
||||
|
||||
case "--engines" when i + 1 < args.Length:
|
||||
@@ -195,7 +210,7 @@ static class BenchmarkConsole
|
||||
)
|
||||
)
|
||||
{
|
||||
if (Size.TryParse(token, out var size))
|
||||
if (JobLoader.TryParseSheetSize(token, out var size))
|
||||
sizes.Add(size);
|
||||
else
|
||||
Console.Error.WriteLine($"Warning: could not parse sheet size '{token}', skipping");
|
||||
@@ -223,16 +238,19 @@ static class BenchmarkConsole
|
||||
"multi-plate/size strategy: how many plates it uses, of which sizes, and how"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
"demand splits across them. Scoring: aggregate material utilization across every"
|
||||
"demand splits across them. Ranking: a run that places every requested part beats"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
"plate used, then (if everything requested was placed) fewer plates as the"
|
||||
"one that does not; then lower cost = sheet area consumed (minus salvage credit for a"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
"tie-break. An invalid layout (out of bounds, overlapping, or over-quantity), a"
|
||||
"usable offcut) + the largest candidate sheet's area per unplaced part; then fewer"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
"thrown exception, or a run exceeding its time budget all score zero."
|
||||
"plates. An invalid layout (out of bounds, overlapping, over-quantity, off-stock, or"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
"breaking a rotation constraint), a thrown exception, or a timeout places nothing."
|
||||
);
|
||||
Console.Error.WriteLine();
|
||||
Console.Error.WriteLine("Usage:");
|
||||
@@ -261,7 +279,10 @@ static class BenchmarkConsole
|
||||
" --sheet-sizes W1xL1,W2xL2,... Candidate sheet-size pool for the whole nest"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
" (default: the distinct sizes already in each file)"
|
||||
" (default: the distinct sizes already in each file,"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
" which hints engines toward the original layout)"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
" --spacing <value> Override part spacing for every job"
|
||||
@@ -273,7 +294,10 @@ static class BenchmarkConsole
|
||||
" --csv <path> Write a flat CSV of all results"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
" --salvage-rate <0..1> Fraction of eligible offcut area credited (default 0)"
|
||||
" --salvage-rate <0..1> Fraction of the usable offcut credited back in the"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
" score (default 0; needs --min-salvage-dimension)"
|
||||
);
|
||||
Console.Error.WriteLine(
|
||||
" --min-salvage-dimension <value> Both offcut dimensions must qualify; 0 disables credit"
|
||||
|
||||
@@ -9,10 +9,12 @@ namespace OpenNest.Benchmark
|
||||
{
|
||||
/// <summary>
|
||||
/// Console + CSV reporting for benchmark results. Ranking rule per job:
|
||||
/// valid beats invalid; higher aggregate utilization wins; if utilization
|
||||
/// ties and both engines fully placed every requested part, fewer plates
|
||||
/// used wins (the multi-plate analogue of "smaller remnant" - both are
|
||||
/// proxies for wasting less material). Ties beyond that are a shared win.
|
||||
/// valid beats invalid; placing every requested part beats not; then lower
|
||||
/// JobResult.Cost wins - salvage-credited sheet area consumed plus a
|
||||
/// largest-sheet penalty per unplaced part, so dropping awkward parts can
|
||||
/// never buy a better score; then fewer plates. Ties beyond that are a
|
||||
/// shared win. Across jobs, costs and areas are summed (not averaged), so
|
||||
/// a big job weighs more than a three-part one.
|
||||
/// </summary>
|
||||
public static class Report
|
||||
{
|
||||
@@ -29,7 +31,7 @@ namespace OpenNest.Benchmark
|
||||
var best = ranked.Count > 0 ? ranked[0] : null;
|
||||
|
||||
Console.WriteLine(
|
||||
$"{"Engine", -16} {"Result", -9} {"Parts", -10} {"Util%", -8} {"Plates", -18} {"Time(ms)", -9} Notes"
|
||||
$"{"Engine", -16} {"Result", -9} {"Parts", -10} {"Util%", -7} {"Net%", -7} {"Cost", -12} {"Plates", -18} {"Time(ms)", -9} Notes"
|
||||
);
|
||||
|
||||
foreach (var r in ranked)
|
||||
@@ -42,12 +44,13 @@ namespace OpenNest.Benchmark
|
||||
: "INVALID";
|
||||
var partsCol = $"{r.PartsPlaced}/{r.PartsRequested}";
|
||||
var utilCol = r.Valid ? $"{r.Utilization * 100:F1}" : "-";
|
||||
var netCol = r.Valid ? $"{r.NetUtilization * 100:F1}" : "-";
|
||||
var platesCol =
|
||||
r.PlatesUsed > 0 ? $"{r.PlatesUsed} ({SizeSummary(r.SizeBreakdown)})" : "-";
|
||||
var notes = r.Crashed ? r.Error : string.Join("; ", r.Violations.Take(2));
|
||||
|
||||
Console.WriteLine(
|
||||
$"{marker}{r.EngineName, -15} {status, -9} {partsCol, -10} {utilCol, -8} {platesCol, -18} {r.ElapsedMs, -9} {notes}"
|
||||
$"{marker}{r.EngineName, -15} {status, -9} {partsCol, -10} {utilCol, -7} {netCol, -7} {r.Cost, -12:F1} {platesCol, -18} {r.ElapsedMs, -9} {notes}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -67,25 +70,33 @@ namespace OpenNest.Benchmark
|
||||
Valid = g.Count(r => r.Valid),
|
||||
Crashed = g.Count(r => r.Crashed),
|
||||
FullyPlaced = g.Count(r => r.FullyPlaced),
|
||||
TotalUtilization = g.Sum(r => r.Utilization),
|
||||
Unplaced = g.Sum(r => r.PartsUnplaced),
|
||||
PlacedArea = g.Where(r => r.Valid).Sum(r => r.PlacedArea),
|
||||
PlateArea = g.Where(r => r.Valid).Sum(r => r.PlateArea),
|
||||
NetSheetArea = g.Where(r => r.Valid).Sum(r => r.NetSheetArea),
|
||||
TotalCost = g.Sum(r => r.Cost),
|
||||
TotalPlates = g.Sum(r => r.PlatesUsed),
|
||||
TotalTimeMs = g.Sum(r => r.ElapsedMs),
|
||||
})
|
||||
.OrderByDescending(e => e.TotalUtilization)
|
||||
.OrderBy(e => e.TotalCost)
|
||||
.ToList();
|
||||
|
||||
var wins = CountWins(results);
|
||||
|
||||
// Util% and Net% are area-weighted over valid runs (sum placed / sum
|
||||
// sheet), not a mean of per-job percentages. TotalCost sums across
|
||||
// jobs, so it is only meaningful when every job uses the same units.
|
||||
Console.WriteLine(
|
||||
$"{"Engine", -16} {"Jobs", -6} {"Valid", -7} {"Complete", -9} {"Wins", -6} {"AvgUtil%", -10} {"Plates", -8} {"TotalTime(ms)", -14}"
|
||||
$"{"Engine", -16} {"Jobs", -6} {"Valid", -7} {"Complete", -9} {"Unplaced", -9} {"Wins", -6} {"Util%", -7} {"Net%", -7} {"TotalCost", -14} {"Plates", -8} {"TotalTime(ms)", -14}"
|
||||
);
|
||||
|
||||
foreach (var e in byEngine)
|
||||
{
|
||||
var avgUtil = e.Jobs > 0 ? e.TotalUtilization / e.Jobs * 100 : 0;
|
||||
var util = e.PlateArea > 0 ? e.PlacedArea / e.PlateArea * 100 : 0;
|
||||
var netUtil = e.NetSheetArea > 0 ? e.PlacedArea / e.NetSheetArea * 100 : 0;
|
||||
var winCount = wins.TryGetValue(e.Engine, out var w) ? w : 0;
|
||||
Console.WriteLine(
|
||||
$"{e.Engine, -16} {e.Jobs, -6} {e.Valid, -7} {e.FullyPlaced, -9} {winCount, -6} {avgUtil, -10:F1} {e.TotalPlates, -8} {e.TotalTimeMs, -14}"
|
||||
$"{e.Engine, -16} {e.Jobs, -6} {e.Valid, -7} {e.FullyPlaced, -9} {e.Unplaced, -9} {winCount, -6} {util, -7:F1} {netUtil, -7:F1} {e.TotalCost, -14:F1} {e.TotalPlates, -8} {e.TotalTimeMs, -14}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -94,7 +105,7 @@ namespace OpenNest.Benchmark
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
sb.AppendLine(
|
||||
"Job,Engine,Valid,Crashed,FullyPlaced,PartsPlaced,PartsRequested,Utilization,PlatesUsed,SizeBreakdown,ElapsedMs,Notes"
|
||||
"Job,Engine,Valid,Crashed,FullyPlaced,PartsPlaced,PartsRequested,Utilization,NetUtilization,PlateArea,NetSheetArea,Cost,PlatesUsed,SizeBreakdown,ElapsedMs,Notes"
|
||||
);
|
||||
|
||||
foreach (var r in results)
|
||||
@@ -111,6 +122,10 @@ namespace OpenNest.Benchmark
|
||||
r.PartsPlaced,
|
||||
r.PartsRequested,
|
||||
r.Utilization.ToString("F4", CultureInfo.InvariantCulture),
|
||||
r.NetUtilization.ToString("F4", CultureInfo.InvariantCulture),
|
||||
r.PlateArea.ToString("F2", CultureInfo.InvariantCulture),
|
||||
r.NetSheetArea.ToString("F2", CultureInfo.InvariantCulture),
|
||||
r.Cost.ToString("F2", CultureInfo.InvariantCulture),
|
||||
r.PlatesUsed,
|
||||
Csv(SizeSummary(r.SizeBreakdown)),
|
||||
r.ElapsedMs,
|
||||
@@ -159,9 +174,10 @@ namespace OpenNest.Benchmark
|
||||
return wins;
|
||||
}
|
||||
|
||||
/// <summary>Lower sorts first (better). Valid beats invalid, then higher
|
||||
/// aggregate utilization, then (if both fully placed) fewer plates used.</summary>
|
||||
private static int Compare(JobResult a, JobResult b)
|
||||
/// <summary>Lower sorts first (better). Valid beats invalid, complete beats
|
||||
/// incomplete, then lower cost (relative tolerance, since costs are areas
|
||||
/// in whatever units the job uses), then fewer plates.</summary>
|
||||
public static int Compare(JobResult a, JobResult b)
|
||||
{
|
||||
if (a.Valid != b.Valid)
|
||||
return a.Valid ? -1 : 1;
|
||||
@@ -169,12 +185,16 @@ namespace OpenNest.Benchmark
|
||||
if (!a.Valid)
|
||||
return 0;
|
||||
|
||||
var utilDiff = b.Utilization - a.Utilization;
|
||||
if (a.FullyPlaced != b.FullyPlaced)
|
||||
return a.FullyPlaced ? -1 : 1;
|
||||
|
||||
if (System.Math.Abs(utilDiff) > Epsilon)
|
||||
return utilDiff > 0 ? 1 : -1;
|
||||
var costDiff = a.Cost - b.Cost;
|
||||
var scale = System.Math.Max(1, System.Math.Max(a.Cost, b.Cost));
|
||||
|
||||
if (a.FullyPlaced && b.FullyPlaced && a.PlatesUsed != b.PlatesUsed)
|
||||
if (System.Math.Abs(costDiff) > Epsilon * scale)
|
||||
return costDiff > 0 ? 1 : -1;
|
||||
|
||||
if (a.PlatesUsed != b.PlatesUsed)
|
||||
return a.PlatesUsed > b.PlatesUsed ? 1 : -1;
|
||||
|
||||
return 0;
|
||||
|
||||
Reference in New Issue
Block a user