aj/OpenNest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(console): improve training data collection and best-fit persistence

Browse Source 
- Add verbose per-file and per-sheet-size console output during collection
- Skip already-processed parts at the sheet-size level instead of all-or-nothing
- Precompute best-fits once per part and reuse across all sheet sizes
- Clear best-fit cache after each part to prevent memory growth
- Save best-fits in separate bestfits/ zip entries instead of embedding in nest.json
- Filter to Keep=true results only and scope to plate sizes in the nest
- Set nest name to match filename (includes sheet size and part count)
- Add TrainingDatabase with per-run skip logic and SQLite schema

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
AJ Isaacs
2026-03-14 12:39:24 -04:00
parent 3133228fc9
commit d6ffa77f35
8 changed files with 497 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
OpenNest.Console/Program.cs
View File

@@ -299,11 +299,6 @@ int RunDataCollection(string dir, string dbPath, string saveDir, double s, strin
Color.FromArgb(215, 130, 130),
};
var dxfFiles = Directory.GetFiles(dir, "*.dxf", SearchOption.AllDirectories);
Console.WriteLine($"Found {dxfFiles.Length} DXF files. Initializing SQLite database at: {dbPath}");
using var db = new TrainingDatabase(dbPath);
var sheetSuite = new[]
{
new Size(96, 48), new Size(120, 48), new Size(144, 48),
@@ -312,17 +307,48 @@ int RunDataCollection(string dir, string dbPath, string saveDir, double s, strin
new Size(48, 24), new Size(120, 10)
};
var dxfFiles = Directory.GetFiles(dir, "*.dxf", SearchOption.AllDirectories);
Console.WriteLine($"Found {dxfFiles.Length} DXF files");
Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}");
Console.WriteLine($"Sheet sizes: {sheetSuite.Length} configurations");
Console.WriteLine($"Spacing: {s:F2}");
if (saveDir != null) Console.WriteLine($"Saving nests to: {saveDir}");
Console.WriteLine("---");
using var db = new TrainingDatabase(dbPath);
var importer = new DxfImporter();
var colorIndex = 0;
var processed = 0;
var skippedGeometry = 0;
var skippedFeatures = 0;
var skippedExisting = 0;
var totalRuns = 0;
var totalSw = Stopwatch.StartNew();
foreach (var file in dxfFiles)
{
var fileNum = processed + skippedGeometry + skippedFeatures + skippedExisting + 1;
var partNo = Path.GetFileNameWithoutExtension(file);
Console.Write($"[{fileNum}/{dxfFiles.Length}] {partNo}");
try
{
if (!importer.GetGeometry(file, out var entities)) continue;
var existingRuns = db.RunCount(Path.GetFileName(file));
if (existingRuns >= sheetSuite.Length)
{
Console.WriteLine(" - SKIP (all sizes done)");
skippedExisting++;
continue;
}
if (!importer.GetGeometry(file, out var entities))
{
Console.WriteLine(" - SKIP (no geometry)");
skippedGeometry++;
continue;
}
var partNo = Path.GetFileNameWithoutExtension(file);
var drawing = new Drawing(Path.GetFileName(file));
drawing.Program = OpenNest.Converters.ConvertGeometry.ToProgram(entities);
drawing.UpdateArea();
@@ -330,14 +356,38 @@ int RunDataCollection(string dir, string dbPath, string saveDir, double s, strin
colorIndex++;
var features = FeatureExtractor.Extract(drawing);
if (features == null) continue;
if (features == null)
{
Console.WriteLine(" - SKIP (feature extraction failed)");
skippedFeatures++;
continue;
}
Console.WriteLine($" (area={features.Area:F1}, verts={features.VertexCount})");
// Precompute best-fits once for all sheet sizes.
var sizes = sheetSuite.Select(sz => (sz.Width, sz.Length)).ToList();
var bfSw = Stopwatch.StartNew();
BestFitCache.ComputeForSizes(drawing, s, sizes);
bfSw.Stop();
Console.WriteLine($" Best-fits computed in {bfSw.ElapsedMilliseconds}ms");
using var txn = db.BeginTransaction();
var partId = db.GetOrAddPart(Path.GetFileName(file), features, drawing.Program.ToString());
var partSw = Stopwatch.StartNew();
var runsThisPart = 0;
var bestUtil = 0.0;
var bestCount = 0;
foreach (var size in sheetSuite)
{
if (db.HasRun(Path.GetFileName(file), size.Width, size.Length, s))
{
Console.WriteLine($" {size.Length}x{size.Width} - skip (exists)");
continue;
}
Plate runPlate;
if (templateNest != null)
{
@@ -350,8 +400,23 @@ int RunDataCollection(string dir, string dbPath, string saveDir, double s, strin
runPlate = new Plate { Size = size, PartSpacing = s };
}
var sizeSw = Stopwatch.StartNew();
var result = BruteForceRunner.Run(drawing, runPlate);
if (result == null) continue;
sizeSw.Stop();
if (result == null)
{
Console.WriteLine($" {size.Length}x{size.Width} - no fit");
continue;
}
if (result.Utilization > bestUtil)
{
bestUtil = result.Utilization;
bestCount = result.PartCount;
}
Console.WriteLine($" {size.Length}x{size.Width} - {result.PartCount}pcs, {result.Utilization:P1}, {sizeSw.ElapsedMilliseconds}ms");
string savedFilePath = null;
if (saveDir != null)
@@ -364,14 +429,15 @@ int RunDataCollection(string dir, string dbPath, string saveDir, double s, strin
var partDir = Path.Combine(saveDir, bucket, partNo);
Directory.CreateDirectory(partDir);
var fileName = $"{partNo}-{size.Length}x{size.Width}-{result.PartCount}pcs.zip";
var nestName = $"{partNo}-{size.Length}x{size.Width}-{result.PartCount}pcs";
var fileName = nestName + ".zip";
savedFilePath = Path.Combine(partDir, fileName);
// Create nest from template or from scratch
Nest nestObj;
if (templateNest != null)
{
nestObj = new Nest(partNo)
nestObj = new Nest(nestName)
{
Units = templateNest.Units,
DateCreated = DateTime.Now
@@ -380,7 +446,7 @@ int RunDataCollection(string dir, string dbPath, string saveDir, double s, strin
}
else
{
nestObj = new Nest(partNo) { Units = Units.Inches, DateCreated = DateTime.Now };
nestObj = new Nest(nestName) { Units = Units.Inches, DateCreated = DateTime.Now };
}
nestObj.Drawings.Add(drawing);
@@ -394,19 +460,29 @@ int RunDataCollection(string dir, string dbPath, string saveDir, double s, strin
}
db.AddRun(partId, size.Width, size.Length, s, result, savedFilePath);
runsThisPart++;
totalRuns++;
}
txn.Commit();
BestFitCache.Invalidate(drawing);
partSw.Stop();
processed++;
if (processed % 10 == 0) Console.WriteLine($"Processed {processed}/{dxfFiles.Length} parts across all sheet sizes...");
Console.WriteLine($" Total: {runsThisPart} runs, best={bestCount}pcs @ {bestUtil:P1}, {partSw.ElapsedMilliseconds}ms");
}
catch (Exception ex)
{
Console.Error.WriteLine($"Error processing {file}: {ex.Message}");
Console.WriteLine();
Console.Error.WriteLine($" ERROR: {ex.Message}");
}
}
Console.WriteLine($"Done! Brute-force data for {processed} parts saved to {dbPath}");
totalSw.Stop();
Console.WriteLine("---");
Console.WriteLine($"Processed: {processed} parts, {totalRuns} total runs");
Console.WriteLine($"Skipped: {skippedExisting} (existing) + {skippedGeometry} (no geometry) + {skippedFeatures} (no features)");
Console.WriteLine($"Time: {totalSw.Elapsed:h\\:mm\\:ss}");
Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}");
return 0;
}

145
OpenNest.Console/TrainingDatabase.cs Normal file
View File

@@ -0,0 +1,145 @@
using System;
using Microsoft.Data.Sqlite;
using OpenNest.Engine.ML;
namespace OpenNest.Console
{
public class TrainingDatabase : IDisposable
{
private readonly SqliteConnection _connection;
public TrainingDatabase(string dbPath)
{
var connectionString = new SqliteConnectionStringBuilder
{
DataSource = dbPath,
Mode = SqliteOpenMode.ReadWriteCreate
}.ToString();
_connection = new SqliteConnection(connectionString);
_connection.Open();
InitializeSchema();
}
private void InitializeSchema()
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = @"
CREATE TABLE IF NOT EXISTS Parts (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
FileName TEXT,
Area REAL,
Convexity REAL,
AspectRatio REAL,
BBFill REAL,
Circularity REAL,
VertexCount INTEGER,
Bitmask BLOB,
GeometryData TEXT
);
CREATE TABLE IF NOT EXISTS Runs (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
PartId INTEGER,
SheetWidth REAL,
SheetHeight REAL,
Spacing REAL,
PartCount INTEGER,
Utilization REAL,
TimeMs INTEGER,
LayoutData TEXT,
FilePath TEXT,
FOREIGN KEY(PartId) REFERENCES Parts(Id)
);
CREATE INDEX IF NOT EXISTS idx_parts_filename ON Parts(FileName);
CREATE INDEX IF NOT EXISTS idx_runs_partid ON Runs(PartId);
";
cmd.ExecuteNonQuery();
}
public long GetOrAddPart(string fileName, PartFeatures features, string geometryData)
{
// Check if part already exists
using (var checkCmd = _connection.CreateCommand())
{
checkCmd.CommandText = "SELECT Id FROM Parts WHERE FileName = @name";
checkCmd.Parameters.AddWithValue("@name", fileName);
var result = checkCmd.ExecuteScalar();
if (result != null) return (long)result;
}
// Add new part
using (var insertCmd = _connection.CreateCommand())
{
insertCmd.CommandText = @"
INSERT INTO Parts (FileName, Area, Convexity, AspectRatio, BBFill, Circularity, VertexCount, Bitmask, GeometryData)
VALUES (@name, @area, @conv, @asp, @fill, @circ, @vert, @mask, @geo);
SELECT last_insert_rowid();";
insertCmd.Parameters.AddWithValue("@name", fileName);
insertCmd.Parameters.AddWithValue("@area", features.Area);
insertCmd.Parameters.AddWithValue("@conv", features.Convexity);
insertCmd.Parameters.AddWithValue("@asp", features.AspectRatio);
insertCmd.Parameters.AddWithValue("@fill", features.BoundingBoxFill);
insertCmd.Parameters.AddWithValue("@circ", features.Circularity);
insertCmd.Parameters.AddWithValue("@vert", features.VertexCount);
insertCmd.Parameters.AddWithValue("@mask", features.Bitmask);
insertCmd.Parameters.AddWithValue("@geo", geometryData);
return (long)insertCmd.ExecuteScalar();
}
}
public bool HasRun(string fileName, double sheetWidth, double sheetHeight, double spacing)
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = @"SELECT COUNT(*) FROM Runs r JOIN Parts p ON r.PartId = p.Id
WHERE p.FileName = @name AND r.SheetWidth = @w AND r.SheetHeight = @h AND r.Spacing = @s";
cmd.Parameters.AddWithValue("@name", fileName);
cmd.Parameters.AddWithValue("@w", sheetWidth);
cmd.Parameters.AddWithValue("@h", sheetHeight);
cmd.Parameters.AddWithValue("@s", spacing);
return (long)cmd.ExecuteScalar() > 0;
}
public int RunCount(string fileName)
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM Runs r JOIN Parts p ON r.PartId = p.Id WHERE p.FileName = @name";
cmd.Parameters.AddWithValue("@name", fileName);
return (int)(long)cmd.ExecuteScalar();
}
public void AddRun(long partId, double w, double h, double s, BruteForceResult result, string filePath)
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = @"
INSERT INTO Runs (PartId, SheetWidth, SheetHeight, Spacing, PartCount, Utilization, TimeMs, LayoutData, FilePath)
VALUES (@pid, @w, @h, @s, @cnt, @util, @time, @layout, @path)";
cmd.Parameters.AddWithValue("@pid", partId);
cmd.Parameters.AddWithValue("@w", w);
cmd.Parameters.AddWithValue("@h", h);
cmd.Parameters.AddWithValue("@s", s);
cmd.Parameters.AddWithValue("@cnt", result.PartCount);
cmd.Parameters.AddWithValue("@util", result.Utilization);
cmd.Parameters.AddWithValue("@time", result.TimeMs);
cmd.Parameters.AddWithValue("@layout", result.LayoutData ?? "");
cmd.Parameters.AddWithValue("@path", filePath ?? "");
cmd.ExecuteNonQuery();
}
public SqliteTransaction BeginTransaction()
{
return _connection.BeginTransaction();
}
public void Dispose()
{
_connection?.Dispose();
}
}
}