aj/OpenNest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: extract training data collection into OpenNest.Training

Browse Source 
Move brute-force data collection, TrainingDatabase, and GPU init from
OpenNest.Console into a dedicated OpenNest.Training project. Replaces
raw Microsoft.Data.Sqlite with EF Core. Console is now a pure nesting
CLI with template support and cleaned-up usage output.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
AJ Isaacs
2026-03-14 14:41:38 -04:00
parent 74272bea80
commit acc75868c0
10 changed files with 591 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
OpenNest.Console/OpenNest.Console.csproj
View File

@@ -10,9 +10,5 @@
<ProjectReference Include="..\OpenNest.Core\OpenNest.Core.csproj" />
<ProjectReference Include="..\OpenNest.Engine\OpenNest.Engine.csproj" />
<ProjectReference Include="..\OpenNest.IO\OpenNest.IO.csproj" />
<ProjectReference Include="..\OpenNest.Gpu\OpenNest.Gpu.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.5" />
</ItemGroup>
</Project>

315
OpenNest.Console/Program.cs
View File

@@ -6,11 +6,6 @@ using System.Linq;
using OpenNest;
using OpenNest.Geometry;
using OpenNest.IO;
using Color = System.Drawing.Color;
using OpenNest.Console;
using OpenNest.Engine.BestFit;
using OpenNest.Engine.ML;
using OpenNest.Gpu;
// Parse arguments.
var nestFile = (string)null;
@@ -26,27 +21,12 @@ var noSave = false;
var noLog = false;
var keepParts = false;
var autoNest = false;
var collectDataDir = (string)null;
var dbPath = "nesting_training.db";
var saveNestsDir = (string)null;
var templateFile = (string)null;
for (var i = 0; i < args.Length; i++)
{
switch (args[i])
{
case "--db" when i + 1 < args.Length:
dbPath = args[++i];
break;
case "--save-nests" when i + 1 < args.Length:
saveNestsDir = args[++i];
break;
case "--template" when i + 1 < args.Length:
templateFile = args[++i];
break;
case "--collect" when i + 1 < args.Length:
collectDataDir = args[++i];
break;
case "--drawing" when i + 1 < args.Length:
drawingName = args[++i];
break;
@@ -82,6 +62,9 @@ for (var i = 0; i < args.Length; i++)
case "--keep-parts":
keepParts = true;
break;
case "--template" when i + 1 < args.Length:
templateFile = args[++i];
break;
case "--autonest":
autoNest = true;
break;
@@ -96,22 +79,6 @@ for (var i = 0; i < args.Length; i++)
}
}
// Initialize GPU if available.
if (GpuEvaluatorFactory.GpuAvailable)
{
BestFitCache.CreateSlideComputer = () => GpuEvaluatorFactory.CreateSlideComputer();
Console.WriteLine($"GPU: {GpuEvaluatorFactory.DeviceName}");
}
else
{
Console.WriteLine("GPU: not available (using CPU)");
}
if (collectDataDir != null)
{
return RunDataCollection(collectDataDir, dbPath, saveNestsDir, spacing ?? 0.5, templateFile);
}
if (string.IsNullOrEmpty(nestFile) || !File.Exists(nestFile))
{
PrintUsage();
@@ -149,6 +116,24 @@ if (plateIndex >= nest.Plates.Count)
var plate = nest.Plates[plateIndex];
// Apply template defaults.
if (templateFile != null)
{
if (!File.Exists(templateFile))
{
Console.Error.WriteLine($"Error: Template not found: {templateFile}");
return 1;
}
var templateNest = new NestReader(templateFile).Read();
var templatePlate = templateNest.PlateDefaults.CreateNew();
plate.Thickness = templatePlate.Thickness;
plate.Quadrant = templatePlate.Quadrant;
plate.Material = templatePlate.Material;
plate.EdgeSpacing = templatePlate.EdgeSpacing;
plate.PartSpacing = templatePlate.PartSpacing;
Console.WriteLine($"Template: {templateFile}");
}
// Apply overrides.
if (spacing.HasValue)
plate.PartSpacing = spacing.Value;
@@ -207,9 +192,9 @@ if (autoNest)
Console.WriteLine($"AutoNest: {nestItems.Count} drawing(s), {nestItems.Sum(i => i.Quantity)} total parts");
var parts = NestEngine.AutoNest(nestItems, plate);
plate.Parts.AddRange(parts);
success = parts.Count > 0;
var nestParts = NestEngine.AutoNest(nestItems, plate);
plate.Parts.AddRange(nestParts);
success = nestParts.Count > 0;
}
else
{
@@ -262,253 +247,25 @@ if (!noSave)
return checkOverlaps && overlapCount > 0 ? 1 : 0;
int RunDataCollection(string dir, string dbPath, string saveDir, double s, string template)
{
if (!Directory.Exists(dir))
{
Console.Error.WriteLine($"Error: Directory not found: {dir}");
return 1;
}
// Load template nest for plate defaults if provided.
Nest templateNest = null;
if (template != null)
{
if (!File.Exists(template))
{
Console.Error.WriteLine($"Error: Template not found: {template}");
return 1;
}
templateNest = new NestReader(template).Read();
Console.WriteLine($"Using template: {template}");
}
var PartColors = new[]
{
Color.FromArgb(205, 92, 92),
Color.FromArgb(148, 103, 189),
Color.FromArgb(75, 180, 175),
Color.FromArgb(210, 190, 75),
Color.FromArgb(190, 85, 175),
Color.FromArgb(185, 115, 85),
Color.FromArgb(120, 100, 190),
Color.FromArgb(200, 100, 140),
Color.FromArgb(80, 175, 155),
Color.FromArgb(195, 160, 85),
Color.FromArgb(175, 95, 160),
Color.FromArgb(215, 130, 130),
};
var sheetSuite = new[]
{
new Size(96, 48), new Size(120, 48), new Size(144, 48),
new Size(96, 60), new Size(120, 60), new Size(144, 60),
new Size(96, 72), new Size(120, 72), new Size(144, 72),
new Size(48, 24), new Size(120, 10)
};
var dxfFiles = Directory.GetFiles(dir, "*.dxf", SearchOption.AllDirectories);
Console.WriteLine($"Found {dxfFiles.Length} DXF files");
Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}");
Console.WriteLine($"Sheet sizes: {sheetSuite.Length} configurations");
Console.WriteLine($"Spacing: {s:F2}");
if (saveDir != null) Console.WriteLine($"Saving nests to: {saveDir}");
Console.WriteLine("---");
using var db = new TrainingDatabase(dbPath);
var importer = new DxfImporter();
var colorIndex = 0;
var processed = 0;
var skippedGeometry = 0;
var skippedFeatures = 0;
var skippedExisting = 0;
var totalRuns = 0;
var totalSw = Stopwatch.StartNew();
foreach (var file in dxfFiles)
{
var fileNum = processed + skippedGeometry + skippedFeatures + skippedExisting + 1;
var partNo = Path.GetFileNameWithoutExtension(file);
Console.Write($"[{fileNum}/{dxfFiles.Length}] {partNo}");
try
{
var existingRuns = db.RunCount(Path.GetFileName(file));
if (existingRuns >= sheetSuite.Length)
{
Console.WriteLine(" - SKIP (all sizes done)");
skippedExisting++;
continue;
}
if (!importer.GetGeometry(file, out var entities))
{
Console.WriteLine(" - SKIP (no geometry)");
skippedGeometry++;
continue;
}
var drawing = new Drawing(Path.GetFileName(file));
drawing.Program = OpenNest.Converters.ConvertGeometry.ToProgram(entities);
drawing.UpdateArea();
drawing.Color = PartColors[colorIndex % PartColors.Length];
colorIndex++;
var features = FeatureExtractor.Extract(drawing);
if (features == null)
{
Console.WriteLine(" - SKIP (feature extraction failed)");
skippedFeatures++;
continue;
}
Console.WriteLine($" (area={features.Area:F1}, verts={features.VertexCount})");
// Precompute best-fits once for all sheet sizes.
var sizes = sheetSuite.Select(sz => (sz.Width, sz.Length)).ToList();
var bfSw = Stopwatch.StartNew();
BestFitCache.ComputeForSizes(drawing, s, sizes);
bfSw.Stop();
Console.WriteLine($" Best-fits computed in {bfSw.ElapsedMilliseconds}ms");
using var txn = db.BeginTransaction();
var partId = db.GetOrAddPart(Path.GetFileName(file), features, drawing.Program.ToString());
var partSw = Stopwatch.StartNew();
var runsThisPart = 0;
var bestUtil = 0.0;
var bestCount = 0;
foreach (var size in sheetSuite)
{
if (db.HasRun(Path.GetFileName(file), size.Width, size.Length, s))
{
Console.WriteLine($" {size.Length}x{size.Width} - skip (exists)");
continue;
}
Plate runPlate;
if (templateNest != null)
{
runPlate = templateNest.PlateDefaults.CreateNew();
runPlate.Size = size;
runPlate.PartSpacing = s;
}
else
{
runPlate = new Plate { Size = size, PartSpacing = s };
}
var sizeSw = Stopwatch.StartNew();
var result = BruteForceRunner.Run(drawing, runPlate);
sizeSw.Stop();
if (result == null)
{
Console.WriteLine($" {size.Length}x{size.Width} - no fit");
continue;
}
if (result.Utilization > bestUtil)
{
bestUtil = result.Utilization;
bestCount = result.PartCount;
}
Console.WriteLine($" {size.Length}x{size.Width} - {result.PartCount}pcs, {result.Utilization:P1}, {sizeSw.ElapsedMilliseconds}ms");
string savedFilePath = null;
if (saveDir != null)
{
// Deterministic bucket (00-FF) based on filename hash
uint hash = 0;
foreach (char c in partNo) hash = (hash * 31) + c;
var bucket = (hash % 256).ToString("X2");
var partDir = Path.Combine(saveDir, bucket, partNo);
Directory.CreateDirectory(partDir);
var nestName = $"{partNo}-{size.Length}x{size.Width}-{result.PartCount}pcs";
var fileName = nestName + ".zip";
savedFilePath = Path.Combine(partDir, fileName);
// Create nest from template or from scratch
Nest nestObj;
if (templateNest != null)
{
nestObj = new Nest(nestName)
{
Units = templateNest.Units,
DateCreated = DateTime.Now
};
nestObj.PlateDefaults.SetFromExisting(templateNest.PlateDefaults.CreateNew());
}
else
{
nestObj = new Nest(nestName) { Units = Units.Inches, DateCreated = DateTime.Now };
}
nestObj.Drawings.Add(drawing);
var plateObj = nestObj.CreatePlate();
plateObj.Size = size;
plateObj.PartSpacing = s;
plateObj.Parts.AddRange(result.PlacedParts);
var writer = new NestWriter(nestObj);
writer.Write(savedFilePath);
}
db.AddRun(partId, size.Width, size.Length, s, result, savedFilePath);
runsThisPart++;
totalRuns++;
}
txn.Commit();
BestFitCache.Invalidate(drawing);
partSw.Stop();
processed++;
Console.WriteLine($" Total: {runsThisPart} runs, best={bestCount}pcs @ {bestUtil:P1}, {partSw.ElapsedMilliseconds}ms");
}
catch (Exception ex)
{
Console.WriteLine();
Console.Error.WriteLine($" ERROR: {ex.Message}");
}
}
totalSw.Stop();
Console.WriteLine("---");
Console.WriteLine($"Processed: {processed} parts, {totalRuns} total runs");
Console.WriteLine($"Skipped: {skippedExisting} (existing) + {skippedGeometry} (no geometry) + {skippedFeatures} (no features)");
Console.WriteLine($"Time: {totalSw.Elapsed:h\\:mm\\:ss}");
Console.WriteLine($"Database: {Path.GetFullPath(dbPath)}");
return 0;
}
void PrintUsage()
{
Console.Error.WriteLine("Usage: OpenNest.Console <nest-file> [options]");
Console.Error.WriteLine(" OpenNest.Console --collect <dxf-dir> [options]");
Console.Error.WriteLine();
Console.Error.WriteLine("Arguments:");
Console.Error.WriteLine(" nest-file Path to a .zip nest file");
Console.Error.WriteLine();
Console.Error.WriteLine("Options:");
Console.Error.WriteLine(" --drawing <name> Drawing name to fill with (default: first drawing)");
Console.WriteLine(" --plate <index> Plate index to fill (default: 0)");
Console.WriteLine(" --quantity <n> Max parts to place (default: 0 = unlimited)");
Console.WriteLine(" --spacing <value> Override part spacing");
Console.WriteLine(" --size <WxH> Override plate size (e.g. 120x60)");
Console.WriteLine(" --output <path> Output nest file path (default: <input>-result.zip)");
Console.WriteLine(" --autonest Use NFP-based mixed-part autonesting instead of linear fill");
Console.WriteLine(" --keep-parts Don't clear existing parts before filling");
Console.WriteLine(" --check-overlaps Run overlap detection after fill (exit code 1 if found)");
Console.WriteLine(" --no-save Skip saving output file");
Console.WriteLine(" --no-log Skip writing debug log file");
Console.WriteLine(" --collect <dir> Brute-force process all DXFs in directory to SQLite");
Console.WriteLine(" --db <path> Path to the SQLite training database (default: nesting_training.db)");
Console.WriteLine(" --save-nests <dir> Directory to save individual .zip nests for each winner");
Console.WriteLine(" --template <path> Nest template (.nstdot) for plate defaults");
Console.WriteLine(" -h, --help Show this help");
Console.Error.WriteLine(" --plate <index> Plate index to fill (default: 0)");
Console.Error.WriteLine(" --quantity <n> Max parts to place (default: 0 = unlimited)");
Console.Error.WriteLine(" --spacing <value> Override part spacing");
Console.Error.WriteLine(" --size <WxH> Override plate size (e.g. 120x60)");
Console.Error.WriteLine(" --output <path> Output nest file path (default: <input>-result.zip)");
Console.Error.WriteLine(" --template <path> Nest template for plate defaults (thickness, quadrant, material, spacing)");
Console.Error.WriteLine(" --autonest Use NFP-based mixed-part autonesting instead of linear fill");
Console.Error.WriteLine(" --keep-parts Don't clear existing parts before filling");
Console.Error.WriteLine(" --check-overlaps Run overlap detection after fill (exit code 1 if found)");
Console.Error.WriteLine(" --no-save Skip saving output file");
Console.Error.WriteLine(" --no-log Skip writing debug log file");
Console.Error.WriteLine(" -h, --help Show this help");
}

145
OpenNest.Console/TrainingDatabase.cs
View File

@@ -1,145 +0,0 @@
using System;
using Microsoft.Data.Sqlite;
using OpenNest.Engine.ML;
namespace OpenNest.Console
{
public class TrainingDatabase : IDisposable
{
private readonly SqliteConnection _connection;
public TrainingDatabase(string dbPath)
{
var connectionString = new SqliteConnectionStringBuilder
{
DataSource = dbPath,
Mode = SqliteOpenMode.ReadWriteCreate
}.ToString();
_connection = new SqliteConnection(connectionString);
_connection.Open();
InitializeSchema();
}
private void InitializeSchema()
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = @"
CREATE TABLE IF NOT EXISTS Parts (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
FileName TEXT,
Area REAL,
Convexity REAL,
AspectRatio REAL,
BBFill REAL,
Circularity REAL,
VertexCount INTEGER,
Bitmask BLOB,
GeometryData TEXT
);
CREATE TABLE IF NOT EXISTS Runs (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
PartId INTEGER,
SheetWidth REAL,
SheetHeight REAL,
Spacing REAL,
PartCount INTEGER,
Utilization REAL,
TimeMs INTEGER,
LayoutData TEXT,
FilePath TEXT,
FOREIGN KEY(PartId) REFERENCES Parts(Id)
);
CREATE INDEX IF NOT EXISTS idx_parts_filename ON Parts(FileName);
CREATE INDEX IF NOT EXISTS idx_runs_partid ON Runs(PartId);
";
cmd.ExecuteNonQuery();
}
public long GetOrAddPart(string fileName, PartFeatures features, string geometryData)
{
// Check if part already exists
using (var checkCmd = _connection.CreateCommand())
{
checkCmd.CommandText = "SELECT Id FROM Parts WHERE FileName = @name";
checkCmd.Parameters.AddWithValue("@name", fileName);
var result = checkCmd.ExecuteScalar();
if (result != null) return (long)result;
}
// Add new part
using (var insertCmd = _connection.CreateCommand())
{
insertCmd.CommandText = @"
INSERT INTO Parts (FileName, Area, Convexity, AspectRatio, BBFill, Circularity, VertexCount, Bitmask, GeometryData)
VALUES (@name, @area, @conv, @asp, @fill, @circ, @vert, @mask, @geo);
SELECT last_insert_rowid();";
insertCmd.Parameters.AddWithValue("@name", fileName);
insertCmd.Parameters.AddWithValue("@area", features.Area);
insertCmd.Parameters.AddWithValue("@conv", features.Convexity);
insertCmd.Parameters.AddWithValue("@asp", features.AspectRatio);
insertCmd.Parameters.AddWithValue("@fill", features.BoundingBoxFill);
insertCmd.Parameters.AddWithValue("@circ", features.Circularity);
insertCmd.Parameters.AddWithValue("@vert", features.VertexCount);
insertCmd.Parameters.AddWithValue("@mask", features.Bitmask);
insertCmd.Parameters.AddWithValue("@geo", geometryData);
return (long)insertCmd.ExecuteScalar();
}
}
public bool HasRun(string fileName, double sheetWidth, double sheetHeight, double spacing)
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = @"SELECT COUNT(*) FROM Runs r JOIN Parts p ON r.PartId = p.Id
WHERE p.FileName = @name AND r.SheetWidth = @w AND r.SheetHeight = @h AND r.Spacing = @s";
cmd.Parameters.AddWithValue("@name", fileName);
cmd.Parameters.AddWithValue("@w", sheetWidth);
cmd.Parameters.AddWithValue("@h", sheetHeight);
cmd.Parameters.AddWithValue("@s", spacing);
return (long)cmd.ExecuteScalar() > 0;
}
public int RunCount(string fileName)
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM Runs r JOIN Parts p ON r.PartId = p.Id WHERE p.FileName = @name";
cmd.Parameters.AddWithValue("@name", fileName);
return (int)(long)cmd.ExecuteScalar();
}
public void AddRun(long partId, double w, double h, double s, BruteForceResult result, string filePath)
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = @"
INSERT INTO Runs (PartId, SheetWidth, SheetHeight, Spacing, PartCount, Utilization, TimeMs, LayoutData, FilePath)
VALUES (@pid, @w, @h, @s, @cnt, @util, @time, @layout, @path)";
cmd.Parameters.AddWithValue("@pid", partId);
cmd.Parameters.AddWithValue("@w", w);
cmd.Parameters.AddWithValue("@h", h);
cmd.Parameters.AddWithValue("@s", s);
cmd.Parameters.AddWithValue("@cnt", result.PartCount);
cmd.Parameters.AddWithValue("@util", result.Utilization);
cmd.Parameters.AddWithValue("@time", result.TimeMs);
cmd.Parameters.AddWithValue("@layout", result.LayoutData ?? "");
cmd.Parameters.AddWithValue("@path", filePath ?? "");
cmd.ExecuteNonQuery();
}
public SqliteTransaction BeginTransaction()
{
return _connection.BeginTransaction();
}
public void Dispose()
{
_connection?.Dispose();
}
}
}