refactor: extract training data collection into OpenNest.Training

Move brute-force data collection, TrainingDatabase, and GPU init from OpenNest.Console into a dedicated OpenNest.Training project. Replaces raw Microsoft.Data.Sqlite with EF Core. Console is now a pure nesting CLI with template support and cleaned-up usage output. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-14 14:41:38 -04:00
parent 74272bea80
commit acc75868c0
10 changed files with 591 additions and 428 deletions
@@ -0,0 +1,38 @@
+using Microsoft.EntityFrameworkCore;
+
+namespace OpenNest.Training.Data
+{
+    public class TrainingDbContext : DbContext
+    {
+        public DbSet<TrainingPart> Parts { get; set; }
+        public DbSet<TrainingRun> Runs { get; set; }
+
+        private readonly string _dbPath;
+
+        public TrainingDbContext(string dbPath)
+        {
+            _dbPath = dbPath;
+        }
+
+        protected override void OnConfiguring(DbContextOptionsBuilder options)
+        {
+            options.UseSqlite($"Data Source={_dbPath}");
+        }
+
+        protected override void OnModelCreating(ModelBuilder modelBuilder)
+        {
+            modelBuilder.Entity<TrainingPart>(e =>
+            {
+                e.HasIndex(p => p.FileName).HasDatabaseName("idx_parts_filename");
+            });
+
+            modelBuilder.Entity<TrainingRun>(e =>
+            {
+                e.HasIndex(r => r.PartId).HasDatabaseName("idx_runs_partid");
+                e.HasOne(r => r.Part)
+                    .WithMany(p => p.Runs)
+                    .HasForeignKey(r => r.PartId);
+            });
+        }
+    }
+}
@@ -0,0 +1,28 @@
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace OpenNest.Training.Data
+{
+    [Table("Parts")]
+    public class TrainingPart
+    {
+        [Key]
+        public long Id { get; set; }
+
+        [MaxLength(260)]
+        public string FileName { get; set; }
+
+        public double Area { get; set; }
+        public double Convexity { get; set; }
+        public double AspectRatio { get; set; }
+        public double BBFill { get; set; }
+        public double Circularity { get; set; }
+        public double PerimeterToAreaRatio { get; set; }
+        public int VertexCount { get; set; }
+        public byte[] Bitmask { get; set; }
+        public string GeometryData { get; set; }
+
+        public List<TrainingRun> Runs { get; set; } = new();
+    }
+}
@@ -0,0 +1,25 @@
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace OpenNest.Training.Data
+{
+    [Table("Runs")]
+    public class TrainingRun
+    {
+        [Key]
+        public long Id { get; set; }
+
+        public long PartId { get; set; }
+        public double SheetWidth { get; set; }
+        public double SheetHeight { get; set; }
+        public double Spacing { get; set; }
+        public int PartCount { get; set; }
+        public double Utilization { get; set; }
+        public long TimeMs { get; set; }
+        public string LayoutData { get; set; }
+        public string FilePath { get; set; }
+
+        [ForeignKey(nameof(PartId))]
+        public TrainingPart Part { get; set; }
+    }
+}
@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0-windows</TargetFramework>
+    <RootNamespace>OpenNest.Training</RootNamespace>
+    <AssemblyName>OpenNest.Training</AssemblyName>
+    <DefineConstants>$(DefineConstants);DEBUG;TRACE</DefineConstants>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\OpenNest.Core\OpenNest.Core.csproj" />
+    <ProjectReference Include="..\OpenNest.Engine\OpenNest.Engine.csproj" />
+    <ProjectReference Include="..\OpenNest.IO\OpenNest.IO.csproj" />
+    <ProjectReference Include="..\OpenNest.Gpu\OpenNest.Gpu.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.10" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.10" />
+  </ItemGroup>
+</Project>
@@ -0,0 +1,300 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using OpenNest;
+using OpenNest.Geometry;
+using OpenNest.IO;
+using Color = System.Drawing.Color;
+using OpenNest.Engine.BestFit;
+using OpenNest.Engine.ML;
+using OpenNest.Gpu;
+using OpenNest.Training;
+
+// Parse arguments.
+var dbPath = "OpenNestTraining";
+var saveNestsDir = (string)null;
+var templateFile = (string)null;
+var spacing = 0.5;
+var collectDir = (string)null;
+
+for (var i = 0; i < args.Length; i++)
+{
+    switch (args[i])
+    {
+        case "--db" when i + 1 < args.Length:
+            dbPath = args[++i];
+            break;
+        case "--save-nests" when i + 1 < args.Length:
+            saveNestsDir = args[++i];
+            break;
+        case "--template" when i + 1 < args.Length:
+            templateFile = args[++i];
+            break;
+        case "--spacing" when i + 1 < args.Length:
+            spacing = double.Parse(args[++i]);
+            break;
+        case "--help":
+        case "-h":
+            PrintUsage();
+            return 0;
+        default:
+            if (!args[i].StartsWith("--") && collectDir == null)
+                collectDir = args[i];
+            break;
+    }
+}
+
+if (string.IsNullOrEmpty(collectDir) || !Directory.Exists(collectDir))
+{
+    PrintUsage();
+    return 1;
+}
+
+// Initialize GPU if available.
+if (GpuEvaluatorFactory.GpuAvailable)
+{
+    BestFitCache.CreateSlideComputer = () => GpuEvaluatorFactory.CreateSlideComputer();
+    Console.WriteLine($"GPU: {GpuEvaluatorFactory.DeviceName}");
+}
+else
+{
+    Console.WriteLine("GPU: not available (using CPU)");
+}
+
+return RunDataCollection(collectDir, dbPath, saveNestsDir, spacing, templateFile);
+
+int RunDataCollection(string dir, string dbPath, string saveDir, double s, string template)
+{
+    // Load template nest for plate defaults if provided.
+    Nest templateNest = null;
+    if (template != null)
+    {
+        if (!File.Exists(template))
+        {
+            Console.Error.WriteLine($"Error: Template not found: {template}");
+            return 1;
+        }
+        templateNest = new NestReader(template).Read();
+        Console.WriteLine($"Using template: {template}");
+    }
+
+    var PartColors = new[]
+    {
+        Color.FromArgb(205, 92, 92),
+        Color.FromArgb(148, 103, 189),
+        Color.FromArgb(75, 180, 175),
+        Color.FromArgb(210, 190, 75),
+        Color.FromArgb(190, 85, 175),
+        Color.FromArgb(185, 115, 85),
+        Color.FromArgb(120, 100, 190),
+        Color.FromArgb(200, 100, 140),
+        Color.FromArgb(80, 175, 155),
+        Color.FromArgb(195, 160, 85),
+        Color.FromArgb(175, 95, 160),
+        Color.FromArgb(215, 130, 130),
+    };
+
+    var sheetSuite = new[]
+    {
+        new Size(96, 48), new Size(120, 48), new Size(144, 48),
+        new Size(96, 60), new Size(120, 60), new Size(144, 60),
+        new Size(96, 72), new Size(120, 72), new Size(144, 72),
+        new Size(48, 24), new Size(120, 10)
+    };
+
+    var dxfFiles = Directory.GetFiles(dir, "*.dxf", SearchOption.AllDirectories);
+    Console.WriteLine($"Found {dxfFiles.Length} DXF files");
+    var resolvedDb = dbPath.EndsWith(".db", StringComparison.OrdinalIgnoreCase) ? dbPath : dbPath + ".db";
+    Console.WriteLine($"Database: {Path.GetFullPath(resolvedDb)}");
+    Console.WriteLine($"Sheet sizes: {sheetSuite.Length} configurations");
+    Console.WriteLine($"Spacing: {s:F2}");
+    if (saveDir != null) Console.WriteLine($"Saving nests to: {saveDir}");
+    Console.WriteLine("---");
+
+    using var db = new TrainingDatabase(dbPath);
+
+    var backfilled = db.BackfillPerimeterToAreaRatio();
+    if (backfilled > 0)
+        Console.WriteLine($"Backfilled PerimeterToAreaRatio for {backfilled} existing parts");
+
+    var importer = new DxfImporter();
+    var colorIndex = 0;
+    var processed = 0;
+    var skippedGeometry = 0;
+    var skippedFeatures = 0;
+    var skippedExisting = 0;
+    var totalRuns = 0;
+    var totalSw = Stopwatch.StartNew();
+
+    foreach (var file in dxfFiles)
+    {
+        var fileNum = processed + skippedGeometry + skippedFeatures + skippedExisting + 1;
+        var partNo = Path.GetFileNameWithoutExtension(file);
+        Console.Write($"[{fileNum}/{dxfFiles.Length}] {partNo}");
+
+        try
+        {
+            var existingRuns = db.RunCount(Path.GetFileName(file));
+            if (existingRuns >= sheetSuite.Length)
+            {
+                Console.WriteLine(" - SKIP (all sizes done)");
+                skippedExisting++;
+                continue;
+            }
+
+            if (!importer.GetGeometry(file, out var entities))
+            {
+                Console.WriteLine(" - SKIP (no geometry)");
+                skippedGeometry++;
+                continue;
+            }
+
+            var drawing = new Drawing(Path.GetFileName(file));
+            drawing.Program = OpenNest.Converters.ConvertGeometry.ToProgram(entities);
+            drawing.UpdateArea();
+            drawing.Color = PartColors[colorIndex % PartColors.Length];
+            colorIndex++;
+
+            var features = FeatureExtractor.Extract(drawing);
+            if (features == null)
+            {
+                Console.WriteLine(" - SKIP (feature extraction failed)");
+                skippedFeatures++;
+                continue;
+            }
+
+            Console.WriteLine($" (area={features.Area:F1}, verts={features.VertexCount})");
+
+            // Precompute best-fits once for all sheet sizes.
+            var sizes = sheetSuite.Select(sz => (sz.Width, sz.Length)).ToList();
+            var bfSw = Stopwatch.StartNew();
+            BestFitCache.ComputeForSizes(drawing, s, sizes);
+            bfSw.Stop();
+            Console.WriteLine($"  Best-fits computed in {bfSw.ElapsedMilliseconds}ms");
+
+            var partId = db.GetOrAddPart(Path.GetFileName(file), features, drawing.Program.ToString());
+            var partSw = Stopwatch.StartNew();
+            var runsThisPart = 0;
+            var bestUtil = 0.0;
+            var bestCount = 0;
+
+            foreach (var size in sheetSuite)
+            {
+                if (db.HasRun(Path.GetFileName(file), size.Width, size.Length, s))
+                {
+                    Console.WriteLine($"  {size.Length}x{size.Width} - skip (exists)");
+                    continue;
+                }
+
+                Plate runPlate;
+                if (templateNest != null)
+                {
+                    runPlate = templateNest.PlateDefaults.CreateNew();
+                    runPlate.Size = size;
+                    runPlate.PartSpacing = s;
+                }
+                else
+                {
+                    runPlate = new Plate { Size = size, PartSpacing = s };
+                }
+
+                var sizeSw = Stopwatch.StartNew();
+                var result = BruteForceRunner.Run(drawing, runPlate);
+                sizeSw.Stop();
+
+                if (result == null)
+                {
+                    Console.WriteLine($"  {size.Length}x{size.Width} - no fit");
+                    continue;
+                }
+
+                if (result.Utilization > bestUtil)
+                {
+                    bestUtil = result.Utilization;
+                    bestCount = result.PartCount;
+                }
+
+                Console.WriteLine($"  {size.Length}x{size.Width} - {result.PartCount}pcs, {result.Utilization:P1}, {sizeSw.ElapsedMilliseconds}ms");
+
+                string savedFilePath = null;
+                if (saveDir != null)
+                {
+                    // Deterministic bucket (00-FF) based on filename hash
+                    uint hash = 0;
+                    foreach (char c in partNo) hash = (hash * 31) + c;
+                    var bucket = (hash % 256).ToString("X2");
+
+                    var partDir = Path.Combine(saveDir, bucket, partNo);
+                    Directory.CreateDirectory(partDir);
+
+                    var nestName = $"{partNo}-{size.Length}x{size.Width}-{result.PartCount}pcs";
+                    var fileName = nestName + ".zip";
+                    savedFilePath = Path.Combine(partDir, fileName);
+
+                    // Create nest from template or from scratch
+                    Nest nestObj;
+                    if (templateNest != null)
+                    {
+                        nestObj = new Nest(nestName)
+                        {
+                            Units = templateNest.Units,
+                            DateCreated = DateTime.Now
+                        };
+                        nestObj.PlateDefaults.SetFromExisting(templateNest.PlateDefaults.CreateNew());
+                    }
+                    else
+                    {
+                        nestObj = new Nest(nestName) { Units = Units.Inches, DateCreated = DateTime.Now };
+                    }
+
+                    nestObj.Drawings.Add(drawing);
+                    var plateObj = nestObj.CreatePlate();
+                    plateObj.Size = size;
+                    plateObj.PartSpacing = s;
+                    plateObj.Parts.AddRange(result.PlacedParts);
+
+                    var writer = new NestWriter(nestObj);
+                    writer.Write(savedFilePath);
+                }
+
+                db.AddRun(partId, size.Width, size.Length, s, result, savedFilePath);
+                runsThisPart++;
+                totalRuns++;
+            }
+
+            BestFitCache.Invalidate(drawing);
+            partSw.Stop();
+            processed++;
+            Console.WriteLine($"  Total: {runsThisPart} runs, best={bestCount}pcs @ {bestUtil:P1}, {partSw.ElapsedMilliseconds}ms");
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine();
+            Console.Error.WriteLine($"  ERROR: {ex.Message}");
+        }
+    }
+
+    totalSw.Stop();
+    Console.WriteLine("---");
+    Console.WriteLine($"Processed: {processed} parts, {totalRuns} total runs");
+    Console.WriteLine($"Skipped:   {skippedExisting} (existing) + {skippedGeometry} (no geometry) + {skippedFeatures} (no features)");
+    Console.WriteLine($"Time:      {totalSw.Elapsed:h\\:mm\\:ss}");
+    Console.WriteLine($"Database:  {Path.GetFullPath(resolvedDb)}");
+    return 0;
+}
+
+void PrintUsage()
+{
+    Console.Error.WriteLine("Usage: OpenNest.Training <dxf-dir> [options]");
+    Console.Error.WriteLine();
+    Console.Error.WriteLine("Arguments:");
+    Console.Error.WriteLine("  dxf-dir                Directory containing DXF files to process");
+    Console.Error.WriteLine();
+    Console.Error.WriteLine("Options:");
+    Console.Error.WriteLine("  --spacing <value>      Part spacing (default: 0.5)");
+    Console.Error.WriteLine("  --db <path>            SQLite database path (default: OpenNestTraining.db)");
+    Console.Error.WriteLine("  --save-nests <dir>     Directory to save individual .zip nests for each winner");
+    Console.Error.WriteLine("  --template <path>      Nest template (.nstdot) for plate defaults");
+    Console.Error.WriteLine("  -h, --help             Show this help");
+}
@@ -0,0 +1,131 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Microsoft.EntityFrameworkCore;
+using OpenNest.Engine.ML;
+using OpenNest.IO;
+using OpenNest.Training.Data;
+
+namespace OpenNest.Training
+{
+    public class TrainingDatabase : IDisposable
+    {
+        private readonly TrainingDbContext _db;
+
+        public TrainingDatabase(string dbPath)
+        {
+            if (!dbPath.EndsWith(".db", StringComparison.OrdinalIgnoreCase))
+                dbPath += ".db";
+
+            _db = new TrainingDbContext(dbPath);
+            _db.Database.EnsureCreated();
+        }
+
+        public long GetOrAddPart(string fileName, PartFeatures features, string geometryData)
+        {
+            var existing = _db.Parts.FirstOrDefault(p => p.FileName == fileName);
+            if (existing != null) return existing.Id;
+
+            var part = new TrainingPart
+            {
+                FileName = fileName,
+                Area = features.Area,
+                Convexity = features.Convexity,
+                AspectRatio = features.AspectRatio,
+                BBFill = features.BoundingBoxFill,
+                Circularity = features.Circularity,
+                PerimeterToAreaRatio = features.PerimeterToAreaRatio,
+                VertexCount = features.VertexCount,
+                Bitmask = features.Bitmask,
+                GeometryData = geometryData
+            };
+
+            _db.Parts.Add(part);
+            _db.SaveChanges();
+            return part.Id;
+        }
+
+        public bool HasRun(string fileName, double sheetWidth, double sheetHeight, double spacing)
+        {
+            return _db.Runs.Any(r =>
+                r.Part.FileName == fileName &&
+                r.SheetWidth == sheetWidth &&
+                r.SheetHeight == sheetHeight &&
+                r.Spacing == spacing);
+        }
+
+        public int RunCount(string fileName)
+        {
+            return _db.Runs.Count(r => r.Part.FileName == fileName);
+        }
+
+        public void AddRun(long partId, double w, double h, double s, BruteForceResult result, string filePath)
+        {
+            var run = new TrainingRun
+            {
+                PartId = partId,
+                SheetWidth = w,
+                SheetHeight = h,
+                Spacing = s,
+                PartCount = result.PartCount,
+                Utilization = result.Utilization,
+                TimeMs = result.TimeMs,
+                LayoutData = result.LayoutData ?? "",
+                FilePath = filePath ?? ""
+            };
+
+            _db.Runs.Add(run);
+            _db.SaveChanges();
+        }
+
+        public int BackfillPerimeterToAreaRatio()
+        {
+            var partsToFix = _db.Parts
+                .Where(p => p.PerimeterToAreaRatio == 0)
+                .Select(p => new { p.Id, p.GeometryData })
+                .ToList();
+
+            if (partsToFix.Count == 0) return 0;
+
+            var updated = 0;
+            foreach (var item in partsToFix)
+            {
+                try
+                {
+                    var stream = new MemoryStream(Encoding.UTF8.GetBytes(item.GeometryData));
+                    var programReader = new ProgramReader(stream);
+                    var program = programReader.Read();
+
+                    var drawing = new Drawing("backfill") { Program = program };
+                    drawing.UpdateArea();
+
+                    var features = FeatureExtractor.Extract(drawing);
+                    if (features == null) continue;
+
+                    var part = _db.Parts.Find(item.Id);
+                    part.PerimeterToAreaRatio = features.PerimeterToAreaRatio;
+                    _db.SaveChanges();
+                    updated++;
+                }
+                catch
+                {
+                    // Skip parts that fail to reconstruct.
+                }
+            }
+
+            return updated;
+        }
+
+        public void SaveChanges()
+        {
+            _db.SaveChanges();
+        }
+
+        public void Dispose()
+        {
+            _db?.Dispose();
+        }
+    }
+}