aj/CutList
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: replace generic catalog DTOs with shape-typed DTOs for type safety

Browse Source 
Replace the single CatalogMaterialDto + CatalogDimensionsDto (bag of nullable
fields) with per-shape DTOs that have strongly-typed dimension properties.
Catalog JSON now groups materials by shape key instead of a flat array.
Delete the old SeedController/SeedDataDtos (superseded by CatalogService).
Scraper updated to emit the new grouped format, resume by default, and
save items incrementally.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
AJ Isaacs
2026-02-25 15:48:35 -05:00
parent c31769a746
commit 7d3c92226c
8 changed files with 13874 additions and 15830 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
CutList.Web/Controllers/Dtos/SeedDataDtos.cs
View File

@@ -1,72 +0,0 @@
using System.Text.Json.Serialization;
namespace CutList.Web.Controllers.Dtos;
public class SeedExportData
{
public DateTime ExportedAt { get; set; }
public List<SeedSupplierDto> Suppliers { get; set; } = [];
public List<SeedCuttingToolDto> CuttingTools { get; set; } = [];
public List<SeedMaterialDto> Materials { get; set; } = [];
}
public class SeedSupplierDto
{
public string Name { get; set; } = "";
public string? ContactInfo { get; set; }
public string? Notes { get; set; }
}
public class SeedCuttingToolDto
{
public string Name { get; set; } = "";
public decimal KerfInches { get; set; }
public bool IsDefault { get; set; }
}
public class SeedMaterialDto
{
public string Shape { get; set; } = "";
public string Type { get; set; } = "";
public string? Grade { get; set; }
public string Size { get; set; } = "";
public string? Description { get; set; }
public SeedDimensionsDto? Dimensions { get; set; }
public List<SeedStockItemDto> StockItems { get; set; } = [];
}
public class SeedDimensionsDto
{
public decimal? Diameter { get; set; }
public decimal? OuterDiameter { get; set; }
public decimal? Width { get; set; }
public decimal? Height { get; set; }
public decimal? Thickness { get; set; }
public decimal? Wall { get; set; }
public decimal? Size { get; set; }
public decimal? Leg1 { get; set; }
public decimal? Leg2 { get; set; }
public decimal? Flange { get; set; }
public decimal? Web { get; set; }
public decimal? WeightPerFoot { get; set; }
public decimal? NominalSize { get; set; }
public string? Schedule { get; set; }
}
public class SeedStockItemDto
{
public decimal LengthInches { get; set; }
public string? Name { get; set; }
public int QuantityOnHand { get; set; }
public string? Notes { get; set; }
public List<SeedSupplierOfferingDto> SupplierOfferings { get; set; } = [];
}
public class SeedSupplierOfferingDto
{
public string SupplierName { get; set; } = "";
public string? PartNumber { get; set; }
public string? SupplierDescription { get; set; }
public decimal? Price { get; set; }
public string? Notes { get; set; }
}

277
CutList.Web/Controllers/SeedController.cs
View File

@@ -1,277 +0,0 @@
using CutList.Web.Controllers.Dtos;
using CutList.Web.Data;
using CutList.Web.Data.Entities;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
namespace CutList.Web.Controllers;
[ApiController]
[Route("api/[controller]")]
public class SeedController : ControllerBase
{
private readonly ApplicationDbContext _context;
public SeedController(ApplicationDbContext context)
{
_context = context;
}
[HttpGet("export")]
public async Task<ActionResult<SeedExportData>> Export()
{
var materials = await _context.Materials
.Include(m => m.Dimensions)
.Include(m => m.StockItems.Where(s => s.IsActive))
.ThenInclude(s => s.SupplierOfferings.Where(o => o.IsActive))
.Where(m => m.IsActive)
.OrderBy(m => m.Shape).ThenBy(m => m.SortOrder)
.AsNoTracking()
.ToListAsync();
var suppliers = await _context.Suppliers
.Where(s => s.IsActive)
.OrderBy(s => s.Name)
.AsNoTracking()
.ToListAsync();
var cuttingTools = await _context.CuttingTools
.Where(t => t.IsActive)
.OrderBy(t => t.Name)
.AsNoTracking()
.ToListAsync();
var export = new SeedExportData
{
ExportedAt = DateTime.UtcNow,
Suppliers = suppliers.Select(s => new SeedSupplierDto
{
Name = s.Name,
ContactInfo = s.ContactInfo,
Notes = s.Notes
}).ToList(),
CuttingTools = cuttingTools.Select(t => new SeedCuttingToolDto
{
Name = t.Name,
KerfInches = t.KerfInches,
IsDefault = t.IsDefault
}).ToList(),
Materials = materials.Select(m => new SeedMaterialDto
{
Shape = m.Shape.ToString(),
Type = m.Type.ToString(),
Grade = m.Grade,
Size = m.Size,
Description = m.Description,
Dimensions = MapDimensionsToDto(m.Dimensions),
StockItems = m.StockItems.OrderBy(s => s.LengthInches).Select(s => new SeedStockItemDto
{
LengthInches = s.LengthInches,
Name = s.Name,
QuantityOnHand = s.QuantityOnHand,
Notes = s.Notes,
SupplierOfferings = s.SupplierOfferings.Select(o => new SeedSupplierOfferingDto
{
SupplierName = suppliers.FirstOrDefault(sup => sup.Id == o.SupplierId)?.Name ?? "Unknown",
PartNumber = o.PartNumber,
SupplierDescription = o.SupplierDescription,
Price = o.Price,
Notes = o.Notes
}).ToList()
}).ToList()
}).ToList()
};
return Ok(export);
}
[HttpPost("import")]
public async Task<ActionResult> Import([FromBody] SeedExportData data)
{
var suppliersCreated = 0;
var toolsCreated = 0;
var materialsCreated = 0;
var materialsSkipped = 0;
var stockCreated = 0;
var offeringsCreated = 0;
// 1. Suppliers - match by name
var supplierMap = new Dictionary<string, Supplier>();
foreach (var dto in data.Suppliers)
{
var existing = await _context.Suppliers.FirstOrDefaultAsync(s => s.Name == dto.Name);
if (existing != null)
{
supplierMap[dto.Name] = existing;
}
else
{
var supplier = new Supplier
{
Name = dto.Name,
ContactInfo = dto.ContactInfo,
Notes = dto.Notes,
CreatedAt = DateTime.UtcNow
};
_context.Suppliers.Add(supplier);
supplierMap[dto.Name] = supplier;
suppliersCreated++;
}
}
await _context.SaveChangesAsync();
// 2. Cutting tools - match by name
foreach (var dto in data.CuttingTools)
{
var exists = await _context.CuttingTools.AnyAsync(t => t.Name == dto.Name);
if (!exists)
{
_context.CuttingTools.Add(new CuttingTool
{
Name = dto.Name,
KerfInches = dto.KerfInches,
IsDefault = dto.IsDefault
});
toolsCreated++;
}
}
await _context.SaveChangesAsync();
// 3. Materials - match by shape + size + grade
foreach (var dto in data.Materials)
{
if (!Enum.TryParse<MaterialShape>(dto.Shape, out var shape))
{
materialsSkipped++;
continue;
}
Enum.TryParse<MaterialType>(dto.Type, out var type);
var existing = await _context.Materials
.Include(m => m.StockItems)
.FirstOrDefaultAsync(m => m.Shape == shape && m.Size == dto.Size && m.Grade == dto.Grade && m.IsActive);
Material material;
if (existing != null)
{
material = existing;
materialsSkipped++;
}
else
{
material = new Material
{
Shape = shape,
Type = type,
Grade = dto.Grade,
Size = dto.Size,
Description = dto.Description,
CreatedAt = DateTime.UtcNow
};
if (dto.Dimensions != null)
material.Dimensions = MapDtoToDimensions(shape, dto.Dimensions);
_context.Materials.Add(material);
await _context.SaveChangesAsync();
materialsCreated++;
}
// 4. Stock items - match by material + length
foreach (var stockDto in dto.StockItems)
{
var existingStock = material.StockItems
.FirstOrDefault(s => s.LengthInches == stockDto.LengthInches && s.IsActive);
StockItem stockItem;
if (existingStock != null)
{
stockItem = existingStock;
}
else
{
stockItem = new StockItem
{
MaterialId = material.Id,
LengthInches = stockDto.LengthInches,
Name = stockDto.Name,
QuantityOnHand = stockDto.QuantityOnHand,
Notes = stockDto.Notes,
CreatedAt = DateTime.UtcNow
};
_context.StockItems.Add(stockItem);
await _context.SaveChangesAsync();
stockCreated++;
}
// 5. Supplier offerings
foreach (var offeringDto in stockDto.SupplierOfferings)
{
if (!supplierMap.TryGetValue(offeringDto.SupplierName, out var supplier))
continue;
var existingOffering = await _context.SupplierOfferings
.AnyAsync(o => o.SupplierId == supplier.Id && o.StockItemId == stockItem.Id);
if (!existingOffering)
{
_context.SupplierOfferings.Add(new SupplierOffering
{
SupplierId = supplier.Id,
StockItemId = stockItem.Id,
PartNumber = offeringDto.PartNumber,
SupplierDescription = offeringDto.SupplierDescription,
Price = offeringDto.Price,
Notes = offeringDto.Notes
});
offeringsCreated++;
}
}
}
}
await _context.SaveChangesAsync();
return Ok(new
{
Message = "Import completed",
SuppliersCreated = suppliersCreated,
CuttingToolsCreated = toolsCreated,
MaterialsCreated = materialsCreated,
MaterialsSkipped = materialsSkipped,
StockItemsCreated = stockCreated,
SupplierOfferingsCreated = offeringsCreated
});
}
private static SeedDimensionsDto? MapDimensionsToDto(MaterialDimensions? dim) => dim switch
{
RoundBarDimensions d => new SeedDimensionsDto { Diameter = d.Diameter },
RoundTubeDimensions d => new SeedDimensionsDto { OuterDiameter = d.OuterDiameter, Wall = d.Wall },
FlatBarDimensions d => new SeedDimensionsDto { Width = d.Width, Thickness = d.Thickness },
SquareBarDimensions d => new SeedDimensionsDto { Size = d.Size },
SquareTubeDimensions d => new SeedDimensionsDto { Size = d.Size, Wall = d.Wall },
RectangularTubeDimensions d => new SeedDimensionsDto { Width = d.Width, Height = d.Height, Wall = d.Wall },
AngleDimensions d => new SeedDimensionsDto { Leg1 = d.Leg1, Leg2 = d.Leg2, Thickness = d.Thickness },
ChannelDimensions d => new SeedDimensionsDto { Height = d.Height, Flange = d.Flange, Web = d.Web },
IBeamDimensions d => new SeedDimensionsDto { Height = d.Height, WeightPerFoot = d.WeightPerFoot },
PipeDimensions d => new SeedDimensionsDto { NominalSize = d.NominalSize, Wall = d.Wall, Schedule = d.Schedule },
_ => null
};
private static MaterialDimensions? MapDtoToDimensions(MaterialShape shape, SeedDimensionsDto dto) => shape switch
{
MaterialShape.RoundBar => new RoundBarDimensions { Diameter = dto.Diameter ?? 0 },
MaterialShape.RoundTube => new RoundTubeDimensions { OuterDiameter = dto.OuterDiameter ?? 0, Wall = dto.Wall ?? 0 },
MaterialShape.FlatBar => new FlatBarDimensions { Width = dto.Width ?? 0, Thickness = dto.Thickness ?? 0 },
MaterialShape.SquareBar => new SquareBarDimensions { Size = dto.Size ?? 0 },
MaterialShape.SquareTube => new SquareTubeDimensions { Size = dto.Size ?? 0, Wall = dto.Wall ?? 0 },
MaterialShape.RectangularTube => new RectangularTubeDimensions { Width = dto.Width ?? 0, Height = dto.Height ?? 0, Wall = dto.Wall ?? 0 },
MaterialShape.Angle => new AngleDimensions { Leg1 = dto.Leg1 ?? 0, Leg2 = dto.Leg2 ?? 0, Thickness = dto.Thickness ?? 0 },
MaterialShape.Channel => new ChannelDimensions { Height = dto.Height ?? 0, Flange = dto.Flange ?? 0, Web = dto.Web ?? 0 },
MaterialShape.IBeam => new IBeamDimensions { Height = dto.Height ?? 0, WeightPerFoot = dto.WeightPerFoot ?? 0 },
MaterialShape.Pipe => new PipeDimensions { NominalSize = dto.NominalSize ?? 0, Wall = dto.Wall ?? 0, Schedule = dto.Schedule },
_ => null
};
}

92
CutList.Web/DTOs/CatalogDtos.cs
View File

@@ -5,7 +5,7 @@ public class CatalogData
public DateTime ExportedAt { get; set; } public DateTime ExportedAt { get; set; }
public List<CatalogSupplierDto> Suppliers { get; set; } = []; public List<CatalogSupplierDto> Suppliers { get; set; } = [];
public List<CatalogCuttingToolDto> CuttingTools { get; set; } = []; public List<CatalogCuttingToolDto> CuttingTools { get; set; } = [];
public List<CatalogMaterialDto> Materials { get; set; } = []; public CatalogMaterialsDto Materials { get; set; } = new();
} }
public class CatalogSupplierDto public class CatalogSupplierDto
@@ -22,35 +22,91 @@ public class CatalogCuttingToolDto
public bool IsDefault { get; set; } public bool IsDefault { get; set; }
} }
public class CatalogMaterialDto public class CatalogMaterialsDto
{
public List<CatalogAngleDto> Angles { get; set; } = [];
public List<CatalogChannelDto> Channels { get; set; } = [];
public List<CatalogFlatBarDto> FlatBars { get; set; } = [];
public List<CatalogIBeamDto> IBeams { get; set; } = [];
public List<CatalogPipeDto> Pipes { get; set; } = [];
public List<CatalogRectangularTubeDto> RectangularTubes { get; set; } = [];
public List<CatalogRoundBarDto> RoundBars { get; set; } = [];
public List<CatalogRoundTubeDto> RoundTubes { get; set; } = [];
public List<CatalogSquareBarDto> SquareBars { get; set; } = [];
public List<CatalogSquareTubeDto> SquareTubes { get; set; } = [];
}
public abstract class CatalogMaterialBaseDto
{ {
public string Shape { get; set; } = "";
public string Type { get; set; } = ""; public string Type { get; set; } = "";
public string? Grade { get; set; } public string? Grade { get; set; }
public string Size { get; set; } = ""; public string Size { get; set; } = "";
public string? Description { get; set; } public string? Description { get; set; }
public CatalogDimensionsDto? Dimensions { get; set; }
public List<CatalogStockItemDto> StockItems { get; set; } = []; public List<CatalogStockItemDto> StockItems { get; set; } = [];
} }
public class CatalogDimensionsDto public class CatalogAngleDto : CatalogMaterialBaseDto
{ {
public decimal? Diameter { get; set; } public decimal Leg1 { get; set; }
public decimal? OuterDiameter { get; set; } public decimal Leg2 { get; set; }
public decimal? Width { get; set; } public decimal Thickness { get; set; }
public decimal? Height { get; set; } }
public decimal? Thickness { get; set; }
public decimal? Wall { get; set; } public class CatalogChannelDto : CatalogMaterialBaseDto
public decimal? Size { get; set; } {
public decimal? Leg1 { get; set; } public decimal Height { get; set; }
public decimal? Leg2 { get; set; } public decimal Flange { get; set; }
public decimal? Flange { get; set; } public decimal Web { get; set; }
public decimal? Web { get; set; } }
public decimal? WeightPerFoot { get; set; }
public decimal? NominalSize { get; set; } public class CatalogFlatBarDto : CatalogMaterialBaseDto
{
public decimal Width { get; set; }
public decimal Thickness { get; set; }
}
public class CatalogIBeamDto : CatalogMaterialBaseDto
{
public decimal Height { get; set; }
public decimal WeightPerFoot { get; set; }
}
public class CatalogPipeDto : CatalogMaterialBaseDto
{
public decimal NominalSize { get; set; }
public decimal Wall { get; set; }
public string? Schedule { get; set; } public string? Schedule { get; set; }
} }
public class CatalogRectangularTubeDto : CatalogMaterialBaseDto
{
public decimal Width { get; set; }
public decimal Height { get; set; }
public decimal Wall { get; set; }
}
public class CatalogRoundBarDto : CatalogMaterialBaseDto
{
public decimal Diameter { get; set; }
}
public class CatalogRoundTubeDto : CatalogMaterialBaseDto
{
public decimal OuterDiameter { get; set; }
public decimal Wall { get; set; }
}
public class CatalogSquareBarDto : CatalogMaterialBaseDto
{
public decimal SideLength { get; set; }
}
public class CatalogSquareTubeDto : CatalogMaterialBaseDto
{
public decimal SideLength { get; set; }
public decimal Wall { get; set; }
}
public class CatalogStockItemDto public class CatalogStockItemDto
{ {
public decimal LengthInches { get; set; } public decimal LengthInches { get; set; }

13
CutList.Web/Data/SeedData/alro-catalog.json
View File

@@ -27,5 +27,16 @@
"isDefault": false "isDefault": false
} }
], ],
"materials": [] "materials": {
"angles": [],
"channels": [],
"flatBars": [],
"iBeams": [],
"pipes": [],
"rectangularTubes": [],
"roundBars": [],
"roundTubes": [],
"squareBars": [],
"squareTubes": []
}
} }

28720
CutList.Web/Data/SeedData/oneals-catalog.json
View File

File diff suppressed because it is too large Load Diff

393
CutList.Web/Services/CatalogService.cs
View File

@@ -39,6 +39,101 @@ public class CatalogService
.AsNoTracking() .AsNoTracking()
.ToListAsync(); .ToListAsync();
var grouped = materials.GroupBy(m => m.Shape);
var materialsDto = new CatalogMaterialsDto();
foreach (var group in grouped)
{
foreach (var m in group)
{
var stockItems = MapStockItems(m, suppliers);
switch (m.Shape)
{
case MaterialShape.Angle when m.Dimensions is AngleDimensions d:
materialsDto.Angles.Add(new CatalogAngleDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
Leg1 = d.Leg1, Leg2 = d.Leg2, Thickness = d.Thickness,
StockItems = stockItems
});
break;
case MaterialShape.Channel when m.Dimensions is ChannelDimensions d:
materialsDto.Channels.Add(new CatalogChannelDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
Height = d.Height, Flange = d.Flange, Web = d.Web,
StockItems = stockItems
});
break;
case MaterialShape.FlatBar when m.Dimensions is FlatBarDimensions d:
materialsDto.FlatBars.Add(new CatalogFlatBarDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
Width = d.Width, Thickness = d.Thickness,
StockItems = stockItems
});
break;
case MaterialShape.IBeam when m.Dimensions is IBeamDimensions d:
materialsDto.IBeams.Add(new CatalogIBeamDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
Height = d.Height, WeightPerFoot = d.WeightPerFoot,
StockItems = stockItems
});
break;
case MaterialShape.Pipe when m.Dimensions is PipeDimensions d:
materialsDto.Pipes.Add(new CatalogPipeDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
NominalSize = d.NominalSize, Wall = d.Wall ?? 0, Schedule = d.Schedule,
StockItems = stockItems
});
break;
case MaterialShape.RectangularTube when m.Dimensions is RectangularTubeDimensions d:
materialsDto.RectangularTubes.Add(new CatalogRectangularTubeDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
Width = d.Width, Height = d.Height, Wall = d.Wall,
StockItems = stockItems
});
break;
case MaterialShape.RoundBar when m.Dimensions is RoundBarDimensions d:
materialsDto.RoundBars.Add(new CatalogRoundBarDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
Diameter = d.Diameter,
StockItems = stockItems
});
break;
case MaterialShape.RoundTube when m.Dimensions is RoundTubeDimensions d:
materialsDto.RoundTubes.Add(new CatalogRoundTubeDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
OuterDiameter = d.OuterDiameter, Wall = d.Wall,
StockItems = stockItems
});
break;
case MaterialShape.SquareBar when m.Dimensions is SquareBarDimensions d:
materialsDto.SquareBars.Add(new CatalogSquareBarDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
SideLength = d.Size,
StockItems = stockItems
});
break;
case MaterialShape.SquareTube when m.Dimensions is SquareTubeDimensions d:
materialsDto.SquareTubes.Add(new CatalogSquareTubeDto
{
Type = m.Type.ToString(), Grade = m.Grade, Size = m.Size, Description = m.Description,
SideLength = d.Size, Wall = d.Wall,
StockItems = stockItems
});
break;
}
}
}
return new CatalogData return new CatalogData
{ {
ExportedAt = DateTime.UtcNow, ExportedAt = DateTime.UtcNow,
@@ -54,30 +149,7 @@ public class CatalogService
KerfInches = t.KerfInches, KerfInches = t.KerfInches,
IsDefault = t.IsDefault IsDefault = t.IsDefault
}).ToList(), }).ToList(),
Materials = materials.Select(m => new CatalogMaterialDto Materials = materialsDto
{
Shape = m.Shape.ToString(),
Type = m.Type.ToString(),
Grade = m.Grade,
Size = m.Size,
Description = m.Description,
Dimensions = MapDimensions(m.Dimensions),
StockItems = m.StockItems.OrderBy(s => s.LengthInches).Select(s => new CatalogStockItemDto
{
LengthInches = s.LengthInches,
Name = s.Name,
QuantityOnHand = s.QuantityOnHand,
Notes = s.Notes,
SupplierOfferings = s.SupplierOfferings.Select(o => new CatalogSupplierOfferingDto
{
SupplierName = suppliers.FirstOrDefault(sup => sup.Id == o.SupplierId)?.Name ?? "Unknown",
PartNumber = o.PartNumber,
SupplierDescription = o.SupplierDescription,
Price = o.Price,
Notes = o.Notes
}).ToList()
}).ToList()
}).ToList()
}; };
} }
@@ -89,14 +161,14 @@ public class CatalogService
try try
{ {
// 1. Suppliers — upsert by name // 1. Suppliers - upsert by name
var supplierMap = await ImportSuppliersAsync(data.Suppliers, result); var supplierMap = await ImportSuppliersAsync(data.Suppliers, result);
// 2. Cutting tools — upsert by name // 2. Cutting tools - upsert by name
await ImportCuttingToolsAsync(data.CuttingTools, result); await ImportCuttingToolsAsync(data.CuttingTools, result);
// 3. Materials + stock items + offerings // 3. Materials + stock items + offerings
await ImportMaterialsAsync(data.Materials, supplierMap, result); await ImportAllMaterialsAsync(data.Materials, supplierMap, result);
await transaction.CommitAsync(); await transaction.CommitAsync();
} }
@@ -173,7 +245,6 @@ public class CatalogService
{ {
existing.KerfInches = dto.KerfInches; existing.KerfInches = dto.KerfInches;
existing.IsActive = true; existing.IsActive = true;
// Skip IsDefault changes to avoid conflicts
result.CuttingToolsUpdated++; result.CuttingToolsUpdated++;
} }
else else
@@ -182,7 +253,7 @@ public class CatalogService
{ {
Name = dto.Name, Name = dto.Name,
KerfInches = dto.KerfInches, KerfInches = dto.KerfInches,
IsDefault = false // Never import as default to avoid conflicts IsDefault = false
}; };
_context.CuttingTools.Add(tool); _context.CuttingTools.Add(tool);
existingTools.Add(tool); existingTools.Add(tool);
@@ -198,94 +269,128 @@ public class CatalogService
await _context.SaveChangesAsync(); await _context.SaveChangesAsync();
} }
private async Task ImportMaterialsAsync( private async Task ImportAllMaterialsAsync(
List<CatalogMaterialDto> materials, Dictionary<string, int> supplierMap, ImportResultDto result) CatalogMaterialsDto materials, Dictionary<string, int> supplierMap, ImportResultDto result)
{ {
// Pre-load existing materials with their dimensions
var existingMaterials = await _context.Materials var existingMaterials = await _context.Materials
.Include(m => m.Dimensions) .Include(m => m.Dimensions)
.Include(m => m.StockItems) .Include(m => m.StockItems)
.ThenInclude(s => s.SupplierOfferings) .ThenInclude(s => s.SupplierOfferings)
.ToListAsync(); .ToListAsync();
foreach (var dto in materials) foreach (var dto in materials.Angles)
await ImportMaterialAsync(dto, MaterialShape.Angle, existingMaterials, supplierMap, result,
() => new AngleDimensions { Leg1 = dto.Leg1, Leg2 = dto.Leg2, Thickness = dto.Thickness },
dim => { var d = (AngleDimensions)dim; d.Leg1 = dto.Leg1; d.Leg2 = dto.Leg2; d.Thickness = dto.Thickness; });
foreach (var dto in materials.Channels)
await ImportMaterialAsync(dto, MaterialShape.Channel, existingMaterials, supplierMap, result,
() => new ChannelDimensions { Height = dto.Height, Flange = dto.Flange, Web = dto.Web },
dim => { var d = (ChannelDimensions)dim; d.Height = dto.Height; d.Flange = dto.Flange; d.Web = dto.Web; });
foreach (var dto in materials.FlatBars)
await ImportMaterialAsync(dto, MaterialShape.FlatBar, existingMaterials, supplierMap, result,
() => new FlatBarDimensions { Width = dto.Width, Thickness = dto.Thickness },
dim => { var d = (FlatBarDimensions)dim; d.Width = dto.Width; d.Thickness = dto.Thickness; });
foreach (var dto in materials.IBeams)
await ImportMaterialAsync(dto, MaterialShape.IBeam, existingMaterials, supplierMap, result,
() => new IBeamDimensions { Height = dto.Height, WeightPerFoot = dto.WeightPerFoot },
dim => { var d = (IBeamDimensions)dim; d.Height = dto.Height; d.WeightPerFoot = dto.WeightPerFoot; });
foreach (var dto in materials.Pipes)
await ImportMaterialAsync(dto, MaterialShape.Pipe, existingMaterials, supplierMap, result,
() => new PipeDimensions { NominalSize = dto.NominalSize, Wall = dto.Wall, Schedule = dto.Schedule },
dim => { var d = (PipeDimensions)dim; d.NominalSize = dto.NominalSize; d.Wall = (decimal?)dto.Wall; d.Schedule = dto.Schedule; });
foreach (var dto in materials.RectangularTubes)
await ImportMaterialAsync(dto, MaterialShape.RectangularTube, existingMaterials, supplierMap, result,
() => new RectangularTubeDimensions { Width = dto.Width, Height = dto.Height, Wall = dto.Wall },
dim => { var d = (RectangularTubeDimensions)dim; d.Width = dto.Width; d.Height = dto.Height; d.Wall = dto.Wall; });
foreach (var dto in materials.RoundBars)
await ImportMaterialAsync(dto, MaterialShape.RoundBar, existingMaterials, supplierMap, result,
() => new RoundBarDimensions { Diameter = dto.Diameter },
dim => { var d = (RoundBarDimensions)dim; d.Diameter = dto.Diameter; });
foreach (var dto in materials.RoundTubes)
await ImportMaterialAsync(dto, MaterialShape.RoundTube, existingMaterials, supplierMap, result,
() => new RoundTubeDimensions { OuterDiameter = dto.OuterDiameter, Wall = dto.Wall },
dim => { var d = (RoundTubeDimensions)dim; d.OuterDiameter = dto.OuterDiameter; d.Wall = dto.Wall; });
foreach (var dto in materials.SquareBars)
await ImportMaterialAsync(dto, MaterialShape.SquareBar, existingMaterials, supplierMap, result,
() => new SquareBarDimensions { Size = dto.SideLength },
dim => { var d = (SquareBarDimensions)dim; d.Size = dto.SideLength; });
foreach (var dto in materials.SquareTubes)
await ImportMaterialAsync(dto, MaterialShape.SquareTube, existingMaterials, supplierMap, result,
() => new SquareTubeDimensions { Size = dto.SideLength, Wall = dto.Wall },
dim => { var d = (SquareTubeDimensions)dim; d.Size = dto.SideLength; d.Wall = dto.Wall; });
}
private async Task ImportMaterialAsync(
CatalogMaterialBaseDto dto, MaterialShape shape,
List<Material> existingMaterials, Dictionary<string, int> supplierMap,
ImportResultDto result,
Func<MaterialDimensions> createDimensions,
Action<MaterialDimensions> updateDimensions)
{
try
{ {
try if (!Enum.TryParse<MaterialType>(dto.Type, ignoreCase: true, out var type))
{ {
if (!Enum.TryParse<MaterialShape>(dto.Shape, ignoreCase: true, out var shape)) type = MaterialType.Steel;
{ result.Warnings.Add($"Material '{shape} - {dto.Size}': Unknown type '{dto.Type}', defaulting to Steel");
result.Errors.Add($"Material '{dto.Shape} - {dto.Size}': Unknown shape '{dto.Shape}'");
continue;
}
if (!Enum.TryParse<MaterialType>(dto.Type, ignoreCase: true, out var type))
{
type = MaterialType.Steel; // Default
result.Warnings.Add($"Material '{dto.Shape} - {dto.Size}': Unknown type '{dto.Type}', defaulting to Steel");
}
var existing = existingMaterials.FirstOrDefault(
m => m.Shape == shape && m.Size.Equals(dto.Size, StringComparison.OrdinalIgnoreCase));
Material material;
if (existing != null)
{
// Update existing material
existing.Type = type;
existing.Grade = dto.Grade ?? existing.Grade;
existing.Description = dto.Description ?? existing.Description;
existing.IsActive = true;
existing.UpdatedAt = DateTime.UtcNow;
// Update dimensions if provided
if (dto.Dimensions != null && existing.Dimensions != null)
{
ApplyDimensionValues(existing.Dimensions, dto.Dimensions);
existing.SortOrder = existing.Dimensions.GetSortOrder();
}
material = existing;
result.MaterialsUpdated++;
}
else
{
// Create new material with dimensions
material = new Material
{
Shape = shape,
Type = type,
Grade = dto.Grade,
Size = dto.Size,
Description = dto.Description,
CreatedAt = DateTime.UtcNow
};
if (dto.Dimensions != null)
{
var dimensions = MaterialService.CreateDimensionsForShape(shape);
ApplyDimensionValues(dimensions, dto.Dimensions);
material = await _materialService.CreateWithDimensionsAsync(material, dimensions);
}
else
{
_context.Materials.Add(material);
await _context.SaveChangesAsync();
}
existingMaterials.Add(material);
result.MaterialsCreated++;
}
await _context.SaveChangesAsync();
// Import stock items for this material
await ImportStockItemsAsync(material, dto.StockItems, supplierMap, result);
} }
catch (Exception ex)
var existing = existingMaterials.FirstOrDefault(
m => m.Shape == shape && m.Size.Equals(dto.Size, StringComparison.OrdinalIgnoreCase));
Material material;
if (existing != null)
{ {
result.Errors.Add($"Material '{dto.Shape} - {dto.Size}': {ex.Message}"); existing.Type = type;
existing.Grade = dto.Grade ?? existing.Grade;
existing.Description = dto.Description ?? existing.Description;
existing.IsActive = true;
existing.UpdatedAt = DateTime.UtcNow;
if (existing.Dimensions != null)
{
updateDimensions(existing.Dimensions);
existing.SortOrder = existing.Dimensions.GetSortOrder();
}
material = existing;
result.MaterialsUpdated++;
} }
else
{
material = new Material
{
Shape = shape,
Type = type,
Grade = dto.Grade,
Size = dto.Size,
Description = dto.Description,
CreatedAt = DateTime.UtcNow
};
var dimensions = createDimensions();
material = await _materialService.CreateWithDimensionsAsync(material, dimensions);
existingMaterials.Add(material);
result.MaterialsCreated++;
}
await _context.SaveChangesAsync();
await ImportStockItemsAsync(material, dto.StockItems, supplierMap, result);
}
catch (Exception ex)
{
result.Errors.Add($"Material '{shape} - {dto.Size}': {ex.Message}");
} }
} }
@@ -293,7 +398,6 @@ public class CatalogService
Material material, List<CatalogStockItemDto> stockItems, Material material, List<CatalogStockItemDto> stockItems,
Dictionary<string, int> supplierMap, ImportResultDto result) Dictionary<string, int> supplierMap, ImportResultDto result)
{ {
// Reload stock items for this material to ensure we have current state
var existingStockItems = await _context.StockItems var existingStockItems = await _context.StockItems
.Include(s => s.SupplierOfferings) .Include(s => s.SupplierOfferings)
.Where(s => s.MaterialId == material.Id) .Where(s => s.MaterialId == material.Id)
@@ -314,7 +418,6 @@ public class CatalogService
existing.Notes = dto.Notes ?? existing.Notes; existing.Notes = dto.Notes ?? existing.Notes;
existing.IsActive = true; existing.IsActive = true;
existing.UpdatedAt = DateTime.UtcNow; existing.UpdatedAt = DateTime.UtcNow;
// Don't overwrite QuantityOnHand — preserve actual inventory
stockItem = existing; stockItem = existing;
result.StockItemsUpdated++; result.StockItemsUpdated++;
} }
@@ -335,7 +438,6 @@ public class CatalogService
result.StockItemsCreated++; result.StockItemsCreated++;
} }
// Import supplier offerings
foreach (var offeringDto in dto.SupplierOfferings) foreach (var offeringDto in dto.SupplierOfferings)
{ {
try try
@@ -394,67 +496,22 @@ public class CatalogService
} }
} }
private static CatalogDimensionsDto? MapDimensions(MaterialDimensions? dim) => dim switch private static List<CatalogStockItemDto> MapStockItems(Material m, List<Supplier> suppliers)
{ {
RoundBarDimensions d => new CatalogDimensionsDto { Diameter = d.Diameter }, return m.StockItems.OrderBy(s => s.LengthInches).Select(s => new CatalogStockItemDto
RoundTubeDimensions d => new CatalogDimensionsDto { OuterDiameter = d.OuterDiameter, Wall = d.Wall },
FlatBarDimensions d => new CatalogDimensionsDto { Width = d.Width, Thickness = d.Thickness },
SquareBarDimensions d => new CatalogDimensionsDto { Size = d.Size },
SquareTubeDimensions d => new CatalogDimensionsDto { Size = d.Size, Wall = d.Wall },
RectangularTubeDimensions d => new CatalogDimensionsDto { Width = d.Width, Height = d.Height, Wall = d.Wall },
AngleDimensions d => new CatalogDimensionsDto { Leg1 = d.Leg1, Leg2 = d.Leg2, Thickness = d.Thickness },
ChannelDimensions d => new CatalogDimensionsDto { Height = d.Height, Flange = d.Flange, Web = d.Web },
IBeamDimensions d => new CatalogDimensionsDto { Height = d.Height, WeightPerFoot = d.WeightPerFoot },
PipeDimensions d => new CatalogDimensionsDto { NominalSize = d.NominalSize, Wall = d.Wall, Schedule = d.Schedule },
_ => null
};
private static void ApplyDimensionValues(MaterialDimensions dimensions, CatalogDimensionsDto dto)
{
switch (dimensions)
{ {
case RoundBarDimensions rb: LengthInches = s.LengthInches,
if (dto.Diameter.HasValue) rb.Diameter = dto.Diameter.Value; Name = s.Name,
break; QuantityOnHand = s.QuantityOnHand,
case RoundTubeDimensions rt: Notes = s.Notes,
if (dto.OuterDiameter.HasValue) rt.OuterDiameter = dto.OuterDiameter.Value; SupplierOfferings = s.SupplierOfferings.Select(o => new CatalogSupplierOfferingDto
if (dto.Wall.HasValue) rt.Wall = dto.Wall.Value; {
break; SupplierName = suppliers.FirstOrDefault(sup => sup.Id == o.SupplierId)?.Name ?? "Unknown",
case FlatBarDimensions fb: PartNumber = o.PartNumber,
if (dto.Width.HasValue) fb.Width = dto.Width.Value; SupplierDescription = o.SupplierDescription,
if (dto.Thickness.HasValue) fb.Thickness = dto.Thickness.Value; Price = o.Price,
break; Notes = o.Notes
case SquareBarDimensions sb: }).ToList()
if (dto.Size.HasValue) sb.Size = dto.Size.Value; }).ToList();
break;
case SquareTubeDimensions st:
if (dto.Size.HasValue) st.Size = dto.Size.Value;
if (dto.Wall.HasValue) st.Wall = dto.Wall.Value;
break;
case RectangularTubeDimensions rect:
if (dto.Width.HasValue) rect.Width = dto.Width.Value;
if (dto.Height.HasValue) rect.Height = dto.Height.Value;
if (dto.Wall.HasValue) rect.Wall = dto.Wall.Value;
break;
case AngleDimensions a:
if (dto.Leg1.HasValue) a.Leg1 = dto.Leg1.Value;
if (dto.Leg2.HasValue) a.Leg2 = dto.Leg2.Value;
if (dto.Thickness.HasValue) a.Thickness = dto.Thickness.Value;
break;
case ChannelDimensions c:
if (dto.Height.HasValue) c.Height = dto.Height.Value;
if (dto.Flange.HasValue) c.Flange = dto.Flange.Value;
if (dto.Web.HasValue) c.Web = dto.Web.Value;
break;
case IBeamDimensions ib:
if (dto.Height.HasValue) ib.Height = dto.Height.Value;
if (dto.WeightPerFoot.HasValue) ib.WeightPerFoot = dto.WeightPerFoot.Value;
break;
case PipeDimensions p:
if (dto.NominalSize.HasValue) p.NominalSize = dto.NominalSize.Value;
if (dto.Wall.HasValue) p.Wall = dto.Wall.Value;
if (dto.Schedule != null) p.Schedule = dto.Schedule;
break;
}
} }
} }

5
scripts/AlroCatalog/alro-scrape-progress.json
View File

@@ -4,6 +4,11 @@
"Bars", "Bars",
"A-36", "A-36",
"ROUND" "ROUND"
],
[
"Bars",
"A-36",
"FLAT"
] ]
], ],
"items": [ "items": [

132
scripts/AlroCatalog/scrape_alro.py
View File

@@ -5,10 +5,10 @@ Scrapes myalro.com's SmartGrid for Carbon Steel materials and outputs
a catalog JSON matching the O'Neal catalog format. a catalog JSON matching the O'Neal catalog format.
Usage: Usage:
python scrape_alro.py # Scrape filtered grades (edit GRADE_FILTER below) python scrape_alro.py # Scrape filtered grades (resumes from saved progress)
python scrape_alro.py --all-grades # Scrape ALL grades (slow) python scrape_alro.py --all-grades # Scrape ALL grades (slow)
python scrape_alro.py --discover # Scrape first item only, dump HTML/screenshots python scrape_alro.py --discover # Scrape first item only, dump HTML/screenshots
python scrape_alro.py --resume # Resume from saved progress python scrape_alro.py --fresh # Start fresh, ignoring saved progress
""" """
import asyncio import asyncio
@@ -342,8 +342,14 @@ async def get_select_options(page: Page, sel_id: str):
async def scrape_dims_panel(page: Page, grade: str, shape_alro: str, async def scrape_dims_panel(page: Page, grade: str, shape_alro: str,
shape_mapped: str, *, save_discovery: bool = False): shape_mapped: str, *, save_discovery: bool = False,
"""Main Level 3 extraction. Returns list of raw item dicts.""" on_item=None, scraped_dim_a: set[str] | None = None):
"""Main Level 3 extraction. Returns list of raw item dicts.
If on_item callback is provided, it is called with each item dict
as soon as it is discovered (for incremental saving).
If scraped_dim_a is provided, DimA values in that set are skipped (resume).
"""
items: list[dict] = [] items: list[dict] = []
if save_discovery: if save_discovery:
@@ -368,9 +374,18 @@ async def scrape_dims_panel(page: Page, grade: str, shape_alro: str,
log.warning(f" Could not dump dims panel: {e}") log.warning(f" Could not dump dims panel: {e}")
return [] return []
log.info(f" DimA: {len(dim_a_opts)} sizes") already_done = scraped_dim_a or set()
remaining = [(v, t) for v, t in dim_a_opts if v not in already_done]
if already_done:
log.info(f" DimA: {len(dim_a_opts)} sizes ({len(dim_a_opts) - len(remaining)} already scraped, {len(remaining)} remaining)")
else:
log.info(f" DimA: {len(dim_a_opts)} sizes")
for a_val, a_text in dim_a_opts: # All DimA values already scraped — combo is complete
if not remaining:
return []
for a_val, a_text in remaining:
# Select DimA → triggers postback → DimB/Length populate # Select DimA → triggers postback → DimB/Length populate
await page.select_option(f"#{ID['dim_a']}", a_val) await page.select_option(f"#{ID['dim_a']}", a_val)
await asyncio.sleep(DELAY) await asyncio.sleep(DELAY)
@@ -394,29 +409,38 @@ async def scrape_dims_panel(page: Page, grade: str, shape_alro: str,
lengths = await get_select_options(page, ID["dim_length"]) lengths = await get_select_options(page, ID["dim_length"])
for l_val, l_text in lengths: for l_val, l_text in lengths:
items.append(_make_item( item = _make_item(
grade, shape_mapped, grade, shape_mapped,
a_val, a_text, b_val, b_text, c_val, c_text, a_val, a_text, b_val, b_text, c_val, c_text,
l_text, l_text,
)) )
items.append(item)
if on_item:
on_item(item)
else: else:
# No DimC — read lengths # No DimC — read lengths
lengths = await get_select_options(page, ID["dim_length"]) lengths = await get_select_options(page, ID["dim_length"])
for l_val, l_text in lengths: for l_val, l_text in lengths:
items.append(_make_item( item = _make_item(
grade, shape_mapped, grade, shape_mapped,
a_val, a_text, b_val, b_text, None, None, a_val, a_text, b_val, b_text, None, None,
l_text, l_text,
)) )
items.append(item)
if on_item:
on_item(item)
else: else:
# No DimB — just DimA + Length # No DimB — just DimA + Length
lengths = await get_select_options(page, ID["dim_length"]) lengths = await get_select_options(page, ID["dim_length"])
for l_val, l_text in lengths: for l_val, l_text in lengths:
items.append(_make_item( item = _make_item(
grade, shape_mapped, grade, shape_mapped,
a_val, a_text, None, None, None, None, a_val, a_text, None, None, None, None,
l_text, l_text,
)) )
items.append(item)
if on_item:
on_item(item)
return items return items
@@ -467,7 +491,7 @@ def build_size_and_dims(shape: str, item: dict):
return f'{a_txt}"', {"width": round(a, 4), "thickness": 0} return f'{a_txt}"', {"width": round(a, 4), "thickness": 0}
if shape == "SquareBar" and a is not None: if shape == "SquareBar" and a is not None:
return f'{a_txt}"', {"size": round(a, 4)} return f'{a_txt}"', {"sideLength": round(a, 4)}
if shape == "Angle": if shape == "Angle":
if a is not None and b is not None: if a is not None and b is not None:
@@ -495,9 +519,9 @@ def build_size_and_dims(shape: str, item: dict):
if shape == "SquareTube": if shape == "SquareTube":
if a is not None and b is not None: if a is not None and b is not None:
return (f'{a_txt}" x {b_txt}" wall', return (f'{a_txt}" x {b_txt}" wall',
{"size": round(a, 4), "wall": round(b, 4)}) {"sideLength": round(a, 4), "wall": round(b, 4)})
if a is not None: if a is not None:
return f'{a_txt}"', {"size": round(a, 4), "wall": 0} return f'{a_txt}"', {"sideLength": round(a, 4), "wall": 0}
if shape == "RectangularTube": if shape == "RectangularTube":
if a is not None and b is not None and c is not None: if a is not None and b is not None and c is not None:
@@ -524,6 +548,20 @@ def build_size_and_dims(shape: str, item: dict):
return a_txt or "", {} return a_txt or "", {}
SHAPE_GROUP_KEY = {
"Angle": "angles",
"Channel": "channels",
"FlatBar": "flatBars",
"IBeam": "iBeams",
"Pipe": "pipes",
"RectangularTube": "rectangularTubes",
"RoundBar": "roundBars",
"RoundTube": "roundTubes",
"SquareBar": "squareBars",
"SquareTube": "squareTubes",
}
def build_catalog(scraped: list[dict]) -> dict: def build_catalog(scraped: list[dict]) -> dict:
"""Assemble the final catalog JSON from scraped item dicts.""" """Assemble the final catalog JSON from scraped item dicts."""
materials: dict[tuple, dict] = {} materials: dict[tuple, dict] = {}
@@ -538,14 +576,14 @@ def build_catalog(scraped: list[dict]) -> dict:
key = (shape, grade, size_str) key = (shape, grade, size_str)
if key not in materials: if key not in materials:
materials[key] = { mat = {
"shape": shape,
"type": "Steel", "type": "Steel",
"grade": grade, "grade": grade,
"size": size_str, "size": size_str,
"dimensions": dims,
"stockItems": [], "stockItems": [],
} }
mat.update(dims)
materials[key] = mat
length = item.get("length_inches") length = item.get("length_inches")
if length and length > 0: if length and length > 0:
@@ -561,7 +599,12 @@ def build_catalog(scraped: list[dict]) -> dict:
}], }],
}) })
sorted_mats = sorted(materials.values(), key=lambda m: (m["shape"], m["grade"], m["size"])) # Group by shape key
grouped: dict[str, list] = {v: [] for v in SHAPE_GROUP_KEY.values()}
for (shape, _, _), mat in sorted(materials.items(), key=lambda kv: (kv[0][0], kv[0][1], kv[0][2])):
group_key = SHAPE_GROUP_KEY.get(shape)
if group_key:
grouped[group_key].append(mat)
return { return {
"exportedAt": datetime.now(timezone.utc).isoformat(), "exportedAt": datetime.now(timezone.utc).isoformat(),
@@ -572,7 +615,7 @@ def build_catalog(scraped: list[dict]) -> dict:
{"name": "Cold Cut Saw", "kerfInches": 0.0625, "isDefault": False}, {"name": "Cold Cut Saw", "kerfInches": 0.0625, "isDefault": False},
{"name": "Hacksaw", "kerfInches": 0.0625, "isDefault": False}, {"name": "Hacksaw", "kerfInches": 0.0625, "isDefault": False},
], ],
"materials": sorted_mats, "materials": grouped,
} }
@@ -596,20 +639,29 @@ def save_progress(progress: dict):
async def main(): async def main():
discover = "--discover" in sys.argv discover = "--discover" in sys.argv
resume = "--resume" in sys.argv fresh = "--fresh" in sys.argv
all_grades = "--all-grades" in sys.argv all_grades = "--all-grades" in sys.argv
progress = load_progress() if resume else {"completed": [], "items": []} progress = {"completed": [], "items": []} if fresh else load_progress()
all_items: list[dict] = progress.get("items", []) all_items: list[dict] = progress.get("items", [])
done_keys: set[tuple] = {tuple(k) for k in progress.get("completed", [])} done_keys: set[tuple] = {tuple(k) for k in progress.get("completed", [])}
# Build index of saved DimA values per (grade, shape) for partial resume
saved_dim_a: dict[tuple[str, str], set[str]] = {}
if all_items and not fresh:
for item in all_items:
key = (item.get("grade", ""), item.get("shape", ""))
saved_dim_a.setdefault(key, set()).add(item.get("dim_a_val", ""))
log.info("Alro Steel SmartGrid Scraper") log.info("Alro Steel SmartGrid Scraper")
if all_grades: if all_grades:
log.info(" Mode: ALL grades") log.info(" Mode: ALL grades")
else: else:
log.info(f" Filtering to {len(GRADE_FILTER)} grades: {', '.join(sorted(GRADE_FILTER))}") log.info(f" Filtering to {len(GRADE_FILTER)} grades: {', '.join(sorted(GRADE_FILTER))}")
if resume: if fresh:
log.info(f" Resuming: {len(done_keys)} combos done, {len(all_items)} items") log.info(" Fresh start — ignoring saved progress")
elif done_keys:
log.info(f" Resuming: {len(done_keys)} combos done, {len(all_items)} items saved")
if discover: if discover:
log.info(" Discovery mode — will scrape first item then stop") log.info(" Discovery mode — will scrape first item then stop")
@@ -677,19 +729,30 @@ async def main():
await asyncio.sleep(DELAY) await asyncio.sleep(DELAY)
combo_count = 0
def on_item_discovered(item):
nonlocal total_scraped, combo_count
all_items.append(item)
total_scraped += 1
combo_count += 1
progress["items"] = all_items
save_progress(progress)
# Pass already-scraped DimA values so partial combos resume correctly
already = saved_dim_a.get((grade_name, shape_mapped), set())
items = await scrape_dims_panel( items = await scrape_dims_panel(
page, grade_name, shape_name, shape_mapped, page, grade_name, shape_name, shape_mapped,
save_discovery=first_item or discover, save_discovery=first_item or discover,
on_item=on_item_discovered,
scraped_dim_a=already,
) )
first_item = False first_item = False
all_items.extend(items) log.info(f" -> {combo_count} items (total {total_scraped})")
total_scraped += len(items)
log.info(f" -> {len(items)} items (total {total_scraped})")
done_keys.add(combo_key) done_keys.add(combo_key)
progress["completed"] = [list(k) for k in done_keys] progress["completed"] = [list(k) for k in done_keys]
progress["items"] = all_items
save_progress(progress) save_progress(progress)
await click_back(page) await click_back(page)
@@ -714,14 +777,13 @@ async def main():
OUTPUT_PATH.write_text(json.dumps(catalog, indent=2, ensure_ascii=False), encoding="utf-8") OUTPUT_PATH.write_text(json.dumps(catalog, indent=2, ensure_ascii=False), encoding="utf-8")
log.info(f"Written: {OUTPUT_PATH}") log.info(f"Written: {OUTPUT_PATH}")
log.info(f"Materials: {len(catalog['materials'])}") total_mats = sum(len(v) for v in catalog["materials"].values())
total_stock = sum(len(m["stockItems"]) for m in catalog["materials"]) total_stock = sum(len(m["stockItems"]) for v in catalog["materials"].values() for m in v)
log.info(f"Materials: {total_mats}")
log.info(f"Stock items: {total_stock}") log.info(f"Stock items: {total_stock}")
by_shape: dict[str, int] = {} for shape_key, mats in sorted(catalog["materials"].items()):
for m in catalog["materials"]: if mats:
by_shape[m["shape"]] = by_shape.get(m["shape"], 0) + 1 log.info(f" {shape_key}: {len(mats)}")
for s, n in sorted(by_shape.items()):
log.info(f" {s}: {n}")
if __name__ == "__main__": if __name__ == "__main__":