diff --git a/.dockerignore b/.dockerignore index 15adda87..4df18af8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,14 @@ !OpenNest.Data/** !OpenNest.Core/ !OpenNest.Core/** +!OpenNest.Server.Worker/ +!OpenNest.Server.Worker/** +!OpenNest.Api/ +!OpenNest.Api/** +!OpenNest.Engine/ +!OpenNest.Engine/** +!OpenNest.IO/ +!OpenNest.IO/** **/bin/** **/obj/** **/data/** diff --git a/.gitattributes b/.gitattributes index 412eeda7..d3a0b90c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -20,3 +20,6 @@ *.PDF diff=astextplain *.rtf diff=astextplain *.RTF diff=astextplain + +# Container scripts run in Linux images. +*.sh text eol=lf diff --git a/.github/workflows/server-image.yml b/.github/workflows/server-image.yml index fcef121c..2f3a57d9 100644 --- a/.github/workflows/server-image.yml +++ b/.github/workflows/server-image.yml @@ -6,6 +6,10 @@ on: - 'OpenNest.Server/**' - 'OpenNest.Data/**' - 'OpenNest.Core/**' + - 'OpenNest.Server.Worker/**' + - 'OpenNest.Api/**' + - 'OpenNest.Engine/**' + - 'OpenNest.IO/**' - 'OpenNest.Server.Tests/**' - 'scripts/Server.Tests/**' - 'scripts/Test-ServerContainer.sh' @@ -21,6 +25,10 @@ on: - 'OpenNest.Server/**' - 'OpenNest.Data/**' - 'OpenNest.Core/**' + - 'OpenNest.Server.Worker/**' + - 'OpenNest.Api/**' + - 'OpenNest.Engine/**' + - 'OpenNest.IO/**' - 'OpenNest.Server.Tests/**' - 'scripts/Server.Tests/**' - 'scripts/Test-ServerContainer.sh' diff --git a/AGENTS.md b/AGENTS.md index 4a21b918..a0b30dc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,7 @@ Releases: follow [the release procedure](docs/releasing.md) and `scripts/Publish - `OpenNest.IO`: ACadSharp and SVG import/export and ZIP-based `.nest` persistence; depends on Core only (an IO.Tests guard enforces it). Best fits persist only when callers pass `BestFitCacheStore.Shared` as `NestReader`/`NestWriter.BestFitStore`. All DXF-to-Drawing conversion goes through `CadImporter`: `Import` + `BuildDrawing` for editable/reporting flows, `ImportDrawing` for headless callers. Preserve source offsets, entity IDs, suppressed entities and bends. Bend repair is opt-in, requires explicit source units and may not alter cut geometry or unrelated marks. - `OpenNest`: WinForms UI (`Forms/`, `Controls/PlateView`, `Actions/`). `OpenNest.Data` holds cross-platform persistence; new-nest defaults live in `%APPDATA%\OpenNest\defaults.json` (in Database mode, the server's shared copy once it holds one). WinForms `Properties.Settings` persist to `%APPDATA%\OpenNest\user-settings.json` through `AppDataSettingsProvider`, never inside the Velopack install folder; `OpenNest.WinForms.Tests` redirects them to a temp file, so a test run never writes the user's settings. Posts live in `Posts/OpenNest.Posts./` and deploy to the desktop output's `Posts/` directory. Desktop nest edits go through `BeginChange`/`RecordChange`; see [undo and redo](docs/undo-redo.md). - `OpenNest.Desktop` (net8.0): toolkit-neutral desktop logic shared by the WinForms app and `OpenNest.Avalonia`, the cross-platform desktop in progress. It must not reference WinForms/WindowsDesktop, `System.Drawing.Common` or Avalonia (`OpenNest.Desktop.Tests` guards this, including transitive packages); put toolkit-specific rendering, dialogs and settings behind adapters in the front ends. +- `OpenNest.Server`: nest storage HTTP service, kept off Engine and IO. `OpenNest.Server.Worker`: the server nesting worker, shipped in the same image. - `OpenNest.Console`, `OpenNest.Mcp`, `OpenNest.Api`: front ends; `OpenNest.Benchmark`: whole-job engine comparisons; `OpenNest.Gpu`: GPU evaluators; `OpenNest.Training`: ML data collection and the Python sketch-search prototype (`sketch-search/README.md`). Benchmark timing comparisons require `--parallel 1`; validate layouts and fulfillment, not just elapsed time. ## Geometry and ownership safeguards @@ -77,6 +78,7 @@ Read the relevant contract before changing its behavior: - [Pre-post verification](docs/post-verification.md) - [Machine cutting defaults](docs/machine-cutting-defaults.md): the nest's machine supplies leads, kerf and cut-off settings for plates without their own parameters - [Shared settings](docs/shared-settings.md): in Database mode machines, post settings, clamps and nest defaults come from the server once it holds them; offline uses the last copy read, read-only +- [Server nesting](docs/server-nesting.md): Database-mode Auto Nest queued to the server's worker container (`OPENNEST_ROLE=worker`), one job at a time with no time limit; the desktop replays the recorded engine answers to validate and bind locally. The HTTP server stays off Engine/IO - [Cut-time estimates](docs/cut-time-estimates.md): simulator/Calculate Cut Time acceleration planner and placeholder machine limits - [Cincinnati CL](docs/cincinnati-post-output.md) and [CI Fiber](docs/cincinnati-ci-fiber-post-output.md) - [Fill verification](docs/performance/fill-verification.md): opt-in benchmarks, frozen oracles, Debug-only counters and predictor initialization. Zero Release counters do not prove work removal. diff --git a/OpenNest.Api/NestJobPackage.cs b/OpenNest.Api/NestJobPackage.cs new file mode 100644 index 00000000..0b517ab3 --- /dev/null +++ b/OpenNest.Api/NestJobPackage.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using OpenNest.Engine; +using OpenNest.Engine.Jobs; +using OpenNest.Engine.Jobs.Remote; +using OpenNest.IO; + +namespace OpenNest.Api; + +/// +/// A whole-job nest request as one upload for the nest server: a ZIP holding +/// job.json () and drawings.nest, a nest archive of +/// exactly the items' drawings in the order the document names them. Drawings travel as the +/// nest file format stores them, so the server nests what the client has, edits included. +/// +public static class NestJobPackage +{ + public const string JobEntry = "job.json"; + public const string DrawingsEntry = "drawings.nest"; + + public static byte[] Write(NestPipelineRequest request, Units units = Units.Inches) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(request.Items); + var drawings = new List(); + foreach (var item in request.Items) + { + if (item?.Drawing != null && !drawings.Any(d => ReferenceEquals(d, item.Drawing))) + drawings.Add(item.Drawing); + } + var document = NestJobDocument.From(request, drawings); + + // A throwaway nest holds the drawings only; the caller's nest is never written. + var nest = new Nest { Units = units }; + foreach (var drawing in drawings) + nest.Drawings.Add(drawing); + + using var buffer = new MemoryStream(); + using (var zip = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + using (var writer = new StreamWriter(zip.CreateEntry(JobEntry).Open(), new UTF8Encoding(false))) + writer.Write(document.ToJson()); + using var archive = zip.CreateEntry(DrawingsEntry, CompressionLevel.NoCompression).Open(); + new NestWriter(nest).Write(archive); + } + return buffer.ToArray(); + } + + /// Reads a package back into a request; throws when it is not one. + public static NestPipelineRequest Read(byte[] package) + { + ArgumentNullException.ThrowIfNull(package); + try + { + using var zip = new ZipArchive(new MemoryStream(package, writable: false), ZipArchiveMode.Read); + var jobEntry = zip.GetEntry(JobEntry) ?? throw new FormatException($"The nest job has no {JobEntry}."); + var drawingsEntry = zip.GetEntry(DrawingsEntry) ?? throw new FormatException($"The nest job has no {DrawingsEntry}."); + + string json; + using (var reader = new StreamReader(jobEntry.Open(), Encoding.UTF8)) + json = reader.ReadToEnd(); + var document = NestJobDocument.FromJson(json); + + // NestReader needs a seekable stream. + using var archive = new MemoryStream(); + using (var entry = drawingsEntry.Open()) + entry.CopyTo(archive); + archive.Position = 0; + var nest = new NestReader(archive).Read(); + return document.ToRequest(nest.Drawings.ToList()); + } + catch (InvalidDataException ex) + { + throw new FormatException("The nest job is not a readable ZIP package.", ex); + } + } +} diff --git a/OpenNest.Data/NestJobRecord.cs b/OpenNest.Data/NestJobRecord.cs new file mode 100644 index 00000000..a0b75b7c --- /dev/null +++ b/OpenNest.Data/NestJobRecord.cs @@ -0,0 +1,97 @@ +namespace OpenNest.Data; + +/// Where a server nest job is. Succeeded, Failed and Cancelled are final. +public enum NestJobState +{ + Queued, + Running, + Succeeded, + Failed, + Cancelled, +} + +/// A whole-job nest the server's worker solves; see docs/server-nesting.md. +public sealed class NestJobRecord +{ + public Guid Id { get; set; } + + /// What the job is for, for people: usually the nest's name. + public string Title { get; set; } = ""; + + public string EngineName { get; set; } = ""; + + public string SubmittedBy { get; set; } = ""; + + public NestJobState State { get; set; } + + /// Someone asked to stop the job while it ran; the worker stops at its next heartbeat. + public bool CancelRequested { get; set; } + + public DateTime SubmittedAt { get; set; } + + public DateTime? StartedAt { get; set; } + + public DateTime? FinishedAt { get; set; } + + /// How many times a worker has taken the job; a worker that stops responding gives it back. + public int Attempts { get; set; } + + /// Why the job failed; empty otherwise. + public string Error { get; set; } = ""; + + /// Queued jobs ahead of this one (zero for a job that is not queued). + public int QueuePosition { get; set; } + + public long PackageSize { get; set; } + + public bool IsFinished => State is NestJobState.Succeeded or NestJobState.Failed or NestJobState.Cancelled; +} + +/// The description sent with a job's package. +public sealed class NestJobSubmission +{ + public string Title { get; set; } = ""; + + public string EngineName { get; set; } = ""; + + public string SubmittedBy { get; set; } = ""; +} + +/// A worker's message about the job it holds. +public sealed class NestJobWorkerMessage +{ + public string WorkerId { get; set; } = ""; + + /// Why the job failed, for fail; ignored otherwise. + public string Error { get; set; } = ""; +} + +/// The server's answer to a worker's heartbeat. +public sealed class NestJobHeartbeat +{ + public bool CancelRequested { get; set; } +} + +/// Limits and timings both sides of server nesting share. +public static class NestJobRules +{ + /// Largest job package (drawings and job) the server accepts. + public const long MaxPackageBytes = 256L * 1024 * 1024; + + /// Largest result the server accepts from a worker. + public const long MaxResultBytes = 64L * 1024 * 1024; + + public const int MaxTextLength = 500; + + /// How long a running job stays with its worker without a heartbeat. + public static readonly TimeSpan Lease = TimeSpan.FromMinutes(2); + + /// How often a worker reports; well inside . + public static readonly TimeSpan HeartbeatInterval = TimeSpan.FromSeconds(20); + + /// Workers that stop responding this many times fail the job instead of retrying it. + public const int MaxAttempts = 3; + + /// Finished jobs, their packages and results are removed after this long. + public static readonly TimeSpan Retention = TimeSpan.FromDays(7); +} diff --git a/OpenNest.Data/RemoteNestJobs.cs b/OpenNest.Data/RemoteNestJobs.cs new file mode 100644 index 00000000..0f162b1f --- /dev/null +++ b/OpenNest.Data/RemoteNestJobs.cs @@ -0,0 +1,155 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; + +namespace OpenNest.Data; + +/// +/// Server nesting over /api/nest-jobs: desktops submit, poll, read results and cancel; +/// the worker claims, heartbeats and reports. Packages and results are opaque bytes here +/// (OpenNest.Api.NestJobPackage, NestSolveTranscript). A server without nest jobs +/// raises ; other failures surface as . +/// +public sealed class RemoteNestJobs : NestServerClient +{ + private const string Feature = "server nesting"; + + public RemoteNestJobs(string baseUrl) + : base(new HttpClient(), baseUrl, ownsClient: true) + { + } + + /// For tests and hosts that pool HttpClient instances. + public RemoteNestJobs(HttpClient httpClient, string baseUrl) + : base(httpClient, baseUrl, ownsClient: false) + { + } + + public async Task SubmitAsync(NestJobSubmission submission, byte[] package, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(submission); + ArgumentNullException.ThrowIfNull(package); + if (package.Length == 0 || package.Length > NestJobRules.MaxPackageBytes) + throw new ArgumentException("The nest job package is empty or too large.", nameof(package)); + + using var content = new MultipartFormDataContent(); + content.Add(new StringContent(JsonSerializer.Serialize(submission, NestServerHttp.JsonOptions), Encoding.UTF8, "application/json"), "metadata"); + var file = new ByteArrayContent(package); + file.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); + content.Add(file, "file", "job.zip"); + using var response = await _httpClient.PostAsync(Url("api/nest-jobs"), content, cancellationToken); + if (response.StatusCode == HttpStatusCode.NotFound) + throw new ServerTooOldException(Feature); + await NestServerHttp.EnsureSuccessAsync(response, "submit the nest", cancellationToken); + return await ReadRecordAsync(response, cancellationToken); + } + + /// The job, or null when the server does not hold it (removed after ). + public async Task GetAsync(Guid id, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.GetAsync(Url($"api/nest-jobs/{id}"), cancellationToken); + if (response.StatusCode == HttpStatusCode.NotFound) + return null; + await NestServerHttp.EnsureSuccessAsync(response, "read the nest job", cancellationToken); + var record = await ReadRecordAsync(response, cancellationToken); + return record.Id == id ? record : throw new IOException("Nest server returned a different nest job."); + } + + /// A succeeded job's result (a NestSolveTranscript as JSON). + public async Task GetResultAsync(Guid id, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.GetAsync(Url($"api/nest-jobs/{id}/result"), cancellationToken); + await NestServerHttp.EnsureSuccessAsync(response, "read the nest result", cancellationToken); + return await response.Content.ReadAsStringAsync(cancellationToken); + } + + /// Stops the job: a queued job is cancelled at once, a running one at its worker's next heartbeat. + public async Task CancelAsync(Guid id, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.PostAsync(Url($"api/nest-jobs/{id}/cancel"), null, cancellationToken); + if (response.StatusCode == HttpStatusCode.NotFound) + return null; + await NestServerHttp.EnsureSuccessAsync(response, "stop the nest job", cancellationToken); + return await ReadRecordAsync(response, cancellationToken); + } + + /// Takes the oldest queued job for , or null when there is + /// none or another job is still running. + public async Task ClaimAsync(string workerId, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.PostAsJsonAsync(Url("api/nest-jobs/claim"), + new NestJobWorkerMessage { WorkerId = workerId }, NestServerHttp.JsonOptions, cancellationToken); + if (response.StatusCode == HttpStatusCode.NoContent) + return null; + if (response.StatusCode == HttpStatusCode.NotFound) + throw new ServerTooOldException(Feature); + await NestServerHttp.EnsureSuccessAsync(response, "claim a nest job", cancellationToken); + return await ReadRecordAsync(response, cancellationToken); + } + + public async Task GetPackageAsync(Guid id, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.GetAsync(Url($"api/nest-jobs/{id}/package"), cancellationToken); + await NestServerHttp.EnsureSuccessAsync(response, "read the nest job package", cancellationToken); + return await response.Content.ReadAsByteArrayAsync(cancellationToken); + } + + /// Keeps the job's lease. Null when the worker no longer holds the job. + public async Task HeartbeatAsync(Guid id, string workerId, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.PostAsJsonAsync(Url($"api/nest-jobs/{id}/heartbeat"), + new NestJobWorkerMessage { WorkerId = workerId }, NestServerHttp.JsonOptions, cancellationToken); + if (response.StatusCode is HttpStatusCode.Conflict or HttpStatusCode.NotFound) + return null; + await NestServerHttp.EnsureSuccessAsync(response, "report on the nest job", cancellationToken); + return await response.Content.ReadFromJsonAsync(NestServerHttp.JsonOptions, cancellationToken) + ?? throw new IOException("Nest server returned an empty heartbeat."); + } + + /// Stores the result. False when the worker no longer holds the job. + public async Task CompleteAsync(Guid id, string workerId, string result, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(result); + using var content = new StringContent(result, Encoding.UTF8, "application/json"); + using var response = await _httpClient.PutAsync( + Url($"api/nest-jobs/{id}/result?workerId={Uri.EscapeDataString(workerId)}"), content, cancellationToken); + return await Held(response, "store the nest result", cancellationToken); + } + + public async Task FailAsync(Guid id, string workerId, string error, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.PostAsJsonAsync(Url($"api/nest-jobs/{id}/fail"), + new NestJobWorkerMessage { WorkerId = workerId, Error = error }, NestServerHttp.JsonOptions, cancellationToken); + return await Held(response, "report the failed nest job", cancellationToken); + } + + /// The worker stopped the job as asked. + public async Task ConfirmCancelledAsync(Guid id, string workerId, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.PostAsJsonAsync(Url($"api/nest-jobs/{id}/cancelled"), + new NestJobWorkerMessage { WorkerId = workerId }, NestServerHttp.JsonOptions, cancellationToken); + return await Held(response, "report the stopped nest job", cancellationToken); + } + + /// Gives the job back to the queue without spending an attempt (the worker is shutting down). + public async Task ReleaseAsync(Guid id, string workerId, CancellationToken cancellationToken = default) + { + using var response = await _httpClient.PostAsJsonAsync(Url($"api/nest-jobs/{id}/release"), + new NestJobWorkerMessage { WorkerId = workerId }, NestServerHttp.JsonOptions, cancellationToken); + return await Held(response, "give back the nest job", cancellationToken); + } + + private static async Task Held(HttpResponseMessage response, string action, CancellationToken cancellationToken) + { + if (response.StatusCode is HttpStatusCode.Conflict or HttpStatusCode.NotFound) + return false; + await NestServerHttp.EnsureSuccessAsync(response, action, cancellationToken); + return true; + } + + private static async Task ReadRecordAsync(HttpResponseMessage response, CancellationToken cancellationToken) => + await response.Content.ReadFromJsonAsync(NestServerHttp.JsonOptions, cancellationToken) + ?? throw new IOException("Nest server returned an empty nest job."); +} diff --git a/OpenNest.Engine.Tests/Jobs/RemoteNestReplayTests.cs b/OpenNest.Engine.Tests/Jobs/RemoteNestReplayTests.cs new file mode 100644 index 00000000..001e22ae --- /dev/null +++ b/OpenNest.Engine.Tests/Jobs/RemoteNestReplayTests.cs @@ -0,0 +1,172 @@ +using OpenNest.CNC; +using OpenNest.Engine.Jobs; +using OpenNest.Engine.Jobs.Remote; +using OpenNest.Geometry; + +namespace OpenNest.Engine.Tests.Jobs; + +/// +/// A nest solved elsewhere (the nest server's worker) comes back as the engine's recorded +/// answers and is replayed here: the proposal must be the local run's, validated and bound +/// to the caller's own drawings, and a replay that does not fit these drawings must refuse. +/// +public class RemoteNestReplayTests +{ + private static NestPipelineRequest Request(Drawing a, Drawing b, string engine = "Rectangles") + { + var template = new Plate(48, 96) { EdgeSpacing = new Spacing(0.25, 0.25, 0.25, 0.25), PartSpacing = 0.25 }; + template.SetClampRail(new ClampRail(1.5, 0.625, new[] { 6.0, 30.0 })); + return new NestPipelineRequest(engine, + [new NestItem { Drawing = a, Quantity = 12 }, new NestItem { Drawing = b, Quantity = 7, Priority = 1 }], + NestStockBuilder.FromTemplate(template, + [new PlateOption { Width = 48, Length = 96, Cost = 100 }, new PlateOption { Width = 60, Length = 120, Quantity = 2, Cost = 160 }]), + new NestJobOptions(maxPlates: 10, salvageRate: 0.5, minimumSalvageDimension: 6)); + } + + private static NestSolveTranscript Solve(NestPipelineRequest request) + { + var recorder = new RecordingNestingEngine(NestingEngineRegistry.Create(request.EngineName)); + NestPipeline.Run(recorder, request.EngineName, request); + // Over the wire and back, as the server sends it. + return NestSolveTranscript.FromJson(recorder.Transcript.ToJson()); + } + + [Theory] + [InlineData("Rectangles")] + [InlineData("Default")] + public void ReplayGivesTheLocalProposalBoundToTheCallersDrawings(string engine) + { + var a = new Drawing("a", TestDrawingFactory.Rectangle(10, 20)); + var b = new Drawing("b", TestDrawingFactory.LShape()); + var request = Request(a, b, engine); + var submission = new RemoteNestSubmission(request); + var transcript = Solve(request); + + var local = NestPipeline.Run(request); + var replayed = submission.Complete(transcript); + + Assert.True(replayed.IsValid, string.Join("; ", replayed.Violations)); + Assert.True(replayed.CanKeep); + Assert.Equal(local.Plates.Count, replayed.Plates.Count); + for (var i = 0; i < local.Plates.Count; i++) + { + Assert.Equal(local.Plates[i].Quantity, replayed.Plates[i].Quantity); + Assert.Equal(local.Plates[i].Stock.Id, replayed.Plates[i].Stock.Id); + Assert.Same(request.Stock.Single(s => s.Id == replayed.Plates[i].Stock.Id), replayed.Plates[i].Stock); + Assert.Equal( + local.Plates[i].Parts.Select(p => (p.BaseDrawing.Name, p.Location.X, p.Location.Y, p.Rotation)), + replayed.Plates[i].Parts.Select(p => (p.BaseDrawing.Name, p.Location.X, p.Location.Y, p.Rotation))); + Assert.All(replayed.Plates[i].Parts, p => Assert.True(ReferenceEquals(p.BaseDrawing, a) || ReferenceEquals(p.BaseDrawing, b))); + } + Assert.Equal(local.Unplaced.Count, replayed.Unplaced.Count); + } + + [Fact] + public void CutoutTrialAndOrdinarySolveReplayInOrder() + { + // A 20 x 20 frame with a 12 x 14 window, a strap that fits the window, and a square + // that fits nowhere on the one sheet: the bundled trial is incomplete, so the pipeline + // also runs the ordinary solve, and the replay must answer both in order. + var frame = TestDrawingFactory.Rectangle(20, 20); + frame.Codes.Add(new RapidMove(4, 3)); + frame.Codes.Add(new LinearMove(16, 3)); + frame.Codes.Add(new LinearMove(16, 17)); + frame.Codes.Add(new LinearMove(4, 17)); + frame.Codes.Add(new LinearMove(4, 3)); + var request = new NestPipelineRequest("Default", + [ + new NestItem { Drawing = new Drawing("frame", frame), Quantity = 1 }, + new NestItem { Drawing = new Drawing("strap", TestDrawingFactory.Rectangle(13, 1.25)), Quantity = 1 }, + new NestItem { Drawing = new Drawing("square", TestDrawingFactory.Rectangle(15, 15)), Quantity = 1 }, + ], + new[] { new NestPlateStock("sheet", new Size(20.5, 20.5), 1, 0.25) }); + var submission = new RemoteNestSubmission(request); + var transcript = Solve(request); + Assert.True(transcript.Solves.Count >= 2, $"Expected the trial and the ordinary solve, got {transcript.Solves.Count}."); + + var local = NestPipeline.Run(request); + var replayed = submission.Complete(transcript); + + Assert.Equal(local.IsValid, replayed.IsValid); + Assert.Equal(local.Plates.SelectMany(p => p.Parts).Select(p => (p.BaseDrawing.Name, p.Location.X, p.Location.Y)), + replayed.Plates.SelectMany(p => p.Parts).Select(p => (p.BaseDrawing.Name, p.Location.X, p.Location.Y))); + Assert.Equal(local.Unplaced.Select(u => u.ToString()), replayed.Unplaced.Select(u => u.ToString())); + } + + [Fact] + public void ADrawingEditedAfterSendingRefusesTheResult() + { + var a = new Drawing("a", TestDrawingFactory.Rectangle(10, 20)); + var b = new Drawing("b", TestDrawingFactory.LShape()); + var request = Request(a, b); + var submission = new RemoteNestSubmission(request); + var transcript = Solve(request); + + a.Program = TestDrawingFactory.Rectangle(11, 20); + + var refused = Assert.Throws(() => submission.Complete(transcript)); + Assert.Contains("'a'", refused.Message); + } + + [Fact] + public void AnswersForADifferentJobDoNotReplay() + { + var a = new Drawing("a", TestDrawingFactory.Rectangle(10, 20)); + var b = new Drawing("b", TestDrawingFactory.LShape()); + var transcript = Solve(Request(a, b)); + var other = Request(a, b); + other.Items[0].Quantity = 13; + + Assert.Throws(() => new RemoteNestSubmission(other).Complete(transcript)); + } + + [Fact] + public void StockSurvivesTheWireBitForBit() + { + var outline = new SheetOutline( + new[] { new Vector(0, 0), new Vector(96, 0), new Vector(96, 48), new Vector(0, 48) }, + new[] { new[] { new Vector(10, 10), new Vector(12, 10), new Vector(12, 12.125), new Vector(10, 12.125) } }); + var template = new Plate(48, 96) { Quadrant = 3, EdgeSpacing = new Spacing(0.1, 0.2, 0.3, 0.4), PartSpacing = 0.1 + 0.2 }; + template.SetClampRail(new ClampRail(1.5, 0.625, new[] { new RailClamp(ClampEdge.OriginSide, 6), new RailClamp(ClampEdge.FarSide, 40.1) }, 7, 0.3)); + var stock = NestStockBuilder.FromTemplate(template, [new PlateOption { Width = 48, Length = 96, Cost = 1.1, Quantity = 3, Id = "ml-7" }]) + .Append(new NestPlateStock("odd", new Size(48, 96), null, 0.25, new Spacing(0, 0, 0, 0), 1, null, null, outline)) + .ToList(); + + foreach (var sheet in stock) + { + var json = System.Text.Json.JsonSerializer.Serialize(NestStockJson.From(sheet)); + var back = System.Text.Json.JsonSerializer.Deserialize(json)!.ToStock(); + Assert.True(RepeatedPlateGrouping.SameStock(sheet, back)); + Assert.Equal(sheet.ClampRail, back.ClampRail); + Assert.Equal(sheet.SheetClampZones.Select(z => (z.Id, z.Bounds.X, z.Bounds.Y, z.Bounds.Length, z.Bounds.Width)), + back.SheetClampZones.Select(z => (z.Id, z.Bounds.X, z.Bounds.Y, z.Bounds.Length, z.Bounds.Width))); + } + } + + [Fact] + public void OnlyShippedEnginesAreBuiltIn() + { + Assert.True(NestingEngineRegistry.IsBuiltIn("Default")); + Assert.True(NestingEngineRegistry.IsBuiltIn("rectangles")); + Assert.True(NestingEngineRegistry.IsBuiltIn("Opus55NestingEngine")); + Assert.False(NestingEngineRegistry.IsBuiltIn("SomePlugIn")); + Assert.False(NestingEngineRegistry.IsBuiltIn(null)); + } + + [Fact] + public void NonfinitePosesSurviveTheWire() + { + var transcript = new NestSolveTranscript(); + var stock = new NestPlateStock("s", new Size(10, 10)); + var job = new NestJob([], [stock]); + transcript.Add(job, new NestJobResult(NestJobStatus.Incomplete, NestJobStopReason.NoPlacementFound, + [new NestJobPlateResult(0, stock, [new NestJobPlacement("p", 0, double.NaN, double.PositiveInfinity, 0)])], [], [])); + + var back = NestSolveTranscript.FromJson(transcript.ToJson()).Solves.Single().Result.ToResult(job); + + var pose = back.Plates.Single().Placements.Single(); + Assert.True(double.IsNaN(pose.X)); + Assert.True(double.IsPositiveInfinity(pose.Y)); + Assert.Same(stock, back.Plates.Single().Stock); + } +} diff --git a/OpenNest.Engine/Jobs/NestPipelineDrawingFreshness.cs b/OpenNest.Engine/Jobs/NestPipelineDrawingFreshness.cs index 43fa931c..f6f8ee24 100644 --- a/OpenNest.Engine/Jobs/NestPipelineDrawingFreshness.cs +++ b/OpenNest.Engine/Jobs/NestPipelineDrawingFreshness.cs @@ -93,7 +93,7 @@ internal static class NestPipelineDrawingFreshness return changed; } - private static bool Same(PartGeometrySnapshot a, PartGeometrySnapshot b) => + internal static bool Same(PartGeometrySnapshot a, PartGeometrySnapshot b) => a.Mode == b.Mode && a.Motions.Count == b.Motions.Count && a.Motions.Zip(b.Motions).All(pair => pair.First.Type == pair.Second.Type diff --git a/OpenNest.Engine/Jobs/NestingEngineRegistry.cs b/OpenNest.Engine/Jobs/NestingEngineRegistry.cs index fb4cb5fa..29539994 100644 --- a/OpenNest.Engine/Jobs/NestingEngineRegistry.cs +++ b/OpenNest.Engine/Jobs/NestingEngineRegistry.cs @@ -24,6 +24,9 @@ public static class NestingEngineRegistry { private static readonly List engines = new(); + // Names registered by this assembly, before any plug-in could load. + private static readonly HashSet builtInNames; + /// /// Registry names used by earlier releases, mapped to the engine that replaced them, so saved /// selections and scripts keep working. Consulted only when no engine has the requested name. @@ -93,6 +96,8 @@ public static class NestingEngineRegistry "Repeats parts in rows and pairs, keeping the largest drop on the top", () => new FixedStrategyNestingEngine("Horizontal Remnant") ); + + builtInNames = new HashSet(engines.Select(e => e.Name), StringComparer.OrdinalIgnoreCase); } public static IReadOnlyList AvailableEngines => engines; @@ -116,6 +121,14 @@ public static class NestingEngineRegistry : null; } + /// + /// Whether (or the engine a renamed legacy name maps to) ships with + /// OpenNest rather than coming from a plug-in. Only built-in engines run on the nest server, + /// which loads no plug-ins. + /// + public static bool IsBuiltIn(string? name) => + ResolveName(name) is { } resolved && builtInNames.Contains(resolved); + /// /// Creates the engine registered under (case-insensitive, renamed legacy /// names accepted). The caller's explicit choice is the whole selection mechanism; unknown names throw. diff --git a/OpenNest.Engine/Jobs/Remote/NestJobDocument.cs b/OpenNest.Engine/Jobs/Remote/NestJobDocument.cs new file mode 100644 index 00000000..0407e7e7 --- /dev/null +++ b/OpenNest.Engine/Jobs/Remote/NestJobDocument.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace OpenNest.Engine.Jobs.Remote; + +/// +/// The parts of a that are not drawings: the engine, each +/// item's quantity, priority and rotation with the index of its drawing, the stock and the job +/// options. Drawings travel beside it in a .nest archive (OpenNest.Api.NestJobPackage). +/// A remote solve never runs with : it has no +/// time limit, so a slow server finds the same nest a fast one does, later. +/// +public sealed class NestJobDocument +{ + public const int CurrentVersion = 1; + + private static readonly JsonSerializerOptions Options = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) }, + }; + + public int Version { get; set; } = CurrentVersion; + public string EngineName { get; set; } + + /// Each drawing's name in list order, so a reader can tell the list arrived in that order. + public List DrawingNames { get; set; } = new(); + public List Items { get; set; } = new(); + public List Stock { get; set; } = new(); + public OptionsJson JobOptions { get; set; } + + /// + /// Describes , naming each item's drawing by its position in + /// , which must hold every item's drawing. + /// + public static NestJobDocument From(NestPipelineRequest request, IReadOnlyList drawings) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(drawings); + var document = new NestJobDocument + { + EngineName = request.EngineName, + DrawingNames = drawings.Select(d => d?.Name ?? "").ToList(), + Stock = request.Stock.Select(NestStockJson.From).ToList(), + JobOptions = request.Options is { } o ? new OptionsJson + { + PlacementStrategy = o.PlacementStrategy, + MaxPlates = o.MaxPlates, + SalvageRate = o.SalvageRate, + MinimumSalvageDimension = o.MinimumSalvageDimension, + OffcutCut = o.OffcutCut, + } : null, + }; + foreach (var item in request.Items) + { + var index = item?.Drawing == null ? -1 : IndexOf(drawings, item.Drawing); + if (item != null && index < 0) + throw new ArgumentException("Every item's drawing must be in the drawing list.", nameof(drawings)); + document.Items.Add(item == null ? null : new ItemJson + { + Drawing = index, + Quantity = item.Quantity, + Priority = item.Priority, + StepAngle = item.StepAngle, + RotationStart = item.RotationStart, + RotationEnd = item.RotationEnd, + }); + } + return document; + } + + /// The request this describes, with drawings taken from by position. + public NestPipelineRequest ToRequest(IReadOnlyList drawings) + { + ArgumentNullException.ThrowIfNull(drawings); + if (Version != CurrentVersion) + throw new FormatException($"Nest job version {Version} is not supported; this build reads {CurrentVersion}."); + if (string.IsNullOrWhiteSpace(EngineName)) + throw new FormatException("The nest job names no engine."); + if (DrawingNames == null || DrawingNames.Count != drawings.Count + || DrawingNames.Where((name, i) => !string.Equals(name, drawings[i]?.Name ?? "", StringComparison.Ordinal)).Any()) + throw new FormatException("The nest job's drawings do not match the drawings sent with it."); + var items = new List(Items?.Count ?? 0); + foreach (var item in Items ?? new()) + { + if (item == null) + { + items.Add(null); + continue; + } + if (item.Drawing < 0 || item.Drawing >= drawings.Count) + throw new FormatException($"A nest job item names drawing {item.Drawing}, which the job does not hold."); + items.Add(new NestItem + { + Drawing = drawings[item.Drawing], + Quantity = item.Quantity, + Priority = item.Priority, + StepAngle = item.StepAngle, + RotationStart = item.RotationStart, + RotationEnd = item.RotationEnd, + }); + } + var options = JobOptions is { } o + ? new NestJobOptions(o.PlacementStrategy ?? "Lattice", o.MaxPlates, o.SalvageRate, o.MinimumSalvageDimension, + searchUntilCancelled: false, o.OffcutCut) + : null; + return new NestPipelineRequest(EngineName, items, + (Stock ?? new()).Select(s => s?.ToStock() ?? throw new FormatException("A nest job stock entry is empty.")).ToList(), + options); + } + + public string ToJson() => JsonSerializer.Serialize(this, Options); + + public static NestJobDocument FromJson(string json) + { + try + { + return JsonSerializer.Deserialize(json, Options) + ?? throw new FormatException("The nest job is empty."); + } + catch (JsonException ex) { throw new FormatException("The nest job is not valid JSON.", ex); } + } + + private static int IndexOf(IReadOnlyList drawings, Drawing drawing) + { + for (var i = 0; i < drawings.Count; i++) + { + if (ReferenceEquals(drawings[i], drawing)) + return i; + } + return -1; + } + + public sealed class ItemJson + { + public int Drawing { get; set; } + public int Quantity { get; set; } + public int Priority { get; set; } + public double StepAngle { get; set; } + public double RotationStart { get; set; } + public double RotationEnd { get; set; } + } + + public sealed class OptionsJson + { + public string PlacementStrategy { get; set; } + public int? MaxPlates { get; set; } + public double SalvageRate { get; set; } + public double MinimumSalvageDimension { get; set; } + public CutOffAxis? OffcutCut { get; set; } + } +} diff --git a/OpenNest.Engine/Jobs/Remote/NestSolveTranscript.cs b/OpenNest.Engine/Jobs/Remote/NestSolveTranscript.cs new file mode 100644 index 00000000..58150e27 --- /dev/null +++ b/OpenNest.Engine/Jobs/Remote/NestSolveTranscript.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace OpenNest.Engine.Jobs.Remote; + +/// +/// Every engine solve one +/// made, in order: what it asked (part and stock ids with their quantities) and the raw result +/// the engine returned. A remote solve sends this back instead of bound plates, so the client +/// replays the pipeline against its own drawings and validates and binds the result itself +/// (). Poses are kept bit for bit, nonfinite ones included, +/// so the client's validation sees exactly what the engine returned. +/// +public sealed class NestSolveTranscript +{ + public const int CurrentVersion = 1; + + private static readonly JsonSerializerOptions Options = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) }, + }; + + public int Version { get; set; } = CurrentVersion; + public List Solves { get; set; } = new(); + + public string ToJson() => JsonSerializer.Serialize(this, Options); + + /// Reads a transcript; throws when it is not one this build understands. + public static NestSolveTranscript FromJson(string json) + { + NestSolveTranscript transcript; + try { transcript = JsonSerializer.Deserialize(json, Options); } + catch (JsonException ex) { throw new FormatException("The server's nest result is not valid JSON.", ex); } + if (transcript?.Solves == null || transcript.Solves.Any(s => s?.Result == null)) + throw new FormatException("The server's nest result is incomplete."); + if (transcript.Version != CurrentVersion) + throw new FormatException($"The server's nest result is version {transcript.Version}; this build reads {CurrentVersion}."); + return transcript; + } + + internal void Add(NestJob job, NestJobResult result) => + Solves.Add(new SolveJson { Request = Fingerprint(job), Result = ResultJson.From(result) }); + + /// What a solve was asked: the job's part and stock ids with their quantities, in order. + internal static List Fingerprint(NestJob job) => + job.Parts.Select(p => $"part:{p.Id}:{p.Quantity}") + .Concat(job.Plates.Select(s => $"stock:{s.Id}:{s.Quantity?.ToString() ?? "*"}")) + .ToList(); + + public sealed class SolveJson + { + public List Request { get; set; } = new(); + public ResultJson Result { get; set; } + } + + public sealed class ResultJson + { + public NestJobStatus Status { get; set; } + public NestJobStopReason StopReason { get; set; } + public List Plates { get; set; } = new(); + public List Fulfillment { get; set; } = new(); + public List StockUsage { get; set; } = new(); + + internal static ResultJson From(NestJobResult result) => new() + { + Status = result.Status, + StopReason = result.StopReason, + Plates = result.Plates.Select(p => new PlateJson + { + PlateIndex = p.PlateIndex, + Stock = NestStockJson.From(p.Stock), + Placements = p.Placements.ToList(), + }).ToList(), + Fulfillment = result.Fulfillment.ToList(), + StockUsage = result.StockUsage.ToList(), + }; + + /// The raw result, with each sheet that matches one of 's + /// stock entries using that entry itself, as an engine solving the job would. + internal NestJobResult ToResult(NestJob job) + { + var plates = new List(Plates.Count); + foreach (var plate in Plates) + { + var stock = plate.Stock?.ToStock() ?? throw new FormatException("A result sheet has no stock."); + var own = job.Plates.FirstOrDefault(s => RepeatedPlateGrouping.SameStock(s, stock)); + plates.Add(new NestJobPlateResult(plate.PlateIndex, own ?? stock, plate.Placements ?? new())); + } + return new NestJobResult(Status, StopReason, plates, Fulfillment ?? new(), StockUsage ?? new()); + } + } + + public sealed class PlateJson + { + public int PlateIndex { get; set; } + public NestStockJson Stock { get; set; } + public List Placements { get; set; } = new(); + } +} diff --git a/OpenNest.Engine/Jobs/Remote/NestStockJson.cs b/OpenNest.Engine/Jobs/Remote/NestStockJson.cs new file mode 100644 index 00000000..3bda0e3d --- /dev/null +++ b/OpenNest.Engine/Jobs/Remote/NestStockJson.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using OpenNest.Geometry; + +namespace OpenNest.Engine.Jobs.Remote; + +/// +/// Wire form of one : every setting an engine or validator reads, +/// clamps in plate coordinates exactly as given and the machine rail they came from, so a +/// sheet rebuilt by equals the original bit for bit +/// (). +/// +public sealed class NestStockJson +{ + public string Id { get; set; } + public double Width { get; set; } + public double Length { get; set; } + public int? Quantity { get; set; } + public double PartSpacing { get; set; } + public double EdgeLeft { get; set; } + public double EdgeBottom { get; set; } + public double EdgeRight { get; set; } + public double EdgeTop { get; set; } + public int Quadrant { get; set; } = 1; + public double? Cost { get; set; } + public List ClampZones { get; set; } = new(); + public ClampRailJson ClampRail { get; set; } + public OutlineJson Outline { get; set; } + + public static NestStockJson From(NestPlateStock stock) + { + ArgumentNullException.ThrowIfNull(stock); + return new NestStockJson + { + Id = stock.Id, + Width = stock.Size.Width, + Length = stock.Size.Length, + Quantity = stock.Quantity, + PartSpacing = stock.PartSpacing, + EdgeLeft = stock.EdgeSpacing.Left, + EdgeBottom = stock.EdgeSpacing.Bottom, + EdgeRight = stock.EdgeSpacing.Right, + EdgeTop = stock.EdgeSpacing.Top, + Quadrant = stock.Quadrant, + Cost = stock.Cost, + ClampZones = stock.SheetClampZones.Select(z => new ClampZoneJson + { + Id = z.Id, + X = z.Bounds.X, + Y = z.Bounds.Y, + Length = z.Bounds.Length, + Width = z.Bounds.Width, + }).ToList(), + ClampRail = stock.ClampRail is { } rail ? new ClampRailJson + { + Width = rail.Width, + Depth = rail.Depth, + RemovalMinutes = rail.RemovalMinutes, + Clearance = rail.Clearance, + Clamps = rail.Clamps.Select(c => new RailClampJson { Edge = c.Edge, Center = c.Center }).ToList(), + } : null, + Outline = stock.Outline is { } outline ? new OutlineJson + { + Points = outline.Points.Select(PointJson.From).ToList(), + Obstacles = outline.Obstacles.Select(o => o.Select(PointJson.From).ToList()).ToList(), + } : null, + }; + } + + public NestPlateStock ToStock() + { + var size = new Size(Width, Length); + var zones = (ClampZones ?? new()).Select(z => new ClampZone(z.Id, new Box(z.X, z.Y, z.Length, z.Width))).ToList(); + var outline = Outline is { } o + ? new SheetOutline(o.Points.Select(p => p.ToVector()), + (o.Obstacles ?? new()).Select(ob => ob.Select(p => p.ToVector()))) + : null; + return new NestPlateStock(Id, size, Quantity, PartSpacing, + new Spacing(EdgeLeft, EdgeBottom, EdgeRight, EdgeTop), Quadrant, Cost, zones, outline) + { + ClampRail = ClampRail is { } rail + ? new ClampRail(rail.Width, rail.Depth, + (rail.Clamps ?? new()).Select(c => new RailClamp(c.Edge, c.Center)), + rail.RemovalMinutes, rail.Clearance) + : null, + }; + } + + public sealed class ClampZoneJson + { + public string Id { get; set; } + public double X { get; set; } + public double Y { get; set; } + public double Length { get; set; } + public double Width { get; set; } + } + + public sealed class ClampRailJson + { + public double Width { get; set; } + public double Depth { get; set; } + public double RemovalMinutes { get; set; } + public double? Clearance { get; set; } + public List Clamps { get; set; } = new(); + } + + public sealed class RailClampJson + { + public ClampEdge Edge { get; set; } + public double Center { get; set; } + } + + public sealed class OutlineJson + { + public List Points { get; set; } = new(); + public List> Obstacles { get; set; } = new(); + } + + public sealed class PointJson + { + public double X { get; set; } + public double Y { get; set; } + + public static PointJson From(Vector v) => new() { X = v.X, Y = v.Y }; + + public Vector ToVector() => new(X, Y); + } +} diff --git a/OpenNest.Engine/Jobs/Remote/RecordingNestingEngine.cs b/OpenNest.Engine/Jobs/Remote/RecordingNestingEngine.cs new file mode 100644 index 00000000..b99d3e49 --- /dev/null +++ b/OpenNest.Engine/Jobs/Remote/RecordingNestingEngine.cs @@ -0,0 +1,32 @@ +using System; +using System.Threading; + +namespace OpenNest.Engine.Jobs.Remote; + +/// +/// Passes every solve to another engine and records what it was asked and what it returned +/// in . A solve that throws records nothing. The nest server's worker +/// wraps the selected engine in one, so the client can replay the run (). +/// +public sealed class RecordingNestingEngine : INestingEngine +{ + private readonly INestingEngine inner; + + public RecordingNestingEngine(INestingEngine inner) + { + this.inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public NestSolveTranscript Transcript { get; } = new(); + + public NestJobResult Solve(NestJob job, IProgress progress = null, CancellationToken token = default) + { + var result = inner.Solve(job, progress, token); + if (result != null) + { + lock (Transcript) + Transcript.Add(job, result); + } + return result; + } +} diff --git a/OpenNest.Engine/Jobs/Remote/RemoteNestSubmission.cs b/OpenNest.Engine/Jobs/Remote/RemoteNestSubmission.cs new file mode 100644 index 00000000..ce023a5f --- /dev/null +++ b/OpenNest.Engine/Jobs/Remote/RemoteNestSubmission.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using OpenNest.Engine.Jobs.Adapters; + +namespace OpenNest.Engine.Jobs.Remote; + +/// +/// The client's side of a nest solved elsewhere. Created when the job is sent, it snapshots +/// each item's geometry; then replays the pipeline on this machine with +/// the engine answers recorded remotely (), so the proposal is +/// validated against, and bound to, the caller's own drawings exactly as a local run would be. +/// Nothing is committed: the caller reviews and keeps the result as it would a local one. +/// +public sealed class RemoteNestSubmission +{ + private readonly List<(NestItem Item, PartGeometrySnapshot Geometry)> sent = new(); + + public RemoteNestSubmission(NestPipelineRequest request) + { + Request = request ?? throw new ArgumentNullException(nameof(request)); + ArgumentNullException.ThrowIfNull(request.Items); + for (var i = 0; i < request.Items.Count; i++) + { + var item = request.Items[i]; + if (item != null && item.Quantity > 0) + sent.Add((item, DrawingJobMapper.FromItem($"part-{i}", item).Geometry)); + } + } + + public NestPipelineRequest Request { get; } + + /// + /// Replays the remote run. Throws when a drawing + /// changed after the job was sent or the run does not replay here; the caller should run + /// the nest again rather than keep anything. + /// + public NestPipelineResult Complete(NestSolveTranscript transcript, CancellationToken token = default) + { + ArgumentNullException.ThrowIfNull(transcript); + foreach (var (item, geometry) in sent) + { + if (item.Drawing?.Program == null + || !NestPipelineDrawingFreshness.Same(geometry, PartGeometrySnapshot.FromProgram(item.Drawing.Program))) + throw new NestReplayMismatchException( + $"Drawing '{item.Drawing?.Name}' changed after the nest was sent to the server."); + } + + var replay = new ReplayNestingEngine(transcript); + var result = NestPipeline.Run(replay, Request.EngineName, Request, progress: null, token); + if (!replay.Finished) + throw new NestReplayMismatchException("The nest needed fewer engine runs here than it did on the server."); + return result; + } +} diff --git a/OpenNest.Engine/Jobs/Remote/ReplayNestingEngine.cs b/OpenNest.Engine/Jobs/Remote/ReplayNestingEngine.cs new file mode 100644 index 00000000..c9871748 --- /dev/null +++ b/OpenNest.Engine/Jobs/Remote/ReplayNestingEngine.cs @@ -0,0 +1,44 @@ +using System; +using System.Linq; +using System.Threading; + +namespace OpenNest.Engine.Jobs.Remote; + +/// +/// Answers each solve with the next recorded result of a , +/// after checking it is asked the same job the recording engine was. A different question +/// (the drawings no longer bundle or recover the same way) or one too many fails the replay +/// with ; nothing is ever solved here. +/// +internal sealed class ReplayNestingEngine : INestingEngine +{ + private readonly NestSolveTranscript transcript; + private int next; + + public ReplayNestingEngine(NestSolveTranscript transcript) + { + this.transcript = transcript ?? throw new ArgumentNullException(nameof(transcript)); + } + + /// True once every recorded solve has been replayed. + public bool Finished => next == transcript.Solves.Count; + + public NestJobResult Solve(NestJob job, IProgress progress = null, CancellationToken token = default) + { + ArgumentNullException.ThrowIfNull(job); + token.ThrowIfCancellationRequested(); + if (next >= transcript.Solves.Count) + throw new NestReplayMismatchException("The nest needed more engine runs here than it did on the server."); + var solve = transcript.Solves[next++]; + if (solve.Request == null + || !solve.Request.SequenceEqual(NestSolveTranscript.Fingerprint(job), StringComparer.Ordinal)) + throw new NestReplayMismatchException("The nest asked the engine a different question here than it did on the server."); + return solve.Result.ToResult(job); + } +} + +/// The server's result cannot be replayed against the drawings and stock here. +public sealed class NestReplayMismatchException : InvalidOperationException +{ + public NestReplayMismatchException(string message) : base(message) { } +} diff --git a/OpenNest.Server.Tests/DrawingLibraryDatabaseTests.cs b/OpenNest.Server.Tests/DrawingLibraryDatabaseTests.cs index 37ee6989..3753c60c 100644 --- a/OpenNest.Server.Tests/DrawingLibraryDatabaseTests.cs +++ b/OpenNest.Server.Tests/DrawingLibraryDatabaseTests.cs @@ -23,7 +23,7 @@ public sealed class DrawingLibraryDatabaseTests using (var database = new NestDatabase(factory.DatabasePath, pooling: false)) Assert.Empty(database.QueryDrawings(new DrawingLibraryQuery { IncludeArchived = true }).Items); Assert.Equal(before, NestDigest(factory.DatabasePath)); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); } using var upgraded = new NestDatabase(factory.DatabasePath, pooling: false); @@ -44,7 +44,7 @@ public sealed class DrawingLibraryDatabaseTests using (var database = new NestDatabase(factory.DatabasePath, pooling: false)) { - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); Assert.Equal(before, NestDigest(factory.DatabasePath)); // Two distinct blobs across four revisions: the bracket's notes edit and the // plate both reuse the bracket's first package. @@ -81,7 +81,7 @@ public sealed class DrawingLibraryDatabaseTests { using var factory = new ServerFactory(); using (new NestDatabase(factory.DatabasePath, pooling: false)) { } - Execute(factory.DatabasePath, "PRAGMA user_version=15"); + Execute(factory.DatabasePath, "PRAGMA user_version=16"); Assert.Throws(() => new NestDatabase(factory.DatabasePath, pooling: false)); } diff --git a/OpenNest.Server.Tests/DrawingSetDatabaseTests.cs b/OpenNest.Server.Tests/DrawingSetDatabaseTests.cs index b9295bde..6aec604f 100644 --- a/OpenNest.Server.Tests/DrawingSetDatabaseTests.cs +++ b/OpenNest.Server.Tests/DrawingSetDatabaseTests.cs @@ -23,7 +23,7 @@ public sealed class DrawingSetDatabaseTests using (var database = new NestDatabase(factory.DatabasePath, pooling: false)) Assert.Empty(database.QueryDrawingSets(new DrawingSetQuery { IncludeArchived = true }).Items); Assert.Equal(before, Digest(factory.DatabasePath)); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); } using var upgraded = new NestDatabase(factory.DatabasePath, pooling: false); @@ -48,7 +48,7 @@ public sealed class DrawingSetDatabaseTests Assert.Equal(before, Digest(factory.DatabasePath)); Execute(factory.DatabasePath, "DROP TABLE ix_drawing_set_members_drawing"); using var database = new NestDatabase(factory.DatabasePath, pooling: false); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); } [Fact] diff --git a/OpenNest.Server.Tests/MaterialListTests.cs b/OpenNest.Server.Tests/MaterialListTests.cs index 83c60812..637bc44d 100644 --- a/OpenNest.Server.Tests/MaterialListTests.cs +++ b/OpenNest.Server.Tests/MaterialListTests.cs @@ -25,7 +25,7 @@ public sealed class MaterialListTests using var upgraded = new NestDatabase(factory.DatabasePath, pooling: false); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); Assert.Empty(upgraded.ListMaterials()); Assert.Equal((" Stainless ", "304"), (upgraded.Get(nest)!.Material, upgraded.Get(nest)!.MaterialGrade)); Assert.Equal(drawing.MaterialName, upgraded.GetDrawing(drawing.Id)!.MaterialName); diff --git a/OpenNest.Server.Tests/NestJobApiTests.cs b/OpenNest.Server.Tests/NestJobApiTests.cs new file mode 100644 index 00000000..99c40c8a --- /dev/null +++ b/OpenNest.Server.Tests/NestJobApiTests.cs @@ -0,0 +1,184 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using OpenNest.Api; +using OpenNest.Data; +using OpenNest.Engine; +using OpenNest.Engine.Jobs; +using OpenNest.Engine.Jobs.Remote; +using OpenNest.Geometry; +using OpenNest.Server.Worker; + +namespace OpenNest.Server.Tests; + +/// +/// Server nesting over HTTP: the desktop's client submits a real job package, the worker +/// solves it through the in-memory server, and the desktop replays the result against its own +/// drawings. Queue rules themselves are pinned by . +/// +public class NestJobApiTests +{ + private static OpenNest.CNC.Program Rectangle(double width, double length) + { + var program = new OpenNest.CNC.Program(); + program.MoveTo(0, 0); + program.LineTo(width, 0); + program.LineTo(width, length); + program.LineTo(0, length); + program.LineTo(0, 0); + return program; + } + + private static NestPipelineRequest Request(string engine = "Rectangles") + { + var template = new Plate(48, 96) { EdgeSpacing = new Spacing(0.25, 0.25, 0.25, 0.25), PartSpacing = 0.25 }; + return new NestPipelineRequest(engine, + new[] + { + new NestItem { Drawing = new Drawing("bracket", Rectangle(10, 20)), Quantity = 9 }, + new NestItem { Drawing = new Drawing("gusset", Rectangle(6.125, 6.125)), Quantity = 14, Priority = 2 }, + }, + NestStockBuilder.FromTemplate(template, null), + new NestJobOptions(maxPlates: 5)); + } + + private static (RemoteNestJobs Jobs, NestJobWorker Worker) Connect(ServerFactory factory) + { + var http = factory.CreateClient(); + var jobs = new RemoteNestJobs(http, http.BaseAddress!.ToString()); + return (jobs, new NestJobWorker(jobs, NullLogger.Instance)); + } + + private static NestJobSubmission Describe(NestPipelineRequest request) => + new() { Title = "N26-0042", EngineName = request.EngineName, SubmittedBy = "aj@shop-pc" }; + + [Fact] + public async Task TheWorkerSolvesASubmittedJobAndTheDesktopKeepsTheSameNest() + { + using var factory = new ServerFactory(); + var (jobs, worker) = Connect(factory); + var request = Request(); + var submission = new RemoteNestSubmission(request); + + var job = await jobs.SubmitAsync(Describe(request), NestJobPackage.Write(request)); + Assert.Equal(NestJobState.Queued, job.State); + Assert.Equal("N26-0042", job.Title); + + Assert.True(await worker.RunOnceAsync(CancellationToken.None)); + Assert.False(await worker.RunOnceAsync(CancellationToken.None)); + + var done = (await jobs.GetAsync(job.Id))!; + Assert.Equal(NestJobState.Succeeded, done.State); + var result = submission.Complete(NestSolveTranscript.FromJson(await jobs.GetResultAsync(job.Id))); + + Assert.True(result.IsValid, string.Join("; ", result.Violations)); + Assert.Empty(result.Unplaced); + var parts = result.Plates.SelectMany(p => Enumerable.Repeat(p.Parts, p.Quantity)).SelectMany(p => p).ToList(); + Assert.Equal(9, parts.Count(p => ReferenceEquals(p.BaseDrawing, request.Items[0].Drawing))); + Assert.Equal(14, parts.Count(p => ReferenceEquals(p.BaseDrawing, request.Items[1].Drawing))); + + // The same nest a local run proposes. + var local = NestPipeline.Run(request); + Assert.Equal(local.Plates.SelectMany(p => p.Parts).Select(p => (p.BaseDrawing.Name, p.Location.X, p.Location.Y)), + result.Plates.SelectMany(p => p.Parts).Select(p => (p.BaseDrawing.Name, p.Location.X, p.Location.Y))); + } + + [Fact] + public void APackageReadsBackAsTheSameRequest() + { + var request = Request("Default"); + var shared = request.Items[0].Drawing; + request = request with { Items = request.Items.Append(new NestItem { Drawing = shared, Quantity = 0 }).ToArray() }; + + var back = NestJobPackage.Read(NestJobPackage.Write(request, Units.Millimeters)); + + Assert.Equal("Default", back.EngineName); + Assert.Equal(request.Items.Select(i => (i.Drawing.Name, i.Quantity, i.Priority)), back.Items.Select(i => (i.Drawing.Name, i.Quantity, i.Priority))); + Assert.Same(back.Items[0].Drawing, back.Items[2].Drawing); + Assert.Equal(request.Stock.Select(s => s.Id), back.Stock.Select(s => s.Id)); + Assert.Equal(5, back.Options!.MaxPlates); + Assert.Throws(() => NestJobPackage.Read(Encoding.UTF8.GetBytes("not a zip"))); + } + + [Fact] + public async Task APlugInEngineFailsTheJobWithAReason() + { + using var factory = new ServerFactory(); + var (jobs, worker) = Connect(factory); + var request = Request() with { EngineName = "MyPlugIn" }; + + var job = await jobs.SubmitAsync(Describe(request), NestJobPackage.Write(request)); + await worker.RunOnceAsync(CancellationToken.None); + + var failed = (await jobs.GetAsync(job.Id))!; + Assert.Equal(NestJobState.Failed, failed.State); + Assert.Contains("MyPlugIn", failed.Error); + await Assert.ThrowsAsync(() => jobs.GetResultAsync(job.Id)); + } + + [Fact] + public async Task AnUnreadablePackageFailsTheJob() + { + using var factory = new ServerFactory(); + var (jobs, worker) = Connect(factory); + + var job = await jobs.SubmitAsync(new NestJobSubmission { EngineName = "Default" }, Encoding.UTF8.GetBytes("junk")); + await worker.RunOnceAsync(CancellationToken.None); + + var failed = (await jobs.GetAsync(job.Id))!; + Assert.Equal(NestJobState.Failed, failed.State); + Assert.Contains("could not be read", failed.Error); + } + + [Fact] + public async Task CancellingOverHttpStopsAQueuedJobAndFlagsARunningOne() + { + using var factory = new ServerFactory(); + var (jobs, _) = Connect(factory); + var request = Request(); + var running = await jobs.SubmitAsync(Describe(request), NestJobPackage.Write(request)); + var queued = await jobs.SubmitAsync(Describe(request), NestJobPackage.Write(request)); + Assert.Equal(1, queued.QueuePosition); + + var claimed = await jobs.ClaimAsync("w1"); + Assert.Equal(running.Id, claimed!.Id); + Assert.Null(await jobs.ClaimAsync("w2")); + + Assert.Equal(NestJobState.Cancelled, (await jobs.CancelAsync(queued.Id))!.State); + Assert.True((await jobs.CancelAsync(running.Id))!.CancelRequested); + Assert.True((await jobs.HeartbeatAsync(running.Id, "w1"))!.CancelRequested); + Assert.Null(await jobs.HeartbeatAsync(running.Id, "w2")); + Assert.True(await jobs.ConfirmCancelledAsync(running.Id, "w1")); + Assert.Equal(NestJobState.Cancelled, (await jobs.GetAsync(running.Id))!.State); + Assert.False(await jobs.CompleteAsync(running.Id, "w1", "{}")); + } + + [Fact] + public async Task UnknownJobsAndBadRequestsAreRefused() + { + using var factory = new ServerFactory(); + var http = factory.CreateClient(); + var (jobs, _) = Connect(factory); + + Assert.Null(await jobs.GetAsync(Guid.NewGuid())); + Assert.Null(await jobs.CancelAsync(Guid.NewGuid())); + Assert.Null(await jobs.ClaimAsync("w1")); + Assert.Equal(HttpStatusCode.NotFound, (await http.GetAsync($"api/nest-jobs/{Guid.NewGuid()}/package")).StatusCode); + Assert.Equal(HttpStatusCode.BadRequest, + (await http.PostAsync("api/nest-jobs", new StringContent("{}", Encoding.UTF8, "application/json"))).StatusCode); + Assert.Equal(HttpStatusCode.BadRequest, + (await http.PostAsync("api/nest-jobs/claim", new StringContent("{}", Encoding.UTF8, "application/json"))).StatusCode); + + using var noEngine = new MultipartFormDataContent + { + { new StringContent("{\"title\":\"x\"}", Encoding.UTF8, "application/json"), "metadata" }, + { new ByteArrayContent(new byte[] { 1 }), "file", "job.zip" }, + }; + Assert.Equal(HttpStatusCode.BadRequest, (await http.PostAsync("api/nest-jobs", noEngine)).StatusCode); + + var request = Request(); + var job = await jobs.SubmitAsync(Describe(request), NestJobPackage.Write(request)); + var early = await http.GetAsync($"api/nest-jobs/{job.Id}/result"); + Assert.Equal(HttpStatusCode.Conflict, early.StatusCode); + } +} diff --git a/OpenNest.Server.Tests/NestJobQueueTests.cs b/OpenNest.Server.Tests/NestJobQueueTests.cs new file mode 100644 index 00000000..a195ce65 --- /dev/null +++ b/OpenNest.Server.Tests/NestJobQueueTests.cs @@ -0,0 +1,309 @@ +using OpenNest.Data; + +namespace OpenNest.Server.Tests; + +/// +/// The server nesting queue's states and leases, driven with explicit times so lease expiry, +/// crash recovery and retention are tested without waiting. +/// +public class NestJobQueueTests +{ + private static readonly DateTimeOffset T0 = new(2026, 11, 2, 8, 0, 0, TimeSpan.Zero); + private static readonly byte[] Package = { 1, 2, 3 }; + + private static Guid Submit(NestDatabase db, string title = "N26-0001", DateTimeOffset? at = null) + { + var id = Guid.NewGuid(); + db.InsertJob(id, new NestJobSubmission { Title = title, EngineName = "Default", SubmittedBy = "aj@pc" }, Package, at ?? T0); + return id; + } + + private static (ServerFactory Factory, NestDatabase Db) Open() + { + var factory = new ServerFactory(); + return (factory, new NestDatabase(factory.DatabasePath, pooling: false)); + } + + [Fact] + public void QueuedJobsReportHowManyAreAhead() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var first = Submit(db); + var second = Submit(db, "N26-0002"); + + var job = db.GetJob(second)!; + Assert.Equal(NestJobState.Queued, job.State); + Assert.Equal(1, job.QueuePosition); + Assert.Equal(0, db.GetJob(first)!.QueuePosition); + Assert.Equal("N26-0002", job.Title); + Assert.Equal(3, job.PackageSize); + Assert.Equal(Package, db.GetJobPackage(second)); + Assert.Null(db.GetJobResult(second)); + } + } + + [Fact] + public void OnlyOneJobRunsAtATime() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var first = Submit(db); + var second = Submit(db, "second"); + + var claimed = db.ClaimJob("w1", T0); + Assert.Equal(first, claimed!.Id); + Assert.Equal(NestJobState.Running, claimed.State); + Assert.Equal(1, claimed.Attempts); + Assert.Null(db.ClaimJob("w1", T0.AddSeconds(1))); + Assert.Null(db.ClaimJob("w2", T0.AddSeconds(1))); + Assert.Equal(0, db.GetJob(second)!.QueuePosition); + + Assert.True(db.CompleteJob(first, "w1", "{\"solves\":[]}", T0.AddMinutes(1))); + Assert.Equal(second, db.ClaimJob("w2", T0.AddMinutes(1))!.Id); + } + } + + [Fact] + public void ConcurrentClaimsGiveTheJobToOneWorker() + { + var (factory, db) = Open(); + using (factory) using (db) + { + Submit(db); + Submit(db, "second"); + + var claims = Enumerable.Range(0, 16).AsParallel() + .Select(i => db.ClaimJob($"w{i}", T0)) + .Where(job => job is not null) + .ToList(); + + Assert.Single(claims); + } + } + + [Fact] + public void CompletedJobKeepsItsResultAndDropsItsPackage() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + db.ClaimJob("w1", T0); + + Assert.False(db.CompleteJob(id, "someone-else", "{}", T0)); + Assert.True(db.CompleteJob(id, "w1", "{\"version\":1}", T0.AddMinutes(3))); + + var job = db.GetJob(id)!; + Assert.Equal(NestJobState.Succeeded, job.State); + Assert.Equal(T0.AddMinutes(3).UtcDateTime, job.FinishedAt); + Assert.Equal("{\"version\":1}", db.GetJobResult(id)); + Assert.Null(db.GetJobPackage(id)); + Assert.Null(db.HeartbeatJob(id, "w1", T0.AddMinutes(4))); + } + } + + [Fact] + public void CancellingAQueuedJobEndsItAtOnce() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + + var job = db.CancelJob(id, T0)!; + + Assert.Equal(NestJobState.Cancelled, job.State); + Assert.Null(db.ClaimJob("w1", T0)); + Assert.Null(db.CancelJob(Guid.NewGuid(), T0)); + } + } + + [Fact] + public void CancellingARunningJobStopsItAtTheNextHeartbeat() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + db.ClaimJob("w1", T0); + Assert.False(db.HeartbeatJob(id, "w1", T0.AddSeconds(20))!.CancelRequested); + + var job = db.CancelJob(id, T0.AddSeconds(30))!; + Assert.Equal(NestJobState.Running, job.State); + Assert.True(job.CancelRequested); + Assert.True(db.HeartbeatJob(id, "w1", T0.AddSeconds(40))!.CancelRequested); + + Assert.True(db.ConfirmJobCancelled(id, "w1", T0.AddSeconds(41))); + Assert.Equal(NestJobState.Cancelled, db.GetJob(id)!.State); + } + } + + [Fact] + public void AResultArrivingAfterACancelRequestIsDropped() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + db.ClaimJob("w1", T0); + db.CancelJob(id, T0.AddSeconds(5)); + + Assert.True(db.CompleteJob(id, "w1", "{}", T0.AddSeconds(6))); + + Assert.Equal(NestJobState.Cancelled, db.GetJob(id)!.State); + Assert.Null(db.GetJobResult(id)); + } + } + + [Fact] + public void AJobWhoseWorkerStoppedRespondingGoesBackToTheQueue() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + db.ClaimJob("crashed", T0); + + // Inside the lease the job stays with its worker. + Assert.Null(db.ClaimJob("w2", T0 + NestJobRules.Lease - TimeSpan.FromSeconds(1))); + + var retried = db.ClaimJob("w2", T0 + NestJobRules.Lease + TimeSpan.FromSeconds(1)); + + Assert.Equal(id, retried!.Id); + Assert.Equal(2, retried.Attempts); + Assert.Null(db.HeartbeatJob(id, "crashed", T0 + NestJobRules.Lease + TimeSpan.FromSeconds(2))); + Assert.False(db.CompleteJob(id, "crashed", "{}", T0 + NestJobRules.Lease + TimeSpan.FromSeconds(2))); + Assert.True(db.CompleteJob(id, "w2", "{}", T0 + NestJobRules.Lease + TimeSpan.FromSeconds(3))); + } + } + + [Fact] + public void HeartbeatsKeepALongJobWithItsWorker() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + db.ClaimJob("w1", T0); + var now = T0; + for (var i = 0; i < 30; i++) + { + now += NestJobRules.HeartbeatInterval; + Assert.NotNull(db.HeartbeatJob(id, "w1", now)); + Assert.Null(db.ClaimJob("w2", now)); + } + Assert.Equal(1, db.GetJob(id)!.Attempts); + } + } + + [Fact] + public void AJobThatKeepsStoppingItsWorkerFails() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + var now = T0; + for (var attempt = 1; attempt <= NestJobRules.MaxAttempts; attempt++) + { + Assert.Equal(attempt, db.ClaimJob($"w{attempt}", now)!.Attempts); + now += NestJobRules.Lease + TimeSpan.FromSeconds(1); + } + + Assert.Null(db.ClaimJob("next", now)); + + var job = db.GetJob(id)!; + Assert.Equal(NestJobState.Failed, job.State); + Assert.Contains($"{NestJobRules.MaxAttempts} times", job.Error); + Assert.Null(db.GetJobPackage(id)); + } + } + + [Fact] + public void ACancelRequestedJobWhoseWorkerStoppedEndsCancelled() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + db.ClaimJob("crashed", T0); + db.CancelJob(id, T0.AddSeconds(1)); + + Assert.Null(db.ClaimJob("w2", T0 + NestJobRules.Lease + TimeSpan.FromSeconds(1))); + Assert.Equal(NestJobState.Cancelled, db.GetJob(id)!.State); + } + } + + [Fact] + public void AReleasedJobGoesBackWithoutSpendingAnAttempt() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + db.ClaimJob("stopping", T0); + + Assert.True(db.ReleaseJob(id, "stopping", T0.AddSeconds(5))); + var job = db.GetJob(id)!; + Assert.Equal(NestJobState.Queued, job.State); + Assert.Equal(0, job.Attempts); + Assert.Null(job.StartedAt); + Assert.Equal(1, db.ClaimJob("w2", T0.AddSeconds(6))!.Attempts); + } + } + + [Fact] + public void FailedJobKeepsItsError() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var id = Submit(db); + db.ClaimJob("w1", T0); + + Assert.True(db.FailJob(id, "w1", new string('x', 2000), T0.AddSeconds(1))); + + var job = db.GetJob(id)!; + Assert.Equal(NestJobState.Failed, job.State); + Assert.Equal(NestJobRules.MaxTextLength, job.Error.Length); + } + } + + [Fact] + public void FinishedJobsAreRemovedAfterTheRetention() + { + var (factory, db) = Open(); + using (factory) using (db) + { + var old = Submit(db); + db.CancelJob(old, T0); + var waiting = Submit(db, "waiting", T0); + + db.ClaimJob("w1", T0 + NestJobRules.Retention + TimeSpan.FromMinutes(1)); + + Assert.Null(db.GetJob(old)); + Assert.Equal(NestJobState.Running, db.GetJob(waiting)!.State); + } + } + + [Fact] + public void JobsSurviveARestart() + { + using var factory = new ServerFactory(); + Guid id; + using (var db = new NestDatabase(factory.DatabasePath, pooling: false)) + { + id = Submit(db); + db.ClaimJob("w1", T0); + } + + using (var db = new NestDatabase(factory.DatabasePath, pooling: false)) + { + Assert.Equal(NestJobState.Running, db.GetJob(id)!.State); + Assert.Equal(id, db.ClaimJob("w2", T0 + NestJobRules.Lease + TimeSpan.FromSeconds(1))!.Id); + } + } +} diff --git a/OpenNest.Server.Tests/NestMaterialGradeTests.cs b/OpenNest.Server.Tests/NestMaterialGradeTests.cs index fe835113..b441f370 100644 --- a/OpenNest.Server.Tests/NestMaterialGradeTests.cs +++ b/OpenNest.Server.Tests/NestMaterialGradeTests.cs @@ -75,7 +75,7 @@ public sealed class NestMaterialGradeTests Assert.Equal("Stainless", record.Material); Assert.Equal("", record.MaterialGrade); Assert.Equal("", Assert.Single(upgraded.History(id, 0, 10)!.Items).MaterialGrade); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); } private static NestRecord Record(string name, string material, string grade) => new() diff --git a/OpenNest.Server.Tests/NestStockCodesTests.cs b/OpenNest.Server.Tests/NestStockCodesTests.cs index 0ee2cd0b..aa80b641 100644 --- a/OpenNest.Server.Tests/NestStockCodesTests.cs +++ b/OpenNest.Server.Tests/NestStockCodesTests.cs @@ -119,7 +119,7 @@ public sealed class NestStockCodesTests database.SetDeleted(deleted, false, record.Revision); Assert.Equal(["SHEET-1", "SHEET-2"], database.StockCodesInUse(null).Select(u => u.ItemCode)); } - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); } private static Nest Nest(string name, NestStatus status, params string[][] plateCodes) diff --git a/OpenNest.Server.Tests/NestUnitsTests.cs b/OpenNest.Server.Tests/NestUnitsTests.cs index a0c223cb..5c057aec 100644 --- a/OpenNest.Server.Tests/NestUnitsTests.cs +++ b/OpenNest.Server.Tests/NestUnitsTests.cs @@ -77,7 +77,7 @@ public sealed class NestUnitsTests Assert.Equal([Units.Millimeters, Units.Millimeters], upgraded.History(metric, 0, 10)!.Items.Select(r => r.Units)); Assert.Equal(Units.Inches, upgraded.Get(inch)!.Units); Assert.Equal(Units.Inches, upgraded.Get(unreadable)!.Units); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); } private static NestRecord Record(string name, Units units) => new() diff --git a/OpenNest.Server.Tests/OpenNest.Server.Tests.csproj b/OpenNest.Server.Tests/OpenNest.Server.Tests.csproj index 79ac2fa2..8acaf889 100644 --- a/OpenNest.Server.Tests/OpenNest.Server.Tests.csproj +++ b/OpenNest.Server.Tests/OpenNest.Server.Tests.csproj @@ -25,6 +25,8 @@ + + diff --git a/OpenNest.Server.Tests/SetReleaseDrawingTests.cs b/OpenNest.Server.Tests/SetReleaseDrawingTests.cs index f56bd2cc..09ce9fa8 100644 --- a/OpenNest.Server.Tests/SetReleaseDrawingTests.cs +++ b/OpenNest.Server.Tests/SetReleaseDrawingTests.cs @@ -152,7 +152,7 @@ public sealed class SetReleaseDrawingTests using var upgraded = new NestDatabase(factory.DatabasePath, pooling: false); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); var kept = upgraded.GetRelease(releaseId)!; Assert.Empty(kept.Drawings); Assert.Equal(((Guid?)kit.Id, drawing, 2), (kept.Nests[0].Items.Single().SetId, kept.Nests[0].Items.Single().DrawingId, kept.Nests[0].Items.Single().Quantity)); diff --git a/OpenNest.Server.Tests/SetReleaseTests.cs b/OpenNest.Server.Tests/SetReleaseTests.cs index 4629d670..ea9c5e1a 100644 --- a/OpenNest.Server.Tests/SetReleaseTests.cs +++ b/OpenNest.Server.Tests/SetReleaseTests.cs @@ -295,7 +295,7 @@ public sealed class SetReleaseTests using var upgraded = new NestDatabase(factory.DatabasePath, pooling: false); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); Assert.Equal(0L, Scalar(factory.DatabasePath, "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'set_release%' OR name LIKE 'ix_set_release%'")); var release = upgraded.GetRelease(releaseId)!; Assert.Equal(("Legacy notes", "Legacy operator", false), (release.Notes, release.ReleasedBy, release.IsUndone)); diff --git a/OpenNest.Server.Tests/SharedSettingsTests.cs b/OpenNest.Server.Tests/SharedSettingsTests.cs index 53727632..8d606a30 100644 --- a/OpenNest.Server.Tests/SharedSettingsTests.cs +++ b/OpenNest.Server.Tests/SharedSettingsTests.cs @@ -68,7 +68,7 @@ public sealed class SharedSettingsTests using var upgraded = new NestDatabase(factory.DatabasePath, pooling: false); - Assert.Equal(14L, Scalar(factory.DatabasePath, "PRAGMA user_version")); + Assert.Equal(15L, Scalar(factory.DatabasePath, "PRAGMA user_version")); Assert.Empty(upgraded.ListSharedSettings()); } diff --git a/OpenNest.Server.Worker/NestJobWorker.cs b/OpenNest.Server.Worker/NestJobWorker.cs new file mode 100644 index 00000000..c86ba6e2 --- /dev/null +++ b/OpenNest.Server.Worker/NestJobWorker.cs @@ -0,0 +1,150 @@ +using OpenNest.Api; +using OpenNest.Data; +using OpenNest.Engine.Jobs; +using OpenNest.Engine.Jobs.Remote; + +namespace OpenNest.Server.Worker; + +/// +/// Solves the nest server's queued jobs one at a time. Each job is read from its package and +/// run through with the named built-in engine wrapped in a +/// , with no time limit; the recorded engine answers are +/// the result the desktop replays. A heartbeat keeps the job's lease and carries cancel requests. +/// +public sealed class NestJobWorker +{ + private readonly RemoteNestJobs _jobs; + private readonly ILogger _logger; + private readonly TimeSpan _heartbeat; + + public NestJobWorker(RemoteNestJobs jobs, ILogger logger, TimeSpan? heartbeat = null) + { + _jobs = jobs; + _logger = logger; + _heartbeat = heartbeat ?? NestJobRules.HeartbeatInterval; + } + + public string WorkerId { get; } = $"{Environment.MachineName}-{Guid.NewGuid():N}"; + + /// Called after each heartbeat the server accepts, so a long job still shows the worker alive. + public Action? Alive { get; set; } + + /// Claims and runs one job. False when there was none to run. + public async Task RunOnceAsync(CancellationToken stoppingToken) + { + var job = await _jobs.ClaimAsync(WorkerId, stoppingToken); + if (job is null) + return false; + + _logger.LogInformation("Nest job {Id} ({Title}, {Engine}) started, attempt {Attempt}.", + job.Id, job.Title, job.EngineName, job.Attempts); + using var solve = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + var stop = StopReason.None; + using var beating = new CancellationTokenSource(); + var heartbeat = HeartbeatAsync(job.Id, solve, reason => stop = reason, beating.Token); + try + { + NestPipelineRequest request; + try + { + request = NestJobPackage.Read(await _jobs.GetPackageAsync(job.Id, solve.Token)); + } + catch (FormatException ex) + { + await Report(_jobs.FailAsync(job.Id, WorkerId, $"The nest job could not be read: {ex.Message}")); + return true; + } + if (!NestingEngineRegistry.IsBuiltIn(request.EngineName)) + { + await Report(_jobs.FailAsync(job.Id, WorkerId, + $"The nest server has no '{request.EngineName}' engine; only built-in engines run there.")); + return true; + } + + var recorder = new RecordingNestingEngine(NestingEngineRegistry.Create(request.EngineName)); + await Task.Factory.StartNew(() => NestPipeline.Run(recorder, request.EngineName, request, progress: null, solve.Token), + solve.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); + if (await _jobs.CompleteAsync(job.Id, WorkerId, recorder.Transcript.ToJson(), CancellationToken.None)) + _logger.LogInformation("Nest job {Id} finished.", job.Id); + else + _logger.LogWarning("Nest job {Id} finished after this worker lost it; the result was dropped.", job.Id); + } + catch (OperationCanceledException) when (solve.IsCancellationRequested) + { + switch (stop) + { + case StopReason.CancelRequested: + await Report(_jobs.ConfirmCancelledAsync(job.Id, WorkerId)); + _logger.LogInformation("Nest job {Id} stopped on request.", job.Id); + break; + case StopReason.LeaseLost: + _logger.LogWarning("Nest job {Id} stopped: this worker no longer holds it.", job.Id); + break; + default: + // Shutting down: the job goes back to the front of the queue. + await Report(_jobs.ReleaseAsync(job.Id, WorkerId)); + _logger.LogInformation("Nest job {Id} given back while the worker stops.", job.Id); + break; + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogError(ex, "Nest job {Id} failed.", job.Id); + await Report(_jobs.FailAsync(job.Id, WorkerId, ex.Message)); + } + finally + { + beating.Cancel(); + await heartbeat; + } + return true; + } + + private async Task HeartbeatAsync(Guid id, CancellationTokenSource solve, Action stopped, CancellationToken token) + { + while (!token.IsCancellationRequested) + { + try { await Task.Delay(_heartbeat, token); } + catch (OperationCanceledException) { return; } + try + { + var beat = await _jobs.HeartbeatAsync(id, WorkerId, token); + if (beat is not null) + Alive?.Invoke(); + if (beat is null || beat.CancelRequested) + { + stopped(beat is null ? StopReason.LeaseLost : StopReason.CancelRequested); + solve.Cancel(); + return; + } + } + catch (OperationCanceledException) when (token.IsCancellationRequested) { return; } + catch (Exception ex) when (ex is IOException or HttpRequestException or TaskCanceledException) + { + // The lease outlasts several missed beats; keep solving and try again. + _logger.LogWarning("Heartbeat for nest job {Id} failed: {Message}", id, ex.Message); + } + } + } + + // Reports run after the solve's token may be cancelled, so they never take it. + private async Task Report(Task report) + { + try + { + if (!await report) + _logger.LogWarning("The nest server no longer gave this worker the job it reported on."); + } + catch (Exception ex) when (ex is IOException or HttpRequestException or TaskCanceledException) + { + _logger.LogWarning("Could not report to the nest server: {Message}", ex.Message); + } + } + + private enum StopReason + { + None, + CancelRequested, + LeaseLost, + } +} diff --git a/OpenNest.Server.Worker/NestJobWorkerService.cs b/OpenNest.Server.Worker/NestJobWorkerService.cs new file mode 100644 index 00000000..3c179fea --- /dev/null +++ b/OpenNest.Server.Worker/NestJobWorkerService.cs @@ -0,0 +1,55 @@ +namespace OpenNest.Server.Worker; + +/// +/// Runs until the host stops: straight on to the next job after +/// one finishes, otherwise every . Touches +/// each round and on each heartbeat of a running job; +/// the container health check reads it. +/// +public sealed class NestJobWorkerService : BackgroundService +{ + private readonly NestJobWorker _worker; + private readonly WorkerOptions _options; + private readonly ILogger _logger; + + public NestJobWorkerService(NestJobWorker worker, WorkerOptions options, ILogger logger) + { + _worker = worker; + _options = options; + _logger = logger; + _worker.Alive = Touch; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Nest worker {Id} polling {Server}.", _worker.WorkerId, _options.ServerUrl); + while (!stoppingToken.IsCancellationRequested) + { + Touch(); + var ran = false; + try + { + ran = await _worker.RunOnceAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is IOException or HttpRequestException or TaskCanceledException) + { + _logger.LogWarning("Nest server unavailable: {Message}", ex.Message); + } + + if (ran) + continue; + try { await Task.Delay(_options.PollInterval, stoppingToken); } + catch (OperationCanceledException) { return; } + } + } + + private void Touch() + { + try { File.WriteAllText(_options.AliveFile, DateTime.UtcNow.ToString("o")); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } +} diff --git a/OpenNest.Server.Worker/OpenNest.Server.Worker.csproj b/OpenNest.Server.Worker/OpenNest.Server.Worker.csproj new file mode 100644 index 00000000..9bf0e099 --- /dev/null +++ b/OpenNest.Server.Worker/OpenNest.Server.Worker.csproj @@ -0,0 +1,22 @@ + + + + net8.0 + OpenNest.Server.Worker + OpenNest.Server.Worker + enable + enable + + + + + + + + + + + + + diff --git a/OpenNest.Server.Worker/Program.cs b/OpenNest.Server.Worker/Program.cs new file mode 100644 index 00000000..82467ddb --- /dev/null +++ b/OpenNest.Server.Worker/Program.cs @@ -0,0 +1,23 @@ +using System.Diagnostics; +using OpenNest.Data; +using OpenNest.Server.Worker; + +// The nest server's worker: takes queued whole-job nests from OPENNEST_SERVER_URL and solves +// them one at a time. Run it in its own container with CPU and memory limits (compose.server.yaml) +// so nesting never slows nest open and save. See docs/server-nesting.md. +var options = WorkerOptions.FromEnvironment(Environment.GetEnvironmentVariable); + +// Below the server and anything else on the host when they share CPUs; the container's +// CPU limit is the real bound. +try { Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.BelowNormal; } +catch (Exception ex) when (ex is System.ComponentModel.Win32Exception or PlatformNotSupportedException) { } + +var builder = Host.CreateApplicationBuilder(args); +builder.Services.AddSingleton(options); +builder.Services.AddSingleton(_ => new RemoteNestJobs(options.ServerUrl)); +builder.Services.AddSingleton(sp => new NestJobWorker( + sp.GetRequiredService(), sp.GetRequiredService>())); +builder.Services.AddHostedService(); +// A running job is given back, not abandoned, when the container stops. +builder.Services.Configure(o => o.ShutdownTimeout = TimeSpan.FromSeconds(20)); +builder.Build().Run(); diff --git a/OpenNest.Server.Worker/WorkerOptions.cs b/OpenNest.Server.Worker/WorkerOptions.cs new file mode 100644 index 00000000..a4bde964 --- /dev/null +++ b/OpenNest.Server.Worker/WorkerOptions.cs @@ -0,0 +1,38 @@ +namespace OpenNest.Server.Worker; + +/// Worker settings read from the environment. +public sealed class WorkerOptions +{ + public const string ServerVariable = "OPENNEST_SERVER_URL"; + public const string PollVariable = "OPENNEST_WORKER_POLL_SECONDS"; + public const string AliveVariable = "OPENNEST_WORKER_ALIVE_FILE"; + + /// The nest server the worker takes jobs from. + public required string ServerUrl { get; init; } + + /// How long to wait before asking again when no job was queued. + public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(5); + + /// Touched every round; the container health check fails when it goes stale. + public string AliveFile { get; init; } = Path.Combine(Path.GetTempPath(), "opennest-worker.alive"); + + public static WorkerOptions FromEnvironment(Func read) + { + var server = read(ServerVariable); + if (string.IsNullOrWhiteSpace(server)) + throw new InvalidOperationException($"{ServerVariable} must name the nest server, for example http://opennest-server:8090/."); + var poll = TimeSpan.FromSeconds(5); + if (read(PollVariable) is { Length: > 0 } text) + { + if (!int.TryParse(text, out var seconds) || seconds is < 1 or > 3600) + throw new InvalidOperationException($"{PollVariable} must be whole seconds from 1 to 3600."); + poll = TimeSpan.FromSeconds(seconds); + } + return new WorkerOptions + { + ServerUrl = server.Trim(), + PollInterval = poll, + AliveFile = read(AliveVariable) is { Length: > 0 } alive ? alive : Path.Combine(Path.GetTempPath(), "opennest-worker.alive"), + }; + } +} diff --git a/OpenNest.Server/Dockerfile b/OpenNest.Server/Dockerfile index a74d914d..6ad4f043 100644 --- a/OpenNest.Server/Dockerfile +++ b/OpenNest.Server/Dockerfile @@ -7,17 +7,30 @@ ARG RUNTIME_IMAGE=mcr.microsoft.com/dotnet/aspnet:8.0 FROM ${SDK_IMAGE} AS build WORKDIR /src +# The worker (OPENNEST_ROLE=worker) solves server nesting jobs, so it also needs the +# engine and the nest file format; the HTTP server references neither. COPY OpenNest.Server/OpenNest.Server.csproj OpenNest.Server/ +COPY OpenNest.Server.Worker/OpenNest.Server.Worker.csproj OpenNest.Server.Worker/ +COPY OpenNest.Api/OpenNest.Api.csproj OpenNest.Api/ +COPY OpenNest.Engine/OpenNest.Engine.csproj OpenNest.Engine/ +COPY OpenNest.IO/OpenNest.IO.csproj OpenNest.IO/ COPY OpenNest.Data/OpenNest.Data.csproj OpenNest.Data/ COPY OpenNest.Core/OpenNest.Core.csproj OpenNest.Core/ -RUN dotnet restore OpenNest.Server/OpenNest.Server.csproj +RUN dotnet restore OpenNest.Server/OpenNest.Server.csproj \ + && dotnet restore OpenNest.Server.Worker/OpenNest.Server.Worker.csproj COPY OpenNest.Server/ OpenNest.Server/ +COPY OpenNest.Server.Worker/ OpenNest.Server.Worker/ +COPY OpenNest.Api/ OpenNest.Api/ +COPY OpenNest.Engine/ OpenNest.Engine/ +COPY OpenNest.IO/ OpenNest.IO/ COPY OpenNest.Data/ OpenNest.Data/ COPY OpenNest.Core/ OpenNest.Core/ ARG VERSION=0.0.0-dev ARG SOURCE_REVISION=unknown RUN dotnet publish OpenNest.Server/OpenNest.Server.csproj -c Release -o /app --no-restore \ + -p:Version="$VERSION" -p:SourceRevisionId="$SOURCE_REVISION" \ + && dotnet publish OpenNest.Server.Worker/OpenNest.Server.Worker.csproj -c Release -o /app/worker --no-restore \ -p:Version="$VERSION" -p:SourceRevisionId="$SOURCE_REVISION" FROM ${RUNTIME_IMAGE} AS runtime @@ -28,6 +41,7 @@ RUN apt-get update \ WORKDIR /app # Runtime files stay root-owned and read-only to the app user. COPY --from=build /app . +COPY --chmod=0755 OpenNest.Server/entrypoint.sh OpenNest.Server/healthcheck.sh /app/ # SQLite database + .nest blobs live here; mount a volume at this path. The directory # belongs to the base image's non-root app user (APP_UID 1654), so a fresh named volume @@ -41,18 +55,20 @@ EXPOSE 8090 USER $APP_UID VOLUME /app/data +# The server answers /healthz; the worker touches its alive file every poll round and heartbeat. HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --start-interval=2s --retries=3 \ - CMD ["curl", "--fail", "--silent", "--show-error", "http://127.0.0.1:8090/healthz"] + CMD ["/app/healthcheck.sh"] ARG VERSION=0.0.0-dev ARG SOURCE_REVISION=unknown ARG RUNTIME_IMAGE LABEL org.opencontainers.image.title="OpenNest.Server" \ - org.opencontainers.image.description="OpenNest nest-storage HTTP service (SQLite)" \ + org.opencontainers.image.description="OpenNest nest-storage HTTP service (SQLite) and server nesting worker" \ org.opencontainers.image.source="https://github.com/ajisaacs/OpenNest" \ org.opencontainers.image.licenses="MIT" \ org.opencontainers.image.version="$VERSION" \ org.opencontainers.image.revision="$SOURCE_REVISION" \ org.opencontainers.image.base.name="$RUNTIME_IMAGE" -ENTRYPOINT ["dotnet", "OpenNest.Server.dll"] +# OPENNEST_ROLE picks the process: server (default) or worker; see docs/server-nesting.md. +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/OpenNest.Server/NestDatabase.Jobs.cs b/OpenNest.Server/NestDatabase.Jobs.cs new file mode 100644 index 00000000..8f992f15 --- /dev/null +++ b/OpenNest.Server/NestDatabase.Jobs.cs @@ -0,0 +1,319 @@ +using System.Globalization; +using Microsoft.Data.Sqlite; +using OpenNest.Data; + +namespace OpenNest.Server; + +/// +/// The server nesting queue (docs/server-nesting.md). A worker claims the oldest queued +/// job and holds it on a lease it renews by heartbeat; only one job runs at a time. A lease that +/// lapses (the worker or its container stopped) puts the job back in the queue, or fails it +/// after . Every method takes the current time so the +/// queue can be tested without waiting. Times are stored as round-trip UTC text, which sorts. +/// +public sealed partial class NestDatabase +{ + private const string NestJobSchema = """ + CREATE TABLE IF NOT EXISTS nest_jobs ( + id TEXT PRIMARY KEY, + seq INTEGER NOT NULL, + title TEXT NOT NULL DEFAULT '', + engineName TEXT NOT NULL, + submittedBy TEXT NOT NULL DEFAULT '', + state TEXT NOT NULL, + cancelRequested INTEGER NOT NULL DEFAULT 0, + submittedAt TEXT NOT NULL, + startedAt TEXT, + finishedAt TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + error TEXT NOT NULL DEFAULT '', + workerId TEXT, + leaseUntil TEXT, + packageSize INTEGER NOT NULL, + package BLOB NOT NULL, + result TEXT + ); + CREATE INDEX IF NOT EXISTS ix_nest_jobs_state ON nest_jobs(state, seq); + PRAGMA user_version=15; + """; + + private const string JobColumns = + "id, title, engineName, submittedBy, state, cancelRequested, submittedAt, startedAt, finishedAt, attempts, error, packageSize, seq"; + + public NestJobRecord InsertJob(Guid id, NestJobSubmission submission, byte[] package, DateTimeOffset now) + { + ArgumentNullException.ThrowIfNull(submission); + ArgumentNullException.ThrowIfNull(package); + lock (_sync) + { + using var command = _connection.CreateCommand(); + command.CommandText = """ + INSERT INTO nest_jobs (id, seq, title, engineName, submittedBy, state, submittedAt, packageSize, package) + VALUES ($id, (SELECT COALESCE(MAX(seq), 0) + 1 FROM nest_jobs), $title, $engine, $by, 'Queued', $now, $size, $package) + """; + command.Parameters.AddWithValue("$id", id.ToString()); + command.Parameters.AddWithValue("$title", submission.Title); + command.Parameters.AddWithValue("$engine", submission.EngineName); + command.Parameters.AddWithValue("$by", submission.SubmittedBy); + command.Parameters.AddWithValue("$now", Stamp(now)); + command.Parameters.AddWithValue("$size", package.LongLength); + command.Parameters.AddWithValue("$package", package); + command.ExecuteNonQuery(); + return GetJob(id)!; + } + } + + public NestJobRecord? GetJob(Guid id) + { + lock (_sync) + { + using var command = _connection.CreateCommand(); + command.CommandText = $"SELECT {JobColumns} FROM nest_jobs WHERE id=$id"; + command.Parameters.AddWithValue("$id", id.ToString()); + using var reader = command.ExecuteReader(); + if (!reader.Read()) + return null; + var record = ReadJob(reader); + if (record.State == NestJobState.Queued) + { + using var ahead = _connection.CreateCommand(); + ahead.CommandText = "SELECT COUNT(*) FROM nest_jobs WHERE state='Queued' AND seq < $seq"; + ahead.Parameters.AddWithValue("$seq", reader.GetInt64(12)); + record.QueuePosition = checked((int)(long)ahead.ExecuteScalar()!); + } + return record; + } + } + + /// The job's package while it is queued or running; finished jobs no longer keep it. + public byte[]? GetJobPackage(Guid id) + { + lock (_sync) + { + using var command = _connection.CreateCommand(); + command.CommandText = "SELECT package FROM nest_jobs WHERE id=$id AND state IN ('Queued', 'Running')"; + command.Parameters.AddWithValue("$id", id.ToString()); + return command.ExecuteScalar() as byte[]; + } + } + + /// A succeeded job's result; null for any other job. + public string? GetJobResult(Guid id) + { + lock (_sync) + { + using var command = _connection.CreateCommand(); + command.CommandText = "SELECT result FROM nest_jobs WHERE id=$id AND state='Succeeded'"; + command.Parameters.AddWithValue("$id", id.ToString()); + return command.ExecuteScalar() as string; + } + } + + /// + /// A queued job is cancelled at once; a running one is marked so its worker stops at the + /// next heartbeat. A finished job is returned unchanged. Null when there is no such job. + /// + public NestJobRecord? CancelJob(Guid id, DateTimeOffset now) + { + lock (_sync) + { + using var command = _connection.CreateCommand(); + command.CommandText = """ + UPDATE nest_jobs SET + state = CASE state WHEN 'Queued' THEN 'Cancelled' ELSE state END, + finishedAt = CASE state WHEN 'Queued' THEN $now ELSE finishedAt END, + package = CASE state WHEN 'Queued' THEN zeroblob(0) ELSE package END, + cancelRequested = CASE state WHEN 'Running' THEN 1 ELSE cancelRequested END + WHERE id=$id + """; + command.Parameters.AddWithValue("$id", id.ToString()); + command.Parameters.AddWithValue("$now", Stamp(now)); + command.ExecuteNonQuery(); + return GetJob(id); + } + } + + /// + /// Gives the oldest queued job on a fresh lease, or null when + /// none is queued or another job still runs. Lapsed leases are settled first, and finished + /// jobs past removed. + /// + public NestJobRecord? ClaimJob(string workerId, DateTimeOffset now) + { + ArgumentException.ThrowIfNullOrWhiteSpace(workerId); + lock (_sync) + { + using var transaction = _connection.BeginTransaction(); + SettleJobLeases(transaction, now); + + using var running = _connection.CreateCommand(); + running.Transaction = transaction; + running.CommandText = "SELECT COUNT(*) FROM nest_jobs WHERE state='Running'"; + if ((long)running.ExecuteScalar()! > 0) + { + transaction.Commit(); + return null; + } + + using var next = _connection.CreateCommand(); + next.Transaction = transaction; + next.CommandText = "SELECT id FROM nest_jobs WHERE state='Queued' ORDER BY seq LIMIT 1"; + if (next.ExecuteScalar() is not string id) + { + transaction.Commit(); + return null; + } + + using var claim = _connection.CreateCommand(); + claim.Transaction = transaction; + claim.CommandText = """ + UPDATE nest_jobs SET state='Running', workerId=$worker, leaseUntil=$lease, + startedAt=$now, attempts=attempts+1 + WHERE id=$id + """; + claim.Parameters.AddWithValue("$id", id); + claim.Parameters.AddWithValue("$worker", workerId); + claim.Parameters.AddWithValue("$lease", Stamp(now + NestJobRules.Lease)); + claim.Parameters.AddWithValue("$now", Stamp(now)); + claim.ExecuteNonQuery(); + transaction.Commit(); + return GetJob(Guid.Parse(id)); + } + } + + /// Renews the lease. Null when no longer holds the job. + public NestJobHeartbeat? HeartbeatJob(Guid id, string workerId, DateTimeOffset now) + { + lock (_sync) + { + using var command = _connection.CreateCommand(); + command.CommandText = """ + UPDATE nest_jobs SET leaseUntil=$lease + WHERE id=$id AND state='Running' AND workerId=$worker + RETURNING cancelRequested + """; + command.Parameters.AddWithValue("$id", id.ToString()); + command.Parameters.AddWithValue("$worker", workerId); + command.Parameters.AddWithValue("$lease", Stamp(now + NestJobRules.Lease)); + return command.ExecuteScalar() is long cancel ? new NestJobHeartbeat { CancelRequested = cancel != 0 } : null; + } + } + + /// + /// Stores the worker's result. A job someone cancelled meanwhile ends Cancelled and the + /// result is dropped. False when no longer holds the job. + /// + public bool CompleteJob(Guid id, string workerId, string result, DateTimeOffset now) + { + ArgumentNullException.ThrowIfNull(result); + return FinishJob(id, workerId, now, """ + state = CASE cancelRequested WHEN 1 THEN 'Cancelled' ELSE 'Succeeded' END, + result = CASE cancelRequested WHEN 1 THEN NULL ELSE $result END + """, ("$result", result)); + } + + public bool FailJob(Guid id, string workerId, string error, DateTimeOffset now) => + FinishJob(id, workerId, now, "state='Failed', error=$error", ("$error", Truncate(error))); + + /// The worker stopped the job after a cancel request. + public bool ConfirmJobCancelled(Guid id, string workerId, DateTimeOffset now) => + FinishJob(id, workerId, now, "state='Cancelled'"); + + /// + /// Puts a running job back at the front of the queue without spending an attempt: its + /// worker is shutting down, not failing. A job someone cancelled ends Cancelled instead. + /// False when no longer holds the job. + /// + public bool ReleaseJob(Guid id, string workerId, DateTimeOffset now) + { + lock (_sync) + { + using var command = _connection.CreateCommand(); + command.CommandText = """ + UPDATE nest_jobs SET + state = CASE cancelRequested WHEN 1 THEN 'Cancelled' ELSE 'Queued' END, + finishedAt = CASE cancelRequested WHEN 1 THEN $now ELSE NULL END, + package = CASE cancelRequested WHEN 1 THEN zeroblob(0) ELSE package END, + startedAt = CASE cancelRequested WHEN 1 THEN startedAt ELSE NULL END, + attempts = CASE cancelRequested WHEN 1 THEN attempts ELSE MAX(attempts - 1, 0) END, + workerId = NULL, leaseUntil = NULL + WHERE id=$id AND state='Running' AND workerId=$worker + """; + command.Parameters.AddWithValue("$id", id.ToString()); + command.Parameters.AddWithValue("$worker", workerId); + command.Parameters.AddWithValue("$now", Stamp(now)); + return command.ExecuteNonQuery() == 1; + } + } + + private bool FinishJob(Guid id, string workerId, DateTimeOffset now, string set, params (string Name, object Value)[] values) + { + lock (_sync) + { + using var command = _connection.CreateCommand(); + command.CommandText = $""" + UPDATE nest_jobs SET {set}, finishedAt=$now, workerId=NULL, leaseUntil=NULL, package=zeroblob(0) + WHERE id=$id AND state='Running' AND workerId=$worker + """; + command.Parameters.AddWithValue("$id", id.ToString()); + command.Parameters.AddWithValue("$worker", workerId); + command.Parameters.AddWithValue("$now", Stamp(now)); + foreach (var (name, value) in values) + command.Parameters.AddWithValue(name, value); + return command.ExecuteNonQuery() == 1; + } + } + + // A lapsed lease means the worker stopped: a cancel request ends the job, otherwise it goes + // back to the queue until it has used its attempts. + private void SettleJobLeases(SqliteTransaction transaction, DateTimeOffset now) + { + using var command = _connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + UPDATE nest_jobs SET + state = CASE WHEN cancelRequested=1 THEN 'Cancelled' + WHEN attempts >= $max THEN 'Failed' ELSE 'Queued' END, + error = CASE WHEN cancelRequested=0 AND attempts >= $max + THEN 'The nest worker stopped responding ' || attempts || ' times while running this job.' + ELSE error END, + finishedAt = CASE WHEN cancelRequested=1 OR attempts >= $max THEN $now ELSE NULL END, + package = CASE WHEN cancelRequested=1 OR attempts >= $max THEN zeroblob(0) ELSE package END, + startedAt = CASE WHEN cancelRequested=1 OR attempts >= $max THEN startedAt ELSE NULL END, + workerId = NULL, leaseUntil = NULL + WHERE state='Running' AND leaseUntil < $now; + DELETE FROM nest_jobs WHERE finishedAt IS NOT NULL AND finishedAt < $expired; + """; + command.Parameters.AddWithValue("$max", NestJobRules.MaxAttempts); + command.Parameters.AddWithValue("$now", Stamp(now)); + command.Parameters.AddWithValue("$expired", Stamp(now - NestJobRules.Retention)); + command.ExecuteNonQuery(); + } + + private static NestJobRecord ReadJob(SqliteDataReader reader) => new() + { + Id = Guid.Parse(reader.GetString(0)), + Title = reader.GetString(1), + EngineName = reader.GetString(2), + SubmittedBy = reader.GetString(3), + State = Enum.Parse(reader.GetString(4)), + CancelRequested = reader.GetInt64(5) != 0, + SubmittedAt = ParseStamp(reader.GetString(6)), + StartedAt = reader.IsDBNull(7) ? null : ParseStamp(reader.GetString(7)), + FinishedAt = reader.IsDBNull(8) ? null : ParseStamp(reader.GetString(8)), + Attempts = reader.GetInt32(9), + Error = reader.GetString(10), + PackageSize = reader.GetInt64(11), + }; + + private static string Truncate(string? text) + { + text ??= ""; + return text.Length <= NestJobRules.MaxTextLength ? text : text[..NestJobRules.MaxTextLength]; + } + + private static string Stamp(DateTimeOffset time) => time.UtcDateTime.ToString("o", CultureInfo.InvariantCulture); + + private static DateTime ParseStamp(string text) => + DateTime.Parse(text, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal); +} diff --git a/OpenNest.Server/NestDatabase.cs b/OpenNest.Server/NestDatabase.cs index 914130b2..81a1c542 100644 --- a/OpenNest.Server/NestDatabase.cs +++ b/OpenNest.Server/NestDatabase.cs @@ -344,7 +344,7 @@ public sealed partial class NestDatabase : IDisposable command.Transaction = transaction; command.CommandText = "PRAGMA user_version"; var version = (long)command.ExecuteScalar()!; - if (version > 14) throw new InvalidOperationException("Database schema is newer than this server."); + if (version > 15) throw new InvalidOperationException("Database schema is newer than this server."); if (version == 0) { command.CommandText = """ @@ -439,6 +439,12 @@ public sealed partial class NestDatabase : IDisposable } if (version < 14) AddStockCodes(command); + if (version < 15) + { + // The server nesting queue; see NestDatabase.Jobs. + command.CommandText = NestJobSchema; + command.ExecuteNonQuery(); + } transaction.Commit(); } } diff --git a/OpenNest.Server/NestJobRoutes.cs b/OpenNest.Server/NestJobRoutes.cs new file mode 100644 index 00000000..1c533740 --- /dev/null +++ b/OpenNest.Server/NestJobRoutes.cs @@ -0,0 +1,158 @@ +using System.Text; +using System.Text.Json; +using Microsoft.AspNetCore.Http.Features; +using OpenNest.Data; + +namespace OpenNest.Server; + +/// +/// /api/nest-jobs: the server nesting queue. Desktops submit a package, poll its state, +/// read the result and cancel; the worker (OpenNest.Server.Worker) claims a job, reads +/// its package, heartbeats and reports. The server never opens a package or result: the worker +/// solves the job and the desktop validates the result against its own drawings. +/// See docs/server-nesting.md. +/// +internal static class NestJobRoutes +{ + private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web); + + public static void MapNestJobs(this IEndpointRouteBuilder app) + { + app.MapPost("/api/nest-jobs", async (HttpContext http, NestDatabase db, TimeProvider clock) => + { + if (http.Features.Get() is { IsReadOnly: false } limit) + limit.MaxRequestBodySize = NestJobRules.MaxPackageBytes + 1024 * 1024; + if (!http.Request.HasFormContentType) + return Results.BadRequest("Expected multipart/form-data with metadata and file parts."); + + IFormCollection form; + try + { + form = await http.Request.ReadFormAsync(new FormOptions { MultipartBodyLengthLimit = NestJobRules.MaxPackageBytes + 1024 * 1024 }); + } + catch (Exception ex) when (ex is InvalidDataException || (ex is IOException && ex is not BadHttpRequestException)) + { + return Results.BadRequest("Malformed multipart body."); + } + + NestJobSubmission? submission; + try { submission = JsonSerializer.Deserialize(form["metadata"].FirstOrDefault() ?? "", Options); } + catch (JsonException) { return Results.BadRequest("Invalid metadata JSON."); } + if (submission is null || string.IsNullOrWhiteSpace(submission.EngineName)) + return Results.BadRequest("The job must name an engine."); + if (new[] { submission.Title, submission.EngineName, submission.SubmittedBy }.Any(t => (t?.Length ?? 0) > NestJobRules.MaxTextLength)) + return Results.BadRequest("A job title, engine or submitter is too long."); + submission.Title ??= ""; + submission.SubmittedBy ??= ""; + + var file = form.Files["file"]; + if (file is null || file.Length == 0) + return Results.BadRequest("Missing or empty 'file' part."); + if (file.Length > NestJobRules.MaxPackageBytes) + return Results.BadRequest("The nest job package is too large."); + using var stream = file.OpenReadStream(); + using var buffer = new MemoryStream((int)file.Length); + await stream.CopyToAsync(buffer); + return Results.Ok(db.InsertJob(Guid.NewGuid(), submission, buffer.ToArray(), clock.GetUtcNow())); + }); + + app.MapGet("/api/nest-jobs/{id:guid}", (Guid id, NestDatabase db) => + db.GetJob(id) is { } job ? Results.Ok(job) : Results.NotFound()); + + app.MapGet("/api/nest-jobs/{id:guid}/result", (Guid id, NestDatabase db) => + { + if (db.GetJobResult(id) is { } result) + return Results.Text(result, "application/json", Encoding.UTF8); + return db.GetJob(id) is { } job + ? Results.Conflict($"The job has no result: it is {job.State}.") + : Results.NotFound(); + }); + + app.MapPost("/api/nest-jobs/{id:guid}/cancel", (Guid id, NestDatabase db, TimeProvider clock) => + db.CancelJob(id, clock.GetUtcNow()) is { } job ? Results.Ok(job) : Results.NotFound()); + + // Worker side. + app.MapPost("/api/nest-jobs/claim", async (HttpRequest request, NestDatabase db, TimeProvider clock) => + { + var (message, error) = await ReadWorkerMessage(request); + if (error is not null) return error; + return db.ClaimJob(message!.WorkerId, clock.GetUtcNow()) is { } job ? Results.Ok(job) : Results.NoContent(); + }); + + app.MapGet("/api/nest-jobs/{id:guid}/package", (Guid id, NestDatabase db) => + db.GetJobPackage(id) is { } package + ? Results.File(package, "application/zip", $"{id}.zip") + : Results.NotFound()); + + app.MapPost("/api/nest-jobs/{id:guid}/heartbeat", async (Guid id, HttpRequest request, NestDatabase db, TimeProvider clock) => + { + var (message, error) = await ReadWorkerMessage(request); + if (error is not null) return error; + return db.HeartbeatJob(id, message!.WorkerId, clock.GetUtcNow()) is { } beat + ? Results.Ok(beat) + : Results.Conflict("This worker no longer holds the job."); + }); + + app.MapPut("/api/nest-jobs/{id:guid}/result", async (Guid id, string? workerId, HttpContext http, NestDatabase db, TimeProvider clock) => + { + if (string.IsNullOrWhiteSpace(workerId)) + return Results.BadRequest("workerId is required."); + if (http.Features.Get() is { IsReadOnly: false } limit) + limit.MaxRequestBodySize = NestJobRules.MaxResultBytes + 1; + if (!http.Request.HasJsonContentType()) + return Results.BadRequest("Expected an application/json result."); + using var body = new MemoryStream(); + var chunk = new byte[81920]; + int read; + while ((read = await http.Request.Body.ReadAsync(chunk, http.RequestAborted)) > 0) + { + if (body.Length + read > NestJobRules.MaxResultBytes) + return Results.BadRequest("The nest result is too large."); + body.Write(chunk, 0, read); + } + var result = Encoding.UTF8.GetString(body.GetBuffer(), 0, (int)body.Length); + return db.CompleteJob(id, workerId, result, clock.GetUtcNow()) + ? Results.Ok(db.GetJob(id)) + : Results.Conflict("This worker no longer holds the job."); + }); + + app.MapPost("/api/nest-jobs/{id:guid}/fail", async (Guid id, HttpRequest request, NestDatabase db, TimeProvider clock) => + { + var (message, error) = await ReadWorkerMessage(request); + if (error is not null) return error; + return db.FailJob(id, message!.WorkerId, message.Error, clock.GetUtcNow()) + ? Results.Ok(db.GetJob(id)) + : Results.Conflict("This worker no longer holds the job."); + }); + + app.MapPost("/api/nest-jobs/{id:guid}/release", async (Guid id, HttpRequest request, NestDatabase db, TimeProvider clock) => + { + var (message, error) = await ReadWorkerMessage(request); + if (error is not null) return error; + return db.ReleaseJob(id, message!.WorkerId, clock.GetUtcNow()) + ? Results.Ok(db.GetJob(id)) + : Results.Conflict("This worker no longer holds the job."); + }); + + app.MapPost("/api/nest-jobs/{id:guid}/cancelled", async (Guid id, HttpRequest request, NestDatabase db, TimeProvider clock) => + { + var (message, error) = await ReadWorkerMessage(request); + if (error is not null) return error; + return db.ConfirmJobCancelled(id, message!.WorkerId, clock.GetUtcNow()) + ? Results.Ok(db.GetJob(id)) + : Results.Conflict("This worker no longer holds the job."); + }); + } + + private static async Task<(NestJobWorkerMessage? Message, IResult? Error)> ReadWorkerMessage(HttpRequest request) + { + var (json, error) = await DrawingSetRoutes.ReadJson(request, "worker message"); + if (error is not null) return (null, error); + NestJobWorkerMessage? message; + try { message = JsonSerializer.Deserialize(json!, Options); } + catch (JsonException) { return (null, Results.BadRequest("Invalid worker message JSON.")); } + if (message is null || string.IsNullOrWhiteSpace(message.WorkerId) || message.WorkerId.Length > NestJobRules.MaxTextLength) + return (null, Results.BadRequest("workerId is required.")); + return (message, null); + } +} diff --git a/OpenNest.Server/Program.cs b/OpenNest.Server/Program.cs index 69493539..f8251163 100644 --- a/OpenNest.Server/Program.cs +++ b/OpenNest.Server/Program.cs @@ -23,6 +23,8 @@ builder.Services.ConfigureHttpJsonOptions(options => // Storage path: --database= or --database , then OPENNEST_DB, default ./data/nests.db (a Docker volume). var databasePath = ServerOptions.DatabasePath(args, Environment.GetEnvironmentVariable(ServerOptions.DatabaseVariable)); builder.Services.AddSingleton(_ => new NestDatabase(databasePath)); +// The server nesting queue's leases read this clock; tests replace it. +builder.Services.AddSingleton(TimeProvider.System); // Desktop update feed mirrored from GitHub beside the database; OPENNEST_UPDATES=off turns it off. var updatesEnabled = ServerOptions.UpdatesEnabled(Environment.GetEnvironmentVariable(ServerOptions.UpdatesVariable)); @@ -167,6 +169,7 @@ app.MapDrawingSets(); app.MapReleases(); app.MapMaterials(); app.MapSettings(); +app.MapNestJobs(); if (updatesEnabled) app.MapUpdates(); diff --git a/OpenNest.Server/entrypoint.sh b/OpenNest.Server/entrypoint.sh new file mode 100644 index 00000000..f217c80e --- /dev/null +++ b/OpenNest.Server/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# OPENNEST_ROLE picks what this container runs: the nest server (default) or the nesting worker. +set -e +case "${OPENNEST_ROLE:-server}" in + server) exec dotnet /app/OpenNest.Server.dll "$@" ;; + worker) exec dotnet /app/worker/OpenNest.Server.Worker.dll "$@" ;; + *) echo "OPENNEST_ROLE must be server or worker, not '$OPENNEST_ROLE'." >&2; exit 64 ;; +esac diff --git a/OpenNest.Server/healthcheck.sh b/OpenNest.Server/healthcheck.sh new file mode 100644 index 00000000..1baffad1 --- /dev/null +++ b/OpenNest.Server/healthcheck.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Server: /healthz answers. Worker: its alive file was touched within the last two minutes. +if [ "${OPENNEST_ROLE:-server}" = worker ]; then + alive="${OPENNEST_WORKER_ALIVE_FILE:-/tmp/opennest-worker.alive}" + [ -n "$(find "$alive" -mmin -2 2>/dev/null)" ] +else + curl --fail --silent --show-error http://127.0.0.1:8090/healthz > /dev/null +fi diff --git a/OpenNest.Server/server.env.example b/OpenNest.Server/server.env.example index e68fc3ae..8c3ab995 100644 --- a/OpenNest.Server/server.env.example +++ b/OpenNest.Server/server.env.example @@ -15,3 +15,8 @@ OPENNEST_DATA_VOLUME=opennest-data # Desktop update mirror: the server checks GitHub Releases (outbound HTTPS to api.github.com # and github.com) and serves updates to Database-mode workstations. "off" turns it off. OPENNEST_UPDATES=on + +# Server nesting worker limits: CPUs (fractions allowed) and memory it may use. A smaller +# limit makes jobs slower, never worse; leave room for the server and the rest of the host. +OPENNEST_WORKER_CPUS=2 +OPENNEST_WORKER_MEMORY=4g diff --git a/OpenNest.sln b/OpenNest.sln index aa7ab17b..2cfb16c8 100644 --- a/OpenNest.sln +++ b/OpenNest.sln @@ -58,6 +58,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenNest.Desktop.Tests", "O EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenNest.Avalonia", "OpenNest.Avalonia\OpenNest.Avalonia.csproj", "{BE640107-4749-4C43-AC3A-0F2349422E16}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenNest.Server.Worker", "OpenNest.Server.Worker\OpenNest.Server.Worker.csproj", "{DFE6F33C-4BB7-4F6A-8062-6D5521423B74}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -368,6 +370,18 @@ Global {BE640107-4749-4C43-AC3A-0F2349422E16}.Release|x64.Build.0 = Release|Any CPU {BE640107-4749-4C43-AC3A-0F2349422E16}.Release|x86.ActiveCfg = Release|Any CPU {BE640107-4749-4C43-AC3A-0F2349422E16}.Release|x86.Build.0 = Release|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Debug|x64.ActiveCfg = Debug|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Debug|x64.Build.0 = Debug|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Debug|x86.ActiveCfg = Debug|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Debug|x86.Build.0 = Debug|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Release|Any CPU.Build.0 = Release|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Release|x64.ActiveCfg = Release|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Release|x64.Build.0 = Release|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Release|x86.ActiveCfg = Release|Any CPU + {DFE6F33C-4BB7-4F6A-8062-6D5521423B74}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/OpenNest/Forms/AutoNestForm.Designer.cs b/OpenNest/Forms/AutoNestForm.Designer.cs index 6fa8d6df..6eb102b0 100644 --- a/OpenNest/Forms/AutoNestForm.Designer.cs +++ b/OpenNest/Forms/AutoNestForm.Designer.cs @@ -24,6 +24,7 @@ namespace OpenNest.Forms this.summaryLabel = new System.Windows.Forms.Label(); this.engineLabel = new System.Windows.Forms.Label(); this.engineComboBox = new System.Windows.Forms.ComboBox(); + this.nestOnServerBox = new System.Windows.Forms.CheckBox(); this.minRemnantLabel = new System.Windows.Forms.Label(); this.minRemnantBox = new System.Windows.Forms.TextBox(); this.plateOptimizerGroup = new System.Windows.Forms.GroupBox(); @@ -76,6 +77,7 @@ namespace OpenNest.Forms this.platesTab.Controls.Add(this.engineLabel); this.platesTab.Controls.Add(this.engineComboBox); this.platesTab.Controls.Add(this.plateOptimizerGroup); + this.platesTab.Controls.Add(this.nestOnServerBox); this.platesTab.Location = new System.Drawing.Point(4, 25); this.platesTab.Name = "platesTab"; this.platesTab.Padding = new System.Windows.Forms.Padding(6); @@ -121,6 +123,18 @@ namespace OpenNest.Forms this.engineComboBox.Name = "engineComboBox"; this.engineComboBox.Size = new System.Drawing.Size(200, 24); this.engineComboBox.TabIndex = 1; + this.engineComboBox.SelectedIndexChanged += new System.EventHandler(this.engineComboBox_SelectedIndexChanged); + // + // nestOnServerBox + // + this.nestOnServerBox.AutoSize = true; + this.nestOnServerBox.Location = new System.Drawing.Point(10, 396); + this.nestOnServerBox.Name = "nestOnServerBox"; + this.nestOnServerBox.Size = new System.Drawing.Size(330, 20); + this.nestOnServerBox.TabIndex = 3; + this.nestOnServerBox.Text = "Nest on the server (keeps this PC free; may be slower)"; + this.nestOnServerBox.UseVisualStyleBackColor = true; + this.nestOnServerBox.Visible = false; // // minRemnantLabel // @@ -295,6 +309,7 @@ namespace OpenNest.Forms private System.Windows.Forms.Label summaryLabel; private System.Windows.Forms.Label engineLabel; private System.Windows.Forms.ComboBox engineComboBox; + private System.Windows.Forms.CheckBox nestOnServerBox; private System.Windows.Forms.Label minRemnantLabel; private System.Windows.Forms.TextBox minRemnantBox; private System.Windows.Forms.GroupBox plateOptimizerGroup; diff --git a/OpenNest/Forms/AutoNestForm.cs b/OpenNest/Forms/AutoNestForm.cs index fcb28c5d..71376bc9 100644 --- a/OpenNest/Forms/AutoNestForm.cs +++ b/OpenNest/Forms/AutoNestForm.cs @@ -9,6 +9,7 @@ using System.Windows.Forms; using OpenNest.Controls; using OpenNest.Data.MaterialLedger; using OpenNest.Engine; +using OpenNest.Engine.Jobs; using OpenNest.Geometry; using OpenNest.Math; @@ -54,6 +55,35 @@ namespace OpenNest.Forms set { engineComboBox.SelectedItem = value; } } + /// + /// Offers "Nest on the server" (Database mode only). The box starts at the last choice and + /// is enabled only for built-in engines, since the server loads no plug-ins. + /// + public bool ServerNestingAvailable + { + get => nestOnServerBox.Visible; + set + { + nestOnServerBox.Visible = value; + nestOnServerBox.Checked = value && Properties.Settings.Default.AutoNestOnServer; + UpdateNestOnServerBox(); + } + } + + /// True when the user chose to nest on the server with an engine it can run. + public bool NestOnServer => nestOnServerBox.Visible && nestOnServerBox.Enabled && nestOnServerBox.Checked; + + private void engineComboBox_SelectedIndexChanged(object sender, EventArgs e) => UpdateNestOnServerBox(); + + private void UpdateNestOnServerBox() + { + var builtIn = NestingEngineRegistry.IsBuiltIn(EngineName); + nestOnServerBox.Enabled = builtIn; + ledgerToolTip.SetToolTip(nestOnServerBox, builtIn + ? "Send this nest to the nest server's worker. This PC stays free while it runs; Keep and Discard work as usual." + : "Plug-in engines run only on this PC."); + } + public bool OptimizePlateSize { get { return optimizePlateSizeBox.Checked; } @@ -447,6 +477,11 @@ namespace OpenNest.Forms DialogResult = DialogResult.None; ReportStockValidationFailure(error); } + else if (DialogResult == DialogResult.OK && nestOnServerBox.Visible && nestOnServerBox.Enabled) + { + Properties.Settings.Default.AutoNestOnServer = nestOnServerBox.Checked; + Properties.Settings.Default.Save(); + } base.OnFormClosing(e); } diff --git a/OpenNest/Forms/Commands/AutoNestCommand.cs b/OpenNest/Forms/Commands/AutoNestCommand.cs index b28672f2..7f17c016 100644 --- a/OpenNest/Forms/Commands/AutoNestCommand.cs +++ b/OpenNest/Forms/Commands/AutoNestCommand.cs @@ -31,6 +31,7 @@ namespace OpenNest.Forms.Commands private readonly double minRemnantSize; private List savedOptions; private IReadOnlyList ledgerRows = Array.Empty(); + private bool nestOnServer; private AutoNestCommand(IWin32Window owner, EditNestForm target, List items, List plateOptions, string engineName, double salvageRate, double minRemnantSize) @@ -53,7 +54,11 @@ namespace OpenNest.Forms.Commands /// public static AutoNestCommand Prompt(IWin32Window owner, EditNestForm target) { - using var form = new AutoNestForm(target.Nest) { ServerNestId = target.Document.RemoteId }; + using var form = new AutoNestForm(target.Nest) + { + ServerNestId = target.Document.RemoteId, + ServerNestingAvailable = NestStorage.Settings.IsDatabaseMode, + }; if (target.Nest.PlateOptions.Count > 0) form.LoadPlateOptions(target.Nest.PlateOptions, target.Nest.SalvageRate); if (target.Nest.MinimumRemnantDimension > 0) @@ -78,6 +83,7 @@ namespace OpenNest.Forms.Commands { savedOptions = savedOptions, ledgerRows = form.OptimizePlateSize ? form.LedgerRows : Array.Empty(), + nestOnServer = form.NestOnServer, }; } @@ -94,7 +100,9 @@ namespace OpenNest.Forms.Commands new NestJobOptions(maxPlates: 100, salvageRate: plateOptions?.Count > 0 ? salvageRate : 0, minimumSalvageDimension: minRemnantSize)); - var result = await SolveAsync(owner, target, request, cts); + var result = nestOnServer + ? await ServerAutoNest.SolveAsync(owner, target, request, cts) + : await SolveAsync(owner, target, request, cts); cts.Token.ThrowIfCancellationRequested(); var allowInvalid = false; diff --git a/OpenNest/Forms/Commands/ServerAutoNest.cs b/OpenNest/Forms/Commands/ServerAutoNest.cs new file mode 100644 index 00000000..7c439bf6 --- /dev/null +++ b/OpenNest/Forms/Commands/ServerAutoNest.cs @@ -0,0 +1,142 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using OpenNest.Api; +using OpenNest.Data; +using OpenNest.Engine; +using OpenNest.Engine.Jobs; +using OpenNest.Engine.Jobs.Remote; + +namespace OpenNest.Forms.Commands +{ + /// + /// Auto Nest solved by the nest server's worker (Database mode): the job is sent as a + /// package, its state polled behind the same owned modal progress a local run uses, and the + /// server's engine answers replayed here (), so validation, + /// the review and Keep are exactly a local run's. Stop cancels the server job and discards + /// the proposal. When the server cannot take the job, the user may nest on this PC instead. + /// See docs/server-nesting.md. + /// + internal static class ServerAutoNest + { + private const string Caption = "Auto Nest"; + + private static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(2); + + internal static async Task SolveAsync(IWin32Window owner, EditNestForm target, + NestPipelineRequest request, CancellationTokenSource cts) + { + var submission = new RemoteNestSubmission(request); + using var jobs = new RemoteNestJobs(NestStorage.Settings.ServerUrl); + NestJobRecord job; + try + { + var package = await Task.Run(() => NestJobPackage.Write(request, target.Nest.Units), cts.Token); + job = await jobs.SubmitAsync(new NestJobSubmission + { + Title = target.Nest.Name ?? "", + EngineName = request.EngineName, + SubmittedBy = $"{Environment.UserName}@{Environment.MachineName}", + }, package, cts.Token); + } + catch (Exception ex) when (!cts.IsCancellationRequested + && ex is IOException or HttpRequestException or TaskCanceledException) + { + var choice = MessageBox.Show(owner, + $"The nest server could not take this nest:\n\n{ex.Message}\n\nNest on this PC instead?", + Caption, MessageBoxButtons.YesNo, MessageBoxIcon.Warning); + if (choice == DialogResult.Yes) + return await AutoNestCommand.SolveAsync(owner, target, request, cts); + cts.Cancel(); + throw new OperationCanceledException(cts.Token); + } + + using var progressForm = new NestProgressForm(cts, showPlateRow: false) + { + AllowAccept = false, + HoldOpenUntilCompleted = true, + }; + var wait = WaitAsync(jobs, job, submission, progressForm, cts); + if (!wait.IsCompleted) + progressForm.ShowDialog(owner); + return await wait; + } + + private static async Task WaitAsync(RemoteNestJobs jobs, NestJobRecord job, + RemoteNestSubmission submission, NestProgressForm progressForm, CancellationTokenSource cts) + { + string problem = null; + try + { + while (true) + { + cts.Token.ThrowIfCancellationRequested(); + Show(progressForm, problem ?? Describe(job)); + switch (job.State) + { + case NestJobState.Succeeded: + Show(progressForm, "Checking the server's nest on this PC"); + var transcript = NestSolveTranscript.FromJson(await jobs.GetResultAsync(job.Id, cts.Token)); + return await Task.Run(() => submission.Complete(transcript, cts.Token), cts.Token); + case NestJobState.Failed: + throw new InvalidOperationException($"The nest server could not nest this job: {job.Error}"); + case NestJobState.Cancelled: + // Stopped from elsewhere: nothing to keep, as after Stop here. + cts.Cancel(); + cts.Token.ThrowIfCancellationRequested(); + break; + } + + await Task.Delay(PollInterval, cts.Token); + try + { + job = await jobs.GetAsync(job.Id, cts.Token) + ?? throw new InvalidOperationException("The nest server no longer has this job."); + problem = null; + } + catch (Exception ex) when (!cts.IsCancellationRequested + && ex is IOException or HttpRequestException or TaskCanceledException) + { + // The job keeps running on the server; keep asking. + problem = $"Waiting for the nest server: {ex.Message}"; + } + } + } + catch (OperationCanceledException) when (cts.IsCancellationRequested) + { + // Best effort: a job left running finishes on the server and is never kept. + try + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await jobs.CancelAsync(job.Id, timeout.Token); + } + catch (Exception ex) when (ex is IOException or HttpRequestException or OperationCanceledException) { } + throw; + } + finally + { + progressForm.ShowCompleted(); + progressForm.Close(); + } + } + + private static string Describe(NestJobRecord job) => job.State switch + { + NestJobState.Queued when job.QueuePosition > 0 => + $"Queued on the nest server, {job.QueuePosition} {(job.QueuePosition == 1 ? "job" : "jobs")} ahead", + NestJobState.Queued => "Queued on the nest server", + NestJobState.Running when job.CancelRequested => "Stopping on the nest server", + NestJobState.Running => "Nesting on the nest server", + _ => job.State.ToString(), + }; + + private static void Show(NestProgressForm form, string text) + { + if (!form.IsDisposed) + form.UpdateProgress(new NestProgress { Description = text }); + } + } +} diff --git a/OpenNest/Properties/Settings.Designer.cs b/OpenNest/Properties/Settings.Designer.cs index 6b686d17..384c732b 100644 --- a/OpenNest/Properties/Settings.Designer.cs +++ b/OpenNest/Properties/Settings.Designer.cs @@ -463,5 +463,18 @@ namespace OpenNest.Properties { this["DrawingSetsWindowState"] = value; } } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("False")] + // Auto Nest sends the job to the nest server's worker (Database mode); the dialog remembers the last choice. + public bool AutoNestOnServer { + get { + return ((bool)(this["AutoNestOnServer"])); + } + set { + this["AutoNestOnServer"] = value; + } + } } } diff --git a/OpenNest/Properties/Settings.settings b/OpenNest/Properties/Settings.settings index 2ff8069f..4bfd58e8 100644 --- a/OpenNest/Properties/Settings.settings +++ b/OpenNest/Properties/Settings.settings @@ -110,5 +110,8 @@ Maximized + + False + diff --git a/compose.server.yaml b/compose.server.yaml index 4d8be859..bcf71499 100644 --- a/compose.server.yaml +++ b/compose.server.yaml @@ -19,6 +19,28 @@ services: - no-new-privileges:true cap_drop: - ALL + + # Server nesting (docs/server-nesting.md): the same image solving queued Auto Nest jobs one + # at a time. It talks to the server over HTTP and never opens the database, and its own CPU + # and memory limits keep a large nest from slowing nest open and save. A job has no time + # limit, so fewer CPUs only make it slower, never a worse nest. + opennest-worker: + image: ${OPENNEST_SERVER_IMAGE:?Set a tested version or digest} + restart: unless-stopped + depends_on: + - opennest-server + environment: + OPENNEST_ROLE: worker + OPENNEST_SERVER_URL: http://opennest-server:8090/ + cpus: ${OPENNEST_WORKER_CPUS:-2} + mem_limit: ${OPENNEST_WORKER_MEMORY:-4g} + # Time to give a running job back to the queue on stop. + stop_grace_period: 30s + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + volumes: opennest-data: # A restore switches to a separately verified volume instead of overwriting this one. diff --git a/docs/automatic-nesting.md b/docs/automatic-nesting.md index 6aa7e133..37b55d82 100644 --- a/docs/automatic-nesting.md +++ b/docs/automatic-nesting.md @@ -51,6 +51,9 @@ Unplaced demand is never dropped silently. `NestPipelineResult.Unplaced` lists e An incomplete but valid proposal is keepable like any other; keeping it is still an explicit action. Desktop Auto Nest lists each unplaced drawing, count and reason in its review before committing (the drawing list keeps showing remaining quantities); Console `--autonest` writes `Unplaced:` lines to standard error; MCP `autonest_plate` adds `Unplaced:` lines and `autonest_job` names the reasons when it refuses an incomplete proposal; the API returns `NestResponse.Unplaced` by request part ID. This check does not replace pre-post CNC verification. +In Database mode, **Nest on the server** sends the job to the nest server's worker instead; +the review and Keep are unchanged. See [server nesting](server-nesting.md). + The selected engine is saved in `%APPDATA%\OpenNest\engine-selection.json`. Selection is restored after plug-ins load. A saved name from an earlier release maps to the engine that replaced it (see [nesting engines](nesting-engines.md)). diff --git a/docs/nest-storage.md b/docs/nest-storage.md index 704d418c..a44100df 100644 --- a/docs/nest-storage.md +++ b/docs/nest-storage.md @@ -1133,7 +1133,10 @@ project files before restore and their source/resources before publish, includin Data's embedded `Defaults/Generic-Laser.json`. The allowlist `.dockerignore` excludes other projects, host `bin`/`obj`, local databases, environment files, and launch settings. The server image uses the .NET 8 SDK to build and the .NET 8 ASP.NET -runtime to run; IO/Engine are used only by the host-side smoke tool, not the image. +runtime to run. The HTTP server references neither IO nor Engine; the image also +carries the server nesting worker (`OpenNest.Server.Worker`, under `/app/worker`), which +references Api, Engine and IO, so the Dockerfile and `.dockerignore` include those +projects too. `OPENNEST_ROLE` picks the process; see [server nesting](server-nesting.md). ```sh docker build --pull -f OpenNest.Server/Dockerfile -t opennest-server:local . diff --git a/docs/server-nesting.md b/docs/server-nesting.md new file mode 100644 index 00000000..a540cbd2 --- /dev/null +++ b/docs/server-nesting.md @@ -0,0 +1,126 @@ +# Server nesting + +In Database mode, Auto Nest can send the job to the nest server instead of nesting on the +workstation. A worker container solves queued jobs one at a time under its own CPU and memory +limits, and the workstation reviews and keeps the result exactly as it would a local run. +Local nesting stays the default and the fallback when the server cannot take a job. + +## Desktop + +The Auto Nest dialog's Plates tab shows **Nest on the server** in Database mode only. It is +enabled for built-in engines only: the server loads no plug-ins from `Engines/`. The dialog +remembers the last choice (`AutoNestOnServer` in the user settings). + +With the box ticked, `ServerAutoNest`: + +1. Snapshots the request (`RemoteNestSubmission`) and writes its package (`NestJobPackage`). +2. Submits it. If the server cannot take it (offline, too old, refused), the user may nest on + this PC instead; declining discards the run. +3. Polls every two seconds behind the same owned modal progress window a local run uses, + showing the queue position or that the job is running. Transient poll failures only + show a waiting message; the job keeps running on the server. A live plan strip is not + shown yet. +4. On success, replays the result here (below) and continues with the usual validation + report, review, Keep or Discard and commit. A failed job shows its error. + +Stop (or closing the progress window) cancels the server job, best effort, and discards the +proposal, exactly like stopping a local run. The input drawings stay locked by the modal +progress for the whole wait. + +## What travels + +**The package** (`OpenNest.Api.NestJobPackage`) is a ZIP of: + +- `job.json`: `NestJobDocument`: engine name, each item's quantity, priority, rotation range + and drawing index, the drawing names in order (a guard against reordering), every stock + entry as `NestStockJson` (size, quantity, spacing, quadrant, cost, clamp zones in plate + coordinates, clamp rail, irregular outline), and the job options. A server job never sets + `SearchUntilCancelled` and never runs under a `DeadlineNestingEngine`: it has no time + limit, so a throttled worker finds the same nest a fast one does, later. +- `drawings.nest`: a throwaway nest of exactly the items' drawings, written by `NestWriter`. + The caller's nest is never written. Drawings are uploaded rather than referenced from the + drawing library or sets, because the nest being edited may hold unsaved edits and drawings + that are in neither; the server must nest what the workstation has. + +**The result** is a `NestSolveTranscript`: every engine solve the pipeline made on the worker, +in order, as what was asked (part and stock ids with quantities) and the raw `NestJobResult` +returned. Poses keep their IEEE values, nonfinite ones included. No bound plates or +validation travel back. + +## Replay on the desktop + +`RemoteNestSubmission.Complete` first checks that each item's drawing still has the geometry +it had when the job was sent; a changed drawing refuses the result. It then runs +`NestPipeline.Run` with a `ReplayNestingEngine` that answers each solve with the next recorded +result, after checking it is asked the same question. Cutout bundling, the ordinary-solve +comparison and clamp-removal recovery therefore happen here exactly as on the worker, and the +result is validated by `NestLayoutCheck` and bound to the caller's own drawings like a local +run. A recorded sheet that matches one of the request's stock entries uses that entry itself. + +A replay that asks a different question, or more or fewer solves than were recorded, fails +with `NestReplayMismatchException`. That happens only if the drawings read back on the worker +differ enough to change a geometric decision, which `.nest` coordinates (10 decimal places) +make unlikely. The user then runs Auto Nest again; nothing partial is kept. + +## Server + +`/api/nest-jobs` (`NestJobRoutes`, `NestDatabase.Jobs`, schema version 15) stores packages and +results as opaque bytes. The HTTP server never opens either and references neither the engine +nor `OpenNest.IO`. + +| Route | Who | Does | +|---|---|---| +| `POST /api/nest-jobs` | desktop | multipart `metadata` (`NestJobSubmission`) and `file` (package, at most 256 MB); returns the queued job | +| `GET /api/nest-jobs/{id}` | desktop | state, attempts, error, queue position | +| `GET /api/nest-jobs/{id}/result` | desktop | the transcript; 409 until the job succeeded | +| `POST /api/nest-jobs/{id}/cancel` | desktop | queued: cancelled at once; running: flagged for the worker | +| `POST /api/nest-jobs/claim` | worker | the oldest queued job on a fresh lease, or 204 | +| `GET /api/nest-jobs/{id}/package` | worker | the package while queued or running | +| `POST /api/nest-jobs/{id}/heartbeat` | worker | renews the lease and reports a cancel request; 409 once the worker no longer holds it | +| `PUT /api/nest-jobs/{id}/result?workerId=` | worker | stores the transcript (at most 64 MB); a job cancelled meanwhile ends Cancelled | +| `POST /api/nest-jobs/{id}/fail` | worker | ends the job Failed with a reason | +| `POST /api/nest-jobs/{id}/cancelled` | worker | confirms a requested stop | +| `POST /api/nest-jobs/{id}/release` | worker | gives the job back without spending an attempt (worker shutting down) | + +States: `Queued`, `Running`, then one of `Succeeded`, `Failed` and `Cancelled`. Only one job +runs at a time: a claim returns nothing while any job is running, so another worker container +never takes a second job. Claims are serialized by the database's monitor and transaction. + +**Crash recovery.** A running job holds a two-minute lease (`NestJobRules.Lease`), which the +worker renews every 20 seconds. Each claim first settles lapsed leases: a job someone asked to +cancel ends Cancelled; otherwise it goes back to the front of the queue, or fails after three +attempts (a job that keeps killing its worker, for example by running out of memory). A +server restart keeps every job; leases lapse and recover the same way. + +Finished jobs drop their package at once, and are deleted with their result seven days after +they finish. Like the rest of the API, these routes have no authentication; keep the server on +a trusted LAN. + +## Worker and container + +`OpenNest.Server.Worker` (Generic Host) polls `OPENNEST_SERVER_URL` every +`OPENNEST_WORKER_POLL_SECONDS` (default 5) and goes straight to the next job after one +finishes. It runs each job through `NestPipeline` with the named built-in engine wrapped in a +`RecordingNestingEngine`, at below-normal process priority. It talks to the server over HTTP +and never opens the SQLite file, so it may run anywhere that reaches the server, including a +host whose Docker data sits on NFS. On stop it gives a running job back (`release`), within the +compose file's 30-second grace period. + +The server image contains both processes. `OPENNEST_ROLE` picks one: `server` (default) or +`worker` (`/app/entrypoint.sh`). The image health check (`/app/healthcheck.sh`) asks `/healthz` +for the server; for the worker it requires the alive file +(`OPENNEST_WORKER_ALIVE_FILE`, default `/tmp/opennest-worker.alive`), touched every poll round +and every accepted heartbeat, to be under two minutes old. + +`compose.server.yaml` adds `opennest-worker` with `cpus: ${OPENNEST_WORKER_CPUS:-2}` and +`mem_limit: ${OPENNEST_WORKER_MEMORY:-4g}`. .NET reads the CPU quota for +`Environment.ProcessorCount`, so the engines' parallel loops scale down to it. Fewer CPUs only +make a job slower. Leave enough CPU and memory for the server itself. + +## Verification + +`OpenNest.Engine.Tests` `RemoteNestReplayTests`: replay equals the local proposal (including +the cutout trial plus ordinary solve), edited drawings and different jobs refuse, and stock +and nonfinite poses survive the wire. `OpenNest.Server.Tests` `NestJobQueueTests` (states, +single running job, concurrent claims, cancellation, lease lapse, retry limit, release, +retention, restart) and `NestJobApiTests` (HTTP plus a real worker run end to end). diff --git a/scripts/Deploy-Server.ps1 b/scripts/Deploy-Server.ps1 index a4d2e70c..a7e55580 100644 --- a/scripts/Deploy-Server.ps1 +++ b/scripts/Deploy-Server.ps1 @@ -65,7 +65,7 @@ $ShortRevision = $Revision.Substring(0, 8) $Branch = (& git -C $RepoRoot rev-parse --abbrev-ref HEAD).Trim() # Only the directories the Dockerfile copies affect the image. -$ServerSources = 'OpenNest.Server', 'OpenNest.Data', 'OpenNest.Core' +$ServerSources = 'OpenNest.Server', 'OpenNest.Server.Worker', 'OpenNest.Api', 'OpenNest.Engine', 'OpenNest.IO', 'OpenNest.Data', 'OpenNest.Core' $Dirty = & git -C $RepoRoot status --porcelain -- $ServerSources if ($Dirty) { Write-Warning "Uncommitted changes in server sources are included in this build:"