using System.Net; using System.Threading.Channels; using Microsoft.Extensions.Logging.Abstractions; using OpenNest.Api; using OpenNest.Data; using OpenNest.Engine; using OpenNest.Engine.Jobs; using OpenNest.Engine.Jobs.Remote; using OpenNest.Geometry; using OpenNest.Server.Worker; namespace OpenNest.Server.Tests; /// /// Live progress for server nesting: the worker's reports, each job's event stream (order, /// the final event, reconnecting with Last-Event-ID), the desktop's watch with its polling /// fallback, and how soon a worker sees a cancel request. /// public class NestJobEventsTests { private static readonly TimeSpan Wait = TimeSpan.FromSeconds(20); private static NestPipelineRequest Request() { var program = new OpenNest.CNC.Program(); program.MoveTo(0, 0); program.LineTo(10, 0); program.LineTo(10, 20); program.LineTo(0, 20); program.LineTo(0, 0); var template = new Plate(48, 96) { EdgeSpacing = new Spacing(0.25, 0.25, 0.25, 0.25), PartSpacing = 0.25 }; return new NestPipelineRequest("Rectangles", new[] { new NestItem { Drawing = new Drawing("bracket", program), Quantity = 9 } }, NestStockBuilder.FromTemplate(template, null), new NestJobOptions(maxPlates: 5)); } private static NestJobSubmission Describe() => new() { Title = "N26-0042", EngineName = "Rectangles", SubmittedBy = "aj@pc" }; private static RemoteNestJobs Client(ServerFactory factory, DelegatingHandler? handler = null) { var http = handler is null ? factory.CreateClient() : factory.CreateDefaultClient(handler); return new RemoteNestJobs(http, http.BaseAddress!.ToString()); } /// Reads a stream into a channel, so a test can act between events. private static (ChannelReader Events, Task Done) Listen(RemoteNestJobs jobs, Guid id, long lastEventId = 0) { var channel = Channel.CreateUnbounded(); var done = Task.Run(async () => { try { await foreach (var update in jobs.WatchAsync(id, lastEventId)) await channel.Writer.WriteAsync(update); channel.Writer.Complete(); } catch (Exception ex) { channel.Writer.Complete(ex); } }); return (channel.Reader, done); } private static async Task Next(ChannelReader events) { using var timeout = new CancellationTokenSource(Wait); return await events.ReadAsync(timeout.Token); } private static async Task NextJob(ChannelReader events) { var update = await Next(events); Assert.Equal(NestJobEventKind.Job, update.Kind); return update; } [Fact] public async Task TheStreamSendsEachChangeInOrderAndEndsAtTheFinalState() { using var factory = new ServerFactory(); var jobs = Client(factory); var job = await jobs.SubmitAsync(Describe(), new byte[] { 1 }); var (events, done) = Listen(jobs, job.Id); Assert.Equal(NestJobEventKind.Open, (await Next(events)).Kind); var queued = await NextJob(events); Assert.Equal(NestJobState.Queued, queued.Job!.State); await jobs.ClaimAsync("w1"); var running = await NextJob(events); Assert.Equal(NestJobState.Running, running.Job!.State); var progress = RemoteNestProgress.From(new NestJobProgress(NestJobStage.PlateCommitted, "s", 0, 1, 9)).ToJson(); Assert.False((await jobs.ReportProgressAsync(job.Id, "w1", progress))!.CancelRequested); var reported = await NextJob(events); Assert.Equal(progress, reported.Job!.Progress); Assert.NotNull(await jobs.ReportPlanAsync(job.Id, "w1", "{\"plan\":1}")); var plan = await Next(events); Assert.Equal(NestJobEventKind.Plan, plan.Kind); Assert.Equal("{\"plan\":1}", plan.Plan); var planned = await NextJob(events); Assert.Equal(plan.Id, planned.Job!.PlanRevision); Assert.Equal(plan.Id, planned.Id); Assert.True(await jobs.CompleteAsync(job.Id, "w1", "{}")); var final = await NextJob(events); Assert.Equal(NestJobState.Succeeded, final.Job!.State); // Nothing after the final state: the server ended the stream. await done.WaitAsync(Wait); Assert.False(await events.WaitToReadAsync()); var ids = new[] { queued, running, reported, planned, final }.Select(e => e.Id).ToList(); Assert.Equal(ids.OrderBy(i => i).Distinct(), ids); } [Fact] public async Task AReconnectSkipsWhatItAlreadyHas() { using var factory = new ServerFactory(); var jobs = Client(factory); var job = await jobs.SubmitAsync(Describe(), new byte[] { 1 }); await jobs.ClaimAsync("w1"); await jobs.ReportPlanAsync(job.Id, "w1", "{\"plan\":1}"); var current = (await jobs.GetAsync(job.Id))!; // Up to date: only the stream opening, then whatever changes next. var (events, _) = Listen(jobs, job.Id, current.Revision); Assert.Equal(NestJobEventKind.Open, (await Next(events)).Kind); await jobs.ReportProgressAsync(job.Id, "w1", "{\"version\":1}"); var next = await NextJob(events); Assert.Equal(current.Revision + 1, next.Id); // Behind the plan: the plan again, then the job. var (behind, _) = Listen(jobs, job.Id, current.PlanRevision - 1); Assert.Equal(NestJobEventKind.Open, (await Next(behind)).Kind); Assert.Equal(NestJobEventKind.Plan, (await Next(behind)).Kind); Assert.Equal(next.Id, (await NextJob(behind)).Id); // A finished job sends its final state even to a stream that has it, then ends. await jobs.CompleteAsync(job.Id, "w1", "{}"); var finished = (await jobs.GetAsync(job.Id))!; var (late, lateDone) = Listen(jobs, job.Id, finished.Revision); Assert.Equal(NestJobEventKind.Open, (await Next(late)).Kind); Assert.Equal(NestJobState.Succeeded, (await NextJob(late)).Job!.State); await lateDone.WaitAsync(Wait); } [Fact] public async Task UnknownJobsHaveNoStream() { using var factory = new ServerFactory(); var jobs = Client(factory); var events = new List(); await foreach (var update in jobs.WatchAsync(Guid.NewGuid())) events.Add(update); Assert.Equal(NestJobEventKind.Gone, Assert.Single(events).Kind); Assert.Null(await jobs.GetPlanAsync(Guid.NewGuid())); } [Fact] public async Task OnlyTheWorkerHoldingAJobReportsOnIt() { using var factory = new ServerFactory(); var jobs = Client(factory); var job = await jobs.SubmitAsync(Describe(), new byte[] { 1 }); Assert.Null(await jobs.ReportProgressAsync(job.Id, "w1", "{}")); await jobs.ClaimAsync("w1"); Assert.Null(await jobs.ReportPlanAsync(job.Id, "w2", "{}")); Assert.Null(await jobs.GetPlanAsync(job.Id)); await jobs.ReportPlanAsync(job.Id, "w1", "{\"a\":1}"); Assert.Equal("{\"a\":1}", (await jobs.GetPlanAsync(job.Id))!.Value.Plan); await jobs.CancelAsync(job.Id); Assert.True((await jobs.ReportProgressAsync(job.Id, "w1", "{}"))!.CancelRequested); // Giving the job back starts the next attempt without the last one's plan or stage. await jobs.ReleaseAsync(job.Id, "w1"); var back = (await jobs.GetAsync(job.Id))!; Assert.Equal(NestJobState.Cancelled, back.State); Assert.Null(await jobs.GetPlanAsync(job.Id)); Assert.Equal("", back.Progress); } /// Lets a test break the event stream or hide it, as a proxy or an older server would. private sealed class StreamTamper : DelegatingHandler { public bool Hide { get; set; } /// The first this many streams drop right after their first job event. public int Cuts { get; set; } public List LastEventIds { get; } = new(); protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (request.RequestUri!.AbsolutePath.EndsWith("/events")) { LastEventIds.Add(request.Headers.TryGetValues("Last-Event-ID", out var ids) ? ids.Single() : null); if (Hide) return new HttpResponseMessage(HttpStatusCode.NotFound); var response = await base.SendAsync(request, cancellationToken); if (Cuts-- > 0) response.Content = new StreamContent(new CutStream(await response.Content.ReadAsStreamAsync(cancellationToken))); return response; } return await base.SendAsync(request, cancellationToken); } } private sealed class CutStream(Stream inner) : Stream { private readonly System.Text.StringBuilder seen = new(); private int read; public override bool CanRead => true; public override bool CanSeek => false; public override bool CanWrite => false; public override long Length => throw new NotSupportedException(); public override long Position { get => read; set => throw new NotSupportedException(); } public override void Flush() { } public override int Read(byte[] buffer, int offset, int count) => ReadAsync(buffer, offset, count).GetAwaiter().GetResult(); public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) { var text = seen.ToString(); var job = text.IndexOf("event: job", StringComparison.Ordinal); if (job >= 0 && text.IndexOf("\n\n", job, StringComparison.Ordinal) >= 0) throw new IOException("Connection reset."); // A byte at a time, so nothing past the first job event is delivered. var n = await inner.ReadAsync(buffer[..System.Math.Min(buffer.Length, 1)], cancellationToken); seen.Append(System.Text.Encoding.UTF8.GetString(buffer.Span[..n])); read += n; return n; } public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); public override void SetLength(long value) => throw new NotSupportedException(); public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); protected override void Dispose(bool disposing) { if (disposing) inner.Dispose(); base.Dispose(disposing); } } private sealed class Sink : IProgress { public readonly List Items = new(); public void Report(T value) { lock (Items) Items.Add(value); } } private static async Task WatchWhileTheWorkerRuns(ServerFactory factory, StreamTamper tamper, Sink seen, Sink plans) { var desktop = Client(factory, tamper); var job = await desktop.SubmitAsync(Describe(), NestJobPackage.Write(Request())); var watch = new NestJobWatch(desktop) { PollInterval = TimeSpan.FromMilliseconds(50), Job = seen, Plan = plans }; var watching = watch.RunAsync(job); var worker = Client(factory); await worker.ClaimAsync("w1"); await worker.ReportPlanAsync(job.Id, "w1", "{\"plan\":1}"); await worker.ReportProgressAsync(job.Id, "w1", "{\"version\":1}"); await worker.CompleteAsync(job.Id, "w1", "{}"); return await watching.WaitAsync(Wait); } [Fact] public async Task TheWatchReconnectsADroppedStreamFromItsLastEvent() { using var factory = new ServerFactory(); // The first two streams drop after their first job event. var tamper = new StreamTamper { Cuts = 2 }; var seen = new Sink(); var plans = new Sink(); var final = await WatchWhileTheWorkerRuns(factory, tamper, seen, plans); Assert.Equal(NestJobState.Succeeded, final.State); Assert.True(tamper.LastEventIds.Count >= 2, "The stream should have been opened again."); Assert.Null(tamper.LastEventIds[0]); Assert.Contains(tamper.LastEventIds.Skip(1), id => id is not null); Assert.Equal(NestJobState.Succeeded, seen.Items[^1].State); } [Fact] public async Task TheWatchPollsAServerWithoutTheStream() { using var factory = new ServerFactory(); var tamper = new StreamTamper { Hide = true }; var seen = new Sink(); var plans = new Sink(); var final = await WatchWhileTheWorkerRuns(factory, tamper, seen, plans); Assert.Equal(NestJobState.Succeeded, final.State); Assert.Single(tamper.LastEventIds); Assert.Equal(NestJobState.Succeeded, seen.Items[^1].State); } [Fact] public async Task AnEnginePlanReachesTheDesktopAsTheSheetsItDraws() { using var factory = new ServerFactory(); var desktop = Client(factory); var request = Request(); var job = await desktop.SubmitAsync(Describe(), NestJobPackage.Write(request)); var plans = new Sink(); var watching = new NestJobWatch(desktop) { Plan = plans }.RunAsync(job); // The engine's plans, as the worker's recording engine passes them on. var reporter = new NestJobReporter(); var engine = new RecordingNestingEngine(NestingEngineRegistry.Create(request.EngineName)) { Progress = reporter }; var worker = Client(factory); await worker.ClaimAsync("w1"); using var stop = new CancellationTokenSource(); var reporting = reporter.RunAsync(worker, job.Id, "w1", TimeSpan.FromMilliseconds(20), NullLogger.Instance, stop.Token); var local = NestPipeline.Run(engine, request.EngineName, request); var placed = local.Plates.Sum(p => p.Quantity * p.Parts.Count); // The engine's last plan is its answer: drawn here, it holds every part of the result. var preview = new RemoteNestPlanPreview(request); using (var timeout = new CancellationTokenSource(Wait)) { while (!plans.Items.ToList().Any(plan => preview.Sheets(plan)?.Sum(s => s.Quantity * s.Parts.Count) == placed)) await Task.Delay(20, timeout.Token); } stop.Cancel(); Assert.Equal(NestJobWorker.StopReason.None, await reporting); await worker.CompleteAsync(job.Id, "w1", engine.Transcript.ToJson()); Assert.Equal(NestJobState.Succeeded, (await watching.WaitAsync(Wait)).State); } [Fact] public async Task AWorkerSeesACancelRequestWithinOneReportInterval() { using var factory = new ServerFactory(); var jobs = Client(factory); var job = await jobs.SubmitAsync(Describe(), new byte[] { 1 }); await jobs.ClaimAsync("w1"); var reporter = new NestJobReporter(); using var stop = new CancellationTokenSource(Wait); var interval = TimeSpan.FromMilliseconds(100); var reporting = reporter.RunAsync(jobs, job.Id, "w1", interval, NullLogger.Instance, stop.Token); reporter.Report(new NestJobProgress(NestJobStage.EvaluatingCandidate, "s", 0, 0, 0)); using (var timeout = new CancellationTokenSource(Wait)) { while (!(await jobs.GetAsync(job.Id))!.Progress.Contains("evaluatingCandidate")) await Task.Delay(50, timeout.Token); } Assert.False(reporting.IsCompleted); var asked = DateTime.UtcNow; await jobs.CancelAsync(job.Id); Assert.Equal(NestJobWorker.StopReason.CancelRequested, await reporting.WaitAsync(Wait)); Assert.True(DateTime.UtcNow - asked < TimeSpan.FromSeconds(5)); } }