diff --git a/internal/server/run_event_websocket_test.go b/internal/server/run_event_websocket_test.go index b6ba611e..f2ec8df8 100644 --- a/internal/server/run_event_websocket_test.go +++ b/internal/server/run_event_websocket_test.go @@ -2,6 +2,7 @@ package server import ( "context" + "fmt" "net/http/httptest" "path/filepath" "strings" @@ -212,6 +213,95 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) { } } +// TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose +// snapshot holds more runs than the subscription's bounded buffer delivers the +// snapshot in full, then the ack, then a live delta for a run started +// afterwards. Before the snapshot fix, the subscription wrote its own snapshot +// through the bounded channel, so sendMessage's drop-a-slow-subscriber policy +// fired on the snapshot itself: the subscription closed part-way through, the +// client saw at most runEventSubscriberBuffer frames, and it never received the +// ack or a single later delta. +func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, handler, ids := eventStore(t, ctx, "one") + handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + const seeded = runEventSubscriberBuffer + 8 + // Seed more finished runs than the buffer holds, each in its own workspace so + // no two runs share a tree, then wait for every one to settle. A blank-scope + // subscribe sees them all. + seededIDs := make([]int64, 0, seeded) + for i := 0; i < seeded; i++ { + run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}}) + if err != nil { + t.Fatal(err) + } + seededIDs = append(seededIDs, run.ID) + } + handler.runner.wg.Wait() + + client := dialRunEvents(t, ctx, handler) + client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""}) + snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second) + defer cancelSnapshot() + seen := map[int64]bool{} + var ack *protocol.Message + for ack == nil || len(seen) < seeded { + var message protocol.Message + if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil { + t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err) + } + switch message.Type { + case protocol.RunEvent: + if message.RunEvent != nil { + seen[message.RunEvent.RunID] = true + } + case protocol.Subscribe: + got := message + ack = &got + // The snapshot precedes the ack: when the ack is read every snapshot + // frame must already have been, or a frame was reordered after it. + if len(seen) != seeded { + t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded) + } + } + } + if ack.ID != "snap" { + t.Fatalf("subscribe ack = %+v", *ack) + } + for _, id := range seededIDs { + if !seen[id] { + t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded) + } + } + + // The subscription must outlive its own snapshot: a run started after it must + // still deliver a delta, through to its terminal frame. + late := eventWorkspace(t) + client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}}) + deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second) + defer cancelDelta() + var terminal *protocol.RunEventFrame + for terminal == nil { + var message protocol.Message + if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil { + t.Fatalf("no terminal delta after the snapshot: %v", err) + } + if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late { + continue + } + switch message.RunEvent.Status { + case "succeeded", "failed", "canceled": + terminal = message.RunEvent + } + } + if terminal.RunID == 0 || terminal.Outcome != "success" { + t.Fatalf("late terminal frame = %+v", terminal) + } +} + // TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's // identity from an earlier transition when a later delta omits it, so the merged // frame a client actually receives still carries what its row needs. diff --git a/internal/server/runevents.go b/internal/server/runevents.go index b17ef92c..e35d683d 100644 --- a/internal/server/runevents.go +++ b/internal/server/runevents.go @@ -54,9 +54,16 @@ type runCoalescer struct { } // runSubscription is one connection's watch over a workspace (and optionally -// explicit run ids). Frames are handed to a dedicated writer goroutine so a slow +// explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow // or broken subscriber never stalls the producer, and its buffered channel // bounds how far behind it may fall. +// +// A subscription starts in its snapshot phase: the subscribing connection +// writes the snapshot and the ack itself (writeSnapshot) and only then starts +// the writer goroutine (ready). mu guards that phase, so the drop-a-slow- +// subscriber policy can never fire on the snapshot or lose its ack: while +// snapshotting a delta is held in pending instead of the bounded channel, and +// nothing writes to the channel until ready has released the held frames. type runSubscription struct { hub *runEventHub clientID string @@ -66,6 +73,10 @@ type runSubscription struct { done chan struct{} once sync.Once write func(protocol.Message) error + + mu sync.Mutex + snapshotting bool + pending []protocol.Message } func newRunEventHub() *runEventHub { @@ -85,8 +96,11 @@ func (h *runEventHub) clock() time.Time { return time.Now() } -// subscribe registers one connection's watch and starts its writer goroutine. It -// returns nil when the connection already holds the per-client subscription cap. +// subscribe registers one connection's watch and returns it in its snapshot +// phase: the caller delivers the snapshot and the ack with writeSnapshot and +// then calls ready, which releases the deltas emitted meanwhile and starts the +// writer goroutine. It returns nil when the connection already holds the +// per-client subscription cap. func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription { if h == nil || write == nil { return nil @@ -97,12 +111,13 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ return nil } sub := &runSubscription{ - hub: h, - clientID: clientID, - workspace: workspace, - ch: make(chan protocol.Message, runEventSubscriberBuffer), - done: make(chan struct{}), - write: write, + hub: h, + clientID: clientID, + workspace: workspace, + ch: make(chan protocol.Message, runEventSubscriberBuffer), + done: make(chan struct{}), + write: write, + snapshotting: true, } for _, id := range runIDs { if id <= 0 { @@ -119,7 +134,8 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ } h.byClient[clientID][sub] = struct{}{} h.mu.Unlock() - go sub.run() + // The writer goroutine starts only in ready, after the snapshot and the ack + // are on the wire, so a delta can never overtake them. return sub } @@ -320,6 +336,47 @@ func (s *runSubscription) run() { } } +// writeSnapshot writes one frame of the subscription's snapshot directly on the +// subscribing connection. It runs only during the snapshot phase, before the +// writer goroutine exists, so it cannot reorder against a delta and is never +// subject to the bounded channel's drop policy. It returns false when the frame +// could not be written, having closed the subscription; the caller must then +// stop writing the snapshot. +func (s *runSubscription) writeSnapshot(msg protocol.Message) bool { + select { + case <-s.done: + return false + default: + } + if err := s.write(msg); err != nil { + s.close() + return false + } + return true +} + +// ready ends the snapshot phase: the deltas held while the snapshot was written +// are moved into the channel in order and the writer goroutine starts. The +// channel is empty until then (sendMessage queues into pending) and pending is +// bounded by the channel's capacity, so the move cannot block. A subscription +// closed during its snapshot starts no goroutine at all. +func (s *runSubscription) ready() { + s.mu.Lock() + select { + case <-s.done: + s.mu.Unlock() + return + default: + } + for _, msg := range s.pending { + s.ch <- msg + } + s.pending = nil + s.snapshotting = false + s.mu.Unlock() + go s.run() +} + func (s *runSubscription) send(frame protocol.RunEventFrame) { s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) } @@ -330,6 +387,23 @@ func (s *runSubscription) sendMessage(msg protocol.Message) { return default: } + s.mu.Lock() + if s.snapshotting { + // The snapshot is still being written on the subscribing connection and + // no writer goroutine is draining yet, so closing here would truncate a + // snapshot in flight and swallow its ack. Hold the delta instead, bounded + // the same as the channel: the policy can only ever affect deltas, since + // the snapshot and the ack are written directly by the subscriber. + if len(s.pending) >= runEventSubscriberBuffer { + s.mu.Unlock() + s.close() + return + } + s.pending = append(s.pending, msg) + s.mu.Unlock() + return + } + s.mu.Unlock() select { case s.ch <- msg: default: @@ -395,9 +469,12 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) { // handleRunEvents dispatches one subscribe/unsubscribe request and reports // whether it was a run-event type (so the caller never falls through to the -// generic unknown-command response). Subscribe registers the watch, sends the -// current state of the watched runs through the subscription, then acks; the -// snapshot therefore precedes every later delta. +// generic unknown-command response). Subscribe registers the watch, writes the +// current state of the watched runs as its snapshot, acks, and only then lets +// deltas flow: the snapshot and the ack go straight out on the subscribing +// connection, so a snapshot larger than the subscription's buffer can no longer +// drop the subscription it is seeding, and the ordering is always snapshot, +// ack, deltas. func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool { switch request.Type { case protocol.Subscribe: @@ -414,6 +491,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner // workspace-scoped read the polling clients use, instead of a new query. summaries, err := runs.listRuns() if err != nil { + sub.close() _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) return true } @@ -421,7 +499,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner if !sub.matches(summary.Workspace, summary.ID) { continue } - sub.send(protocol.RunEventFrame{ + frame := protocol.RunEventFrame{ RunID: summary.ID, Workspace: summary.Workspace, PipelineID: summary.PipelineID, @@ -434,9 +512,19 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner CompletedSteps: summary.CompletedSteps, CreatedAt: summary.CreatedAt, FinishedAt: summary.FinishedAt, - }) + } + if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) { + // The connection write failed; the subscription is closed and the + // read loop will stop on its next read error, so there is nothing + // left to reply to. + return true + } } - sub.sendMessage(protocol.Message{Type: protocol.Subscribe, ID: request.ID}) + if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil { + sub.close() + return true + } + sub.ready() return true case protocol.Unsubscribe: events.unsubscribeClient(clientID)