Merge runevent-snapshot: write a subscribe snapshot in full before its ack
The run-event hub seeded a new subscription by sending every matching run through the subscription's bounded 32-frame channel and only then acking. A snapshot larger than that buffer tripped the drop-a-slow-subscriber policy on the snapshot itself, so the remaining snapshot frames, the ack and every later delta were lost while the websocket stayed open - the client could not tell it had been unsubscribed. The snapshot and the ack now go straight out on the subscribing connection, deltas emitted meanwhile are held and released in order before the writer goroutine starts, and the drop policy now applies only to deltas. Guarded by TestRunEventSubscribeSnapshotLargerThanBuffer.
This commit is contained in:
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -212,6 +213,95 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose
|
||||
// snapshot holds more runs than the subscription's bounded buffer delivers the
|
||||
// snapshot in full, then the ack, then a live delta for a run started
|
||||
// afterwards. Before the snapshot fix, the subscription wrote its own snapshot
|
||||
// through the bounded channel, so sendMessage's drop-a-slow-subscriber policy
|
||||
// fired on the snapshot itself: the subscription closed part-way through, the
|
||||
// client saw at most runEventSubscriberBuffer frames, and it never received the
|
||||
// ack or a single later delta.
|
||||
func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
_, handler, ids := eventStore(t, ctx, "one")
|
||||
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
const seeded = runEventSubscriberBuffer + 8
|
||||
// Seed more finished runs than the buffer holds, each in its own workspace so
|
||||
// no two runs share a tree, then wait for every one to settle. A blank-scope
|
||||
// subscribe sees them all.
|
||||
seededIDs := make([]int64, 0, seeded)
|
||||
for i := 0; i < seeded; i++ {
|
||||
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seededIDs = append(seededIDs, run.ID)
|
||||
}
|
||||
handler.runner.wg.Wait()
|
||||
|
||||
client := dialRunEvents(t, ctx, handler)
|
||||
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""})
|
||||
snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second)
|
||||
defer cancelSnapshot()
|
||||
seen := map[int64]bool{}
|
||||
var ack *protocol.Message
|
||||
for ack == nil || len(seen) < seeded {
|
||||
var message protocol.Message
|
||||
if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil {
|
||||
t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err)
|
||||
}
|
||||
switch message.Type {
|
||||
case protocol.RunEvent:
|
||||
if message.RunEvent != nil {
|
||||
seen[message.RunEvent.RunID] = true
|
||||
}
|
||||
case protocol.Subscribe:
|
||||
got := message
|
||||
ack = &got
|
||||
// The snapshot precedes the ack: when the ack is read every snapshot
|
||||
// frame must already have been, or a frame was reordered after it.
|
||||
if len(seen) != seeded {
|
||||
t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ack.ID != "snap" {
|
||||
t.Fatalf("subscribe ack = %+v", *ack)
|
||||
}
|
||||
for _, id := range seededIDs {
|
||||
if !seen[id] {
|
||||
t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded)
|
||||
}
|
||||
}
|
||||
|
||||
// The subscription must outlive its own snapshot: a run started after it must
|
||||
// still deliver a delta, through to its terminal frame.
|
||||
late := eventWorkspace(t)
|
||||
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}})
|
||||
deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second)
|
||||
defer cancelDelta()
|
||||
var terminal *protocol.RunEventFrame
|
||||
for terminal == nil {
|
||||
var message protocol.Message
|
||||
if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil {
|
||||
t.Fatalf("no terminal delta after the snapshot: %v", err)
|
||||
}
|
||||
if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late {
|
||||
continue
|
||||
}
|
||||
switch message.RunEvent.Status {
|
||||
case "succeeded", "failed", "canceled":
|
||||
terminal = message.RunEvent
|
||||
}
|
||||
}
|
||||
if terminal.RunID == 0 || terminal.Outcome != "success" {
|
||||
t.Fatalf("late terminal frame = %+v", terminal)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's
|
||||
// identity from an earlier transition when a later delta omits it, so the merged
|
||||
// frame a client actually receives still carries what its row needs.
|
||||
|
||||
+104
-16
@@ -54,9 +54,16 @@ type runCoalescer struct {
|
||||
}
|
||||
|
||||
// runSubscription is one connection's watch over a workspace (and optionally
|
||||
// explicit run ids). Frames are handed to a dedicated writer goroutine so a slow
|
||||
// explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow
|
||||
// or broken subscriber never stalls the producer, and its buffered channel
|
||||
// bounds how far behind it may fall.
|
||||
//
|
||||
// A subscription starts in its snapshot phase: the subscribing connection
|
||||
// writes the snapshot and the ack itself (writeSnapshot) and only then starts
|
||||
// the writer goroutine (ready). mu guards that phase, so the drop-a-slow-
|
||||
// subscriber policy can never fire on the snapshot or lose its ack: while
|
||||
// snapshotting a delta is held in pending instead of the bounded channel, and
|
||||
// nothing writes to the channel until ready has released the held frames.
|
||||
type runSubscription struct {
|
||||
hub *runEventHub
|
||||
clientID string
|
||||
@@ -66,6 +73,10 @@ type runSubscription struct {
|
||||
done chan struct{}
|
||||
once sync.Once
|
||||
write func(protocol.Message) error
|
||||
|
||||
mu sync.Mutex
|
||||
snapshotting bool
|
||||
pending []protocol.Message
|
||||
}
|
||||
|
||||
func newRunEventHub() *runEventHub {
|
||||
@@ -85,8 +96,11 @@ func (h *runEventHub) clock() time.Time {
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// subscribe registers one connection's watch and starts its writer goroutine. It
|
||||
// returns nil when the connection already holds the per-client subscription cap.
|
||||
// subscribe registers one connection's watch and returns it in its snapshot
|
||||
// phase: the caller delivers the snapshot and the ack with writeSnapshot and
|
||||
// then calls ready, which releases the deltas emitted meanwhile and starts the
|
||||
// writer goroutine. It returns nil when the connection already holds the
|
||||
// per-client subscription cap.
|
||||
func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription {
|
||||
if h == nil || write == nil {
|
||||
return nil
|
||||
@@ -97,12 +111,13 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ
|
||||
return nil
|
||||
}
|
||||
sub := &runSubscription{
|
||||
hub: h,
|
||||
clientID: clientID,
|
||||
workspace: workspace,
|
||||
ch: make(chan protocol.Message, runEventSubscriberBuffer),
|
||||
done: make(chan struct{}),
|
||||
write: write,
|
||||
hub: h,
|
||||
clientID: clientID,
|
||||
workspace: workspace,
|
||||
ch: make(chan protocol.Message, runEventSubscriberBuffer),
|
||||
done: make(chan struct{}),
|
||||
write: write,
|
||||
snapshotting: true,
|
||||
}
|
||||
for _, id := range runIDs {
|
||||
if id <= 0 {
|
||||
@@ -119,7 +134,8 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ
|
||||
}
|
||||
h.byClient[clientID][sub] = struct{}{}
|
||||
h.mu.Unlock()
|
||||
go sub.run()
|
||||
// The writer goroutine starts only in ready, after the snapshot and the ack
|
||||
// are on the wire, so a delta can never overtake them.
|
||||
return sub
|
||||
}
|
||||
|
||||
@@ -320,6 +336,47 @@ func (s *runSubscription) run() {
|
||||
}
|
||||
}
|
||||
|
||||
// writeSnapshot writes one frame of the subscription's snapshot directly on the
|
||||
// subscribing connection. It runs only during the snapshot phase, before the
|
||||
// writer goroutine exists, so it cannot reorder against a delta and is never
|
||||
// subject to the bounded channel's drop policy. It returns false when the frame
|
||||
// could not be written, having closed the subscription; the caller must then
|
||||
// stop writing the snapshot.
|
||||
func (s *runSubscription) writeSnapshot(msg protocol.Message) bool {
|
||||
select {
|
||||
case <-s.done:
|
||||
return false
|
||||
default:
|
||||
}
|
||||
if err := s.write(msg); err != nil {
|
||||
s.close()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ready ends the snapshot phase: the deltas held while the snapshot was written
|
||||
// are moved into the channel in order and the writer goroutine starts. The
|
||||
// channel is empty until then (sendMessage queues into pending) and pending is
|
||||
// bounded by the channel's capacity, so the move cannot block. A subscription
|
||||
// closed during its snapshot starts no goroutine at all.
|
||||
func (s *runSubscription) ready() {
|
||||
s.mu.Lock()
|
||||
select {
|
||||
case <-s.done:
|
||||
s.mu.Unlock()
|
||||
return
|
||||
default:
|
||||
}
|
||||
for _, msg := range s.pending {
|
||||
s.ch <- msg
|
||||
}
|
||||
s.pending = nil
|
||||
s.snapshotting = false
|
||||
s.mu.Unlock()
|
||||
go s.run()
|
||||
}
|
||||
|
||||
func (s *runSubscription) send(frame protocol.RunEventFrame) {
|
||||
s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame})
|
||||
}
|
||||
@@ -330,6 +387,23 @@ func (s *runSubscription) sendMessage(msg protocol.Message) {
|
||||
return
|
||||
default:
|
||||
}
|
||||
s.mu.Lock()
|
||||
if s.snapshotting {
|
||||
// The snapshot is still being written on the subscribing connection and
|
||||
// no writer goroutine is draining yet, so closing here would truncate a
|
||||
// snapshot in flight and swallow its ack. Hold the delta instead, bounded
|
||||
// the same as the channel: the policy can only ever affect deltas, since
|
||||
// the snapshot and the ack are written directly by the subscriber.
|
||||
if len(s.pending) >= runEventSubscriberBuffer {
|
||||
s.mu.Unlock()
|
||||
s.close()
|
||||
return
|
||||
}
|
||||
s.pending = append(s.pending, msg)
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case s.ch <- msg:
|
||||
default:
|
||||
@@ -395,9 +469,12 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
||||
|
||||
// handleRunEvents dispatches one subscribe/unsubscribe request and reports
|
||||
// whether it was a run-event type (so the caller never falls through to the
|
||||
// generic unknown-command response). Subscribe registers the watch, sends the
|
||||
// current state of the watched runs through the subscription, then acks; the
|
||||
// snapshot therefore precedes every later delta.
|
||||
// generic unknown-command response). Subscribe registers the watch, writes the
|
||||
// current state of the watched runs as its snapshot, acks, and only then lets
|
||||
// deltas flow: the snapshot and the ack go straight out on the subscribing
|
||||
// connection, so a snapshot larger than the subscription's buffer can no longer
|
||||
// drop the subscription it is seeding, and the ordering is always snapshot,
|
||||
// ack, deltas.
|
||||
func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool {
|
||||
switch request.Type {
|
||||
case protocol.Subscribe:
|
||||
@@ -414,6 +491,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
// workspace-scoped read the polling clients use, instead of a new query.
|
||||
summaries, err := runs.listRuns()
|
||||
if err != nil {
|
||||
sub.close()
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
||||
return true
|
||||
}
|
||||
@@ -421,7 +499,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
if !sub.matches(summary.Workspace, summary.ID) {
|
||||
continue
|
||||
}
|
||||
sub.send(protocol.RunEventFrame{
|
||||
frame := protocol.RunEventFrame{
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
@@ -434,9 +512,19 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
})
|
||||
}
|
||||
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
|
||||
// The connection write failed; the subscription is closed and the
|
||||
// read loop will stop on its next read error, so there is nothing
|
||||
// left to reply to.
|
||||
return true
|
||||
}
|
||||
}
|
||||
sub.sendMessage(protocol.Message{Type: protocol.Subscribe, ID: request.ID})
|
||||
if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil {
|
||||
sub.close()
|
||||
return true
|
||||
}
|
||||
sub.ready()
|
||||
return true
|
||||
case protocol.Unsubscribe:
|
||||
events.unsubscribeClient(clientID)
|
||||
|
||||
Reference in New Issue
Block a user