server: write a subscribe snapshot in full before its ack

handleRunEvents seeded a new subscription by pushing every matching run
through the subscription channel (runEventSubscriberBuffer = 32) and only
then acked. sendMessage is a non-blocking send that closes the
subscription when the channel is full, so a snapshot larger than the
buffer dropped the subscription it was seeding: the remaining snapshot
frames hit the closed-done early return, the ack was never sent, and no
later delta could arrive. The connection stayed open, so the client could
not tell it was no longer subscribed. This was the app's only
subscription - AppViewModel subscribes with a blank workspace, matching
every run - so the app's live state froze.

The subscription now starts in a snapshot phase. The subscribing
connection itself writes the snapshot frames via writeSnapshot (one
direct write per frame, outside the bounded channel, so the drop policy
cannot fire on them) and the ack, and ready then moves any deltas held
meanwhile into the channel in order and starts the writer goroutine.
sendMessage holds deltas in a bounded pending queue while snapshotting
instead of closing, so the drop-a-slow-subscriber policy still applies,
but only to deltas. Ordering is snapshot, ack, then deltas; a write
failure or a listRuns error closes the subscription instead of leaving
it registered.

Numbers reported by this run: the probe against the deployed server saw
blank-scope frames=2 ack=0 (scoped frames=1 ack=0) because the snapshot
was ~252 frames into a 32-frame buffer; the scratch-server probe with the
fix saw 40 snapshot frames, ack=1 and a delta after a run transition.
Guarded now by TestRunEventSubscribeSnapshotLargerThanBuffer (seeds 40
finished runs, asserts every snapshot frame, the ack strictly after them,
and a terminal delta for a run started after subscribing), which fails
against the previous runevents.go.

Checked for staleness: the packaged templates and the packaged authoring
skill references internal/author/skill/references/ carry no run-event
subscription contract (their "snapshot" mentions are the workspace-change
snapshots in operations.md), and README carries none either, so none of
them needs updating. No UI surface changed.

Checks run before committing: gofmt -l . prints nothing, go build ./...,
go vet ./..., go test ./... (all packages ok), go test -race
./internal/server (ok).
This commit is contained in:
2026-10-18 02:23:58 +01:00
parent 0d9430e4ad
commit cff6f4dfac
2 changed files with 194 additions and 16 deletions
@@ -2,6 +2,7 @@ package server
import (
"context"
"fmt"
"net/http/httptest"
"path/filepath"
"strings"
@@ -212,6 +213,95 @@ func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
}
}
// TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose
// snapshot holds more runs than the subscription's bounded buffer delivers the
// snapshot in full, then the ack, then a live delta for a run started
// afterwards. Before the snapshot fix, the subscription wrote its own snapshot
// through the bounded channel, so sendMessage's drop-a-slow-subscriber policy
// fired on the snapshot itself: the subscription closed part-way through, the
// client saw at most runEventSubscriberBuffer frames, and it never received the
// ack or a single later delta.
func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
const seeded = runEventSubscriberBuffer + 8
// Seed more finished runs than the buffer holds, each in its own workspace so
// no two runs share a tree, then wait for every one to settle. A blank-scope
// subscribe sees them all.
seededIDs := make([]int64, 0, seeded)
for i := 0; i < seeded; i++ {
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
seededIDs = append(seededIDs, run.ID)
}
handler.runner.wg.Wait()
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""})
snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second)
defer cancelSnapshot()
seen := map[int64]bool{}
var ack *protocol.Message
for ack == nil || len(seen) < seeded {
var message protocol.Message
if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil {
t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err)
}
switch message.Type {
case protocol.RunEvent:
if message.RunEvent != nil {
seen[message.RunEvent.RunID] = true
}
case protocol.Subscribe:
got := message
ack = &got
// The snapshot precedes the ack: when the ack is read every snapshot
// frame must already have been, or a frame was reordered after it.
if len(seen) != seeded {
t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded)
}
}
}
if ack.ID != "snap" {
t.Fatalf("subscribe ack = %+v", *ack)
}
for _, id := range seededIDs {
if !seen[id] {
t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded)
}
}
// The subscription must outlive its own snapshot: a run started after it must
// still deliver a delta, through to its terminal frame.
late := eventWorkspace(t)
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}})
deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second)
defer cancelDelta()
var terminal *protocol.RunEventFrame
for terminal == nil {
var message protocol.Message
if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil {
t.Fatalf("no terminal delta after the snapshot: %v", err)
}
if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late {
continue
}
switch message.RunEvent.Status {
case "succeeded", "failed", "canceled":
terminal = message.RunEvent
}
}
if terminal.RunID == 0 || terminal.Outcome != "success" {
t.Fatalf("late terminal frame = %+v", terminal)
}
}
// TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's
// identity from an earlier transition when a later delta omits it, so the merged
// frame a client actually receives still carries what its row needs.
+104 -16
View File
@@ -54,9 +54,16 @@ type runCoalescer struct {
}
// runSubscription is one connection's watch over a workspace (and optionally
// explicit run ids). Frames are handed to a dedicated writer goroutine so a slow
// explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow
// or broken subscriber never stalls the producer, and its buffered channel
// bounds how far behind it may fall.
//
// A subscription starts in its snapshot phase: the subscribing connection
// writes the snapshot and the ack itself (writeSnapshot) and only then starts
// the writer goroutine (ready). mu guards that phase, so the drop-a-slow-
// subscriber policy can never fire on the snapshot or lose its ack: while
// snapshotting a delta is held in pending instead of the bounded channel, and
// nothing writes to the channel until ready has released the held frames.
type runSubscription struct {
hub *runEventHub
clientID string
@@ -66,6 +73,10 @@ type runSubscription struct {
done chan struct{}
once sync.Once
write func(protocol.Message) error
mu sync.Mutex
snapshotting bool
pending []protocol.Message
}
func newRunEventHub() *runEventHub {
@@ -85,8 +96,11 @@ func (h *runEventHub) clock() time.Time {
return time.Now()
}
// subscribe registers one connection's watch and starts its writer goroutine. It
// returns nil when the connection already holds the per-client subscription cap.
// subscribe registers one connection's watch and returns it in its snapshot
// phase: the caller delivers the snapshot and the ack with writeSnapshot and
// then calls ready, which releases the deltas emitted meanwhile and starts the
// writer goroutine. It returns nil when the connection already holds the
// per-client subscription cap.
func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription {
if h == nil || write == nil {
return nil
@@ -97,12 +111,13 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ
return nil
}
sub := &runSubscription{
hub: h,
clientID: clientID,
workspace: workspace,
ch: make(chan protocol.Message, runEventSubscriberBuffer),
done: make(chan struct{}),
write: write,
hub: h,
clientID: clientID,
workspace: workspace,
ch: make(chan protocol.Message, runEventSubscriberBuffer),
done: make(chan struct{}),
write: write,
snapshotting: true,
}
for _, id := range runIDs {
if id <= 0 {
@@ -119,7 +134,8 @@ func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, writ
}
h.byClient[clientID][sub] = struct{}{}
h.mu.Unlock()
go sub.run()
// The writer goroutine starts only in ready, after the snapshot and the ack
// are on the wire, so a delta can never overtake them.
return sub
}
@@ -320,6 +336,47 @@ func (s *runSubscription) run() {
}
}
// writeSnapshot writes one frame of the subscription's snapshot directly on the
// subscribing connection. It runs only during the snapshot phase, before the
// writer goroutine exists, so it cannot reorder against a delta and is never
// subject to the bounded channel's drop policy. It returns false when the frame
// could not be written, having closed the subscription; the caller must then
// stop writing the snapshot.
func (s *runSubscription) writeSnapshot(msg protocol.Message) bool {
select {
case <-s.done:
return false
default:
}
if err := s.write(msg); err != nil {
s.close()
return false
}
return true
}
// ready ends the snapshot phase: the deltas held while the snapshot was written
// are moved into the channel in order and the writer goroutine starts. The
// channel is empty until then (sendMessage queues into pending) and pending is
// bounded by the channel's capacity, so the move cannot block. A subscription
// closed during its snapshot starts no goroutine at all.
func (s *runSubscription) ready() {
s.mu.Lock()
select {
case <-s.done:
s.mu.Unlock()
return
default:
}
for _, msg := range s.pending {
s.ch <- msg
}
s.pending = nil
s.snapshotting = false
s.mu.Unlock()
go s.run()
}
func (s *runSubscription) send(frame protocol.RunEventFrame) {
s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame})
}
@@ -330,6 +387,23 @@ func (s *runSubscription) sendMessage(msg protocol.Message) {
return
default:
}
s.mu.Lock()
if s.snapshotting {
// The snapshot is still being written on the subscribing connection and
// no writer goroutine is draining yet, so closing here would truncate a
// snapshot in flight and swallow its ack. Hold the delta instead, bounded
// the same as the channel: the policy can only ever affect deltas, since
// the snapshot and the ack are written directly by the subscriber.
if len(s.pending) >= runEventSubscriberBuffer {
s.mu.Unlock()
s.close()
return
}
s.pending = append(s.pending, msg)
s.mu.Unlock()
return
}
s.mu.Unlock()
select {
case s.ch <- msg:
default:
@@ -395,9 +469,12 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
// handleRunEvents dispatches one subscribe/unsubscribe request and reports
// whether it was a run-event type (so the caller never falls through to the
// generic unknown-command response). Subscribe registers the watch, sends the
// current state of the watched runs through the subscription, then acks; the
// snapshot therefore precedes every later delta.
// generic unknown-command response). Subscribe registers the watch, writes the
// current state of the watched runs as its snapshot, acks, and only then lets
// deltas flow: the snapshot and the ack go straight out on the subscribing
// connection, so a snapshot larger than the subscription's buffer can no longer
// drop the subscription it is seeding, and the ordering is always snapshot,
// ack, deltas.
func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool {
switch request.Type {
case protocol.Subscribe:
@@ -414,6 +491,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
// workspace-scoped read the polling clients use, instead of a new query.
summaries, err := runs.listRuns()
if err != nil {
sub.close()
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return true
}
@@ -421,7 +499,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
if !sub.matches(summary.Workspace, summary.ID) {
continue
}
sub.send(protocol.RunEventFrame{
frame := protocol.RunEventFrame{
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
@@ -434,9 +512,19 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
})
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
// read loop will stop on its next read error, so there is nothing
// left to reply to.
return true
}
}
sub.sendMessage(protocol.Message{Type: protocol.Subscribe, ID: request.ID})
if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil {
sub.close()
return true
}
sub.ready()
return true
case protocol.Unsubscribe:
events.unsubscribeClient(clientID)