Files
pipelines/internal/server/run_event_websocket_test.go
T
alan cff6f4dfac server: write a subscribe snapshot in full before its ack
handleRunEvents seeded a new subscription by pushing every matching run
through the subscription channel (runEventSubscriberBuffer = 32) and only
then acked. sendMessage is a non-blocking send that closes the
subscription when the channel is full, so a snapshot larger than the
buffer dropped the subscription it was seeding: the remaining snapshot
frames hit the closed-done early return, the ack was never sent, and no
later delta could arrive. The connection stayed open, so the client could
not tell it was no longer subscribed. This was the app's only
subscription - AppViewModel subscribes with a blank workspace, matching
every run - so the app's live state froze.

The subscription now starts in a snapshot phase. The subscribing
connection itself writes the snapshot frames via writeSnapshot (one
direct write per frame, outside the bounded channel, so the drop policy
cannot fire on them) and the ack, and ready then moves any deltas held
meanwhile into the channel in order and starts the writer goroutine.
sendMessage holds deltas in a bounded pending queue while snapshotting
instead of closing, so the drop-a-slow-subscriber policy still applies,
but only to deltas. Ordering is snapshot, ack, then deltas; a write
failure or a listRuns error closes the subscription instead of leaving
it registered.

Numbers reported by this run: the probe against the deployed server saw
blank-scope frames=2 ack=0 (scoped frames=1 ack=0) because the snapshot
was ~252 frames into a 32-frame buffer; the scratch-server probe with the
fix saw 40 snapshot frames, ack=1 and a delta after a run transition.
Guarded now by TestRunEventSubscribeSnapshotLargerThanBuffer (seeds 40
finished runs, asserts every snapshot frame, the ack strictly after them,
and a terminal delta for a run started after subscribing), which fails
against the previous runevents.go.

Checked for staleness: the packaged templates and the packaged authoring
skill references internal/author/skill/references/ carry no run-event
subscription contract (their "snapshot" mentions are the workspace-change
snapshots in operations.md), and README carries none either, so none of
them needs updating. No UI surface changed.

Checks run before committing: gofmt -l . prints nothing, go build ./...,
go vet ./..., go test ./... (all packages ok), go test -race
./internal/server (ok).
2026-10-18 02:23:58 +01:00

486 lines
21 KiB
Go

package server
import (
"context"
"fmt"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
// runEventClient is a thin websocket client for the run-event tests: it dials
// /connect, consumes the unsolicited welcome, and reads/writes protocol
// envelopes.
type runEventClient struct {
t *testing.T
ctx context.Context
conn *websocket.Conn
}
func dialRunEvents(t *testing.T, ctx context.Context, handler *runHandler) *runEventClient {
t.Helper()
srv := httptest.NewServer(handler)
t.Cleanup(srv.Close)
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { conn.CloseNow() })
var welcome protocol.Message
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
t.Fatal(err)
}
return &runEventClient{t: t, ctx: ctx, conn: conn}
}
func (c *runEventClient) send(message protocol.Message) {
c.t.Helper()
if err := wsjson.Write(c.ctx, c.conn, message); err != nil {
c.t.Fatal(err)
}
}
func (c *runEventClient) read() protocol.Message {
c.t.Helper()
var message protocol.Message
if err := wsjson.Read(c.ctx, c.conn, &message); err != nil {
c.t.Fatal(err)
}
return message
}
// eventStore starts a fresh store holding one pipeline template and a handler
// wired to it, with the execute seam returning a deterministic success.
func eventStore(t *testing.T, ctx context.Context, stages ...string) (*pipeline.Store, *runHandler, []int64) {
t.Helper()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "events.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { store.Close() })
if _, err := store.Submit(ctx, "events", pipeline.Template(stages, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
var ids []int64
for _, item := range items {
ids = append(ids, item.ID)
}
handler := Handler(ctx, store).(*runHandler)
t.Cleanup(func() { WaitRuns(handler) })
return store, handler, ids
}
// eventWorkspace returns a temporary directory canonicalised the same way the
// runner canonicalises a run workspace (EvalSymlinks). A subscription must use
// this resolved path to match: t.TempDir() alone is a symlinked /var path on
// macOS and would never equal the run's /private/var workspace, so no frame
// would be delivered.
func eventWorkspace(t *testing.T) string {
t.Helper()
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
return dir
}
// TestRunEventWebSocketPushesTransitions starts a run, subscribes to its
// workspace, and proves the client observes the created, queued->running /
// step-started, step-finished (with the right outcome and target) and terminal
// frames, all carrying the run id.
func TestRunEventWebSocketPushesTransitions(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
workspace := eventWorkspace(t)
release := make(chan struct{})
var releaseOnce sync.Once
handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
var runID int64
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
for terminal == nil {
message := client.read()
switch message.Type {
case protocol.RunStarted:
if message.RunDetail == nil {
t.Fatal("run_started without detail")
}
runID = message.RunDetail.ID
case protocol.RunEvent:
frame := message.RunEvent
if frame == nil {
t.Fatal("run_event without payload")
}
switch {
case frame.Step == nil && frame.Status == protocol.RunStatusQueued:
created = frame
case frame.Step != nil && frame.Step.Status == protocol.StepStatusRunning:
if stepStarted == nil {
stepStarted = frame
releaseOnce.Do(func() { close(release) })
}
case frame.Step != nil && frame.Step.Status == "completed":
stepFinished = frame
case frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled":
terminal = frame
}
}
}
if runID == 0 {
t.Fatal("run_started never carried the run id")
}
if created == nil || created.RunID != runID || created.Step != nil || created.CreatedAt == "" {
t.Fatalf("created frame = %+v", created)
}
if stepStarted == nil || stepStarted.RunID != runID || stepStarted.Status != protocol.RunStatusRunning || stepStarted.CurrentStage != "one" || stepStarted.Step.Sequence != 1 || stepStarted.Step.Stage != "one" {
t.Fatalf("step-started frame = %+v", stepStarted)
}
// Every frame carries the run's pipeline identity and stored title, so a
// client that has never held the run can build its list row from the frame.
for name, frame := range map[string]*protocol.RunEventFrame{"created": created, "step-started": stepStarted} {
if frame.PipelineID != ids[0] || frame.Name == "" || frame.Title == "" || frame.Version != 1 {
t.Fatalf("%s frame identity = %+v, want pipeline %d with name/title and version 1", name, frame, ids[0])
}
}
if stepFinished == nil || stepFinished.RunID != runID || stepFinished.Step.Status != "completed" || stepFinished.Step.Outcome != "success" || stepFinished.Step.Target != protocol.SuccessEnd || stepFinished.CompletedSteps != 1 {
t.Fatalf("step-finished frame = %+v", stepFinished)
}
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
t.Fatalf("terminal frame = %+v", terminal)
}
}
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
// already-existing run receives its current state as a run_event frame before
// the subscribe ack, so a reconnecting client re-syncs without polling.
func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
handler.runner.wg.Wait()
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: workspace})
first := client.read()
if first.Type != protocol.RunEvent || first.RunEvent == nil {
t.Fatalf("first frame after subscribe = %+v, want a run_event snapshot", first)
}
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
t.Fatalf("snapshot frame = %+v", first.RunEvent)
}
// A snapshot for a run the client never saw also carries the identity its row
// needs, so a fresh subscribe renders it without a second read.
if first.RunEvent.PipelineID != ids[0] || first.RunEvent.Name == "" || first.RunEvent.Version != 1 {
t.Fatalf("snapshot identity = %+v, want pipeline %d version 1", first.RunEvent, ids[0])
}
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "snap" {
t.Fatalf("subscribe ack = %+v", ack)
}
}
// TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose
// snapshot holds more runs than the subscription's bounded buffer delivers the
// snapshot in full, then the ack, then a live delta for a run started
// afterwards. Before the snapshot fix, the subscription wrote its own snapshot
// through the bounded channel, so sendMessage's drop-a-slow-subscriber policy
// fired on the snapshot itself: the subscription closed part-way through, the
// client saw at most runEventSubscriberBuffer frames, and it never received the
// ack or a single later delta.
func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
const seeded = runEventSubscriberBuffer + 8
// Seed more finished runs than the buffer holds, each in its own workspace so
// no two runs share a tree, then wait for every one to settle. A blank-scope
// subscribe sees them all.
seededIDs := make([]int64, 0, seeded)
for i := 0; i < seeded; i++ {
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
seededIDs = append(seededIDs, run.ID)
}
handler.runner.wg.Wait()
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""})
snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second)
defer cancelSnapshot()
seen := map[int64]bool{}
var ack *protocol.Message
for ack == nil || len(seen) < seeded {
var message protocol.Message
if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil {
t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err)
}
switch message.Type {
case protocol.RunEvent:
if message.RunEvent != nil {
seen[message.RunEvent.RunID] = true
}
case protocol.Subscribe:
got := message
ack = &got
// The snapshot precedes the ack: when the ack is read every snapshot
// frame must already have been, or a frame was reordered after it.
if len(seen) != seeded {
t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded)
}
}
}
if ack.ID != "snap" {
t.Fatalf("subscribe ack = %+v", *ack)
}
for _, id := range seededIDs {
if !seen[id] {
t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded)
}
}
// The subscription must outlive its own snapshot: a run started after it must
// still deliver a delta, through to its terminal frame.
late := eventWorkspace(t)
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}})
deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second)
defer cancelDelta()
var terminal *protocol.RunEventFrame
for terminal == nil {
var message protocol.Message
if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil {
t.Fatalf("no terminal delta after the snapshot: %v", err)
}
if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late {
continue
}
switch message.RunEvent.Status {
case "succeeded", "failed", "canceled":
terminal = message.RunEvent
}
}
if terminal.RunID == 0 || terminal.Outcome != "success" {
t.Fatalf("late terminal frame = %+v", terminal)
}
}
// TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's
// identity from an earlier transition when a later delta omits it, so the merged
// frame a client actually receives still carries what its row needs.
func TestMergeRunEventFrameKeepsIdentity(t *testing.T) {
into := &protocol.RunEventFrame{RunID: 1, PipelineID: 4, Name: "release", Title: "Ship it", Version: 3, Status: protocol.RunStatusQueued}
mergeRunEventFrame(into, &protocol.RunEventFrame{RunID: 1, Status: protocol.RunStatusRunning, CurrentStage: "build"})
if into.PipelineID != 4 || into.Name != "release" || into.Title != "Ship it" || into.Version != 3 || into.Status != protocol.RunStatusRunning {
t.Fatalf("merged identity lost: %+v", into)
}
}
// assertNoRunEvent reads for d, ignoring responses, and fails if any run_event
// frame arrives.
func assertNoRunEvent(t *testing.T, client *runEventClient, d time.Duration) {
t.Helper()
readCtx, cancel := context.WithTimeout(client.ctx, d)
defer cancel()
for {
var message protocol.Message
if err := wsjson.Read(readCtx, client.conn, &message); err != nil {
return
}
if message.Type == protocol.RunEvent {
t.Fatalf("client received an unexpected run_event: %+v", message.RunEvent)
}
}
}
// TestRunEventUnsubscribeAndNonSubscriberReceiveNothing proves an unsubscribed
// client and a client that never subscribed receive no run_event frames.
func TestRunEventUnsubscribeAndNonSubscriberReceiveNothing(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
subscribed := dialRunEvents(t, ctx, handler)
unsubscribedWorkspace := eventWorkspace(t)
subscribed.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: unsubscribedWorkspace})
if ack := subscribed.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
subscribed.send(protocol.Message{Type: protocol.Unsubscribe, ID: "unsub"})
if ack := subscribed.read(); ack.Type != protocol.Unsubscribe || ack.ID != "unsub" {
t.Fatalf("unsubscribe ack = %+v", ack)
}
subscribed.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-a", PipelineID: ids[0], Version: 1, RequestKey: "after-unsub", Workspace: unsubscribedWorkspace, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, subscribed, 400*time.Millisecond)
stranger := dialRunEvents(t, ctx, handler)
strangerWorkspace := eventWorkspace(t)
stranger.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-b", PipelineID: ids[0], Version: 1, RequestKey: "never-subscribed", Workspace: strangerWorkspace, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, stranger, 400*time.Millisecond)
}
// TestRunEventWorkspaceIsolation proves a subscriber to one workspace never
// receives a delta for a run in another workspace, and still receives its own.
func TestRunEventWorkspaceIsolation(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
watched := eventWorkspace(t)
other := eventWorkspace(t)
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: watched})
if ack := client.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
// The watched workspace still delivers its own run's terminal frame.
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "mine", PipelineID: ids[0], Version: 1, RequestKey: "watched-ws", Workspace: watched, Input: map[string]any{"prompt": "test"}})
var terminal *protocol.RunEventFrame
var runID int64
for terminal == nil {
message := client.read()
if message.Type == protocol.RunStarted && message.RunDetail != nil {
runID = message.RunDetail.ID
}
if message.Type == protocol.RunEvent && message.RunEvent != nil &&
(message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled") {
terminal = message.RunEvent
}
}
if runID == 0 || terminal.RunID != runID || terminal.Workspace != watched {
t.Fatalf("watched run terminal = %+v (run=%d), want workspace %q", terminal, runID, watched)
}
// A run in another workspace must not leak. This assertion reads until a
// bounded deadline, and that deadline closes the connection, so it runs last.
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "other", PipelineID: ids[0], Version: 1, RequestKey: "other-ws", Workspace: other, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, client, 400*time.Millisecond)
}
// TestRunEventAllWorkspacesSubscription proves the blank ("any workspace")
// subscribe scope — the one the app now asks for because its runs list is
// server-wide — delivers a run from every workspace, while an explicit
// workspace scope stays isolated (TestRunEventWorkspaceIsolation). It is what
// lets the app see a run executing in a lane worktree, a different workspace
// from the one the operator configured.
func TestRunEventAllWorkspacesSubscription(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
first := eventWorkspace(t)
second := eventWorkspace(t)
client := dialRunEvents(t, ctx, handler)
// The blank workspace is the "every workspace" scope; no run_ids narrows it.
client.send(protocol.Message{Type: protocol.Subscribe, ID: "all", Workspace: ""})
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "all" {
t.Fatalf("subscribe ack = %+v", ack)
}
// Subscribe first, then start a run in each workspace, so both are delivered
// as live deltas rather than only in the snapshot.
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "a", PipelineID: ids[0], Version: 1, RequestKey: "all-ws-a", Workspace: first, Input: map[string]any{"prompt": "test"}})
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "b", PipelineID: ids[0], Version: 1, RequestKey: "all-ws-b", Workspace: second, Input: map[string]any{"prompt": "test"}})
terminals := map[string]*protocol.RunEventFrame{}
deadline := time.Now().Add(8 * time.Second)
for len(terminals) < 2 && time.Now().Before(deadline) {
message := client.read()
if message.Type != protocol.RunEvent || message.RunEvent == nil {
continue
}
frame := message.RunEvent
if frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled" {
terminals[frame.Workspace] = frame
}
}
if terminals[first] == nil || terminals[second] == nil {
t.Fatalf("blank-scope subscription missed a workspace: got %v", terminals)
}
}
// TestRunEventCoalescingBoundsFrameCount records how many run_event frames a
// typical multi-stage run produces and asserts the count stays bounded: rapid
// transitions coalesce, and the terminal frame is never dropped.
func TestRunEventCoalescingBoundsFrameCount(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stages := []string{"a", "b", "c"}
_, handler, ids := eventStore(t, ctx, stages...)
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
if ack := client.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "coalesce", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
frames := 0
var terminal *protocol.RunEventFrame
for terminal == nil {
message := client.read()
if message.Type != protocol.RunEvent || message.RunEvent == nil {
continue
}
frames++
if message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled" {
terminal = message.RunEvent
}
}
bound := 2*len(stages) + 4
if frames > bound {
t.Fatalf("run produced %d run_event frames for %d stages, want at most %d", frames, len(stages), bound)
}
if terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.CompletedSteps != len(stages) {
t.Fatalf("terminal frame = %+v", terminal)
}
t.Logf("run_event frames for a %d-stage run: %d (bound %d)", len(stages), frames, bound)
}