Files
pipelines/internal/server/run_event_websocket_test.go
T

494 lines
21 KiB
Go

package server
import (
"context"
"fmt"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
// runEventClient is a thin websocket client for the run-event tests: it dials
// /connect, consumes the unsolicited welcome, and reads/writes protocol
// envelopes.
type runEventClient struct {
t *testing.T
ctx context.Context
conn *websocket.Conn
}
func dialRunEvents(t *testing.T, ctx context.Context, handler *runHandler) *runEventClient {
t.Helper()
srv := httptest.NewServer(handler)
t.Cleanup(srv.Close)
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { conn.CloseNow() })
var welcome protocol.Message
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
t.Fatal(err)
}
return &runEventClient{t: t, ctx: ctx, conn: conn}
}
func (c *runEventClient) send(message protocol.Message) {
c.t.Helper()
if err := wsjson.Write(c.ctx, c.conn, message); err != nil {
c.t.Fatal(err)
}
}
func (c *runEventClient) read() protocol.Message {
c.t.Helper()
var message protocol.Message
if err := wsjson.Read(c.ctx, c.conn, &message); err != nil {
c.t.Fatal(err)
}
return message
}
// eventStore starts a fresh store holding one pipeline template and a handler
// wired to it, with the execute seam returning a deterministic success.
func eventStore(t *testing.T, ctx context.Context, stages ...string) (*pipeline.Store, *runHandler, []int64) {
t.Helper()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "events.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { store.Close() })
if _, err := store.Submit(ctx, "events", pipeline.Template(stages, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
var ids []int64
for _, item := range items {
ids = append(ids, item.ID)
}
handler := Handler(ctx, store).(*runHandler)
t.Cleanup(func() { WaitRuns(handler) })
return store, handler, ids
}
// eventWorkspace returns a temporary directory canonicalised the same way the
// runner canonicalises a run workspace (EvalSymlinks). A subscription must use
// this resolved path to match: t.TempDir() alone is a symlinked /var path on
// macOS and would never equal the run's /private/var workspace, so no frame
// would be delivered.
func eventWorkspace(t *testing.T) string {
t.Helper()
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
return dir
}
// TestRunEventWebSocketPushesTransitions starts a run, subscribes to its
// workspace, and proves the client observes the created, queued->running /
// step-started, step-finished (with the right outcome and target) and terminal
// frames, all carrying the run id.
func TestRunEventWebSocketPushesTransitions(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
workspace := eventWorkspace(t)
release := make(chan struct{})
var releaseOnce sync.Once
handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
var runID int64
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
for terminal == nil {
message := client.read()
switch message.Type {
case protocol.RunStarted:
if message.RunDetail == nil {
t.Fatal("run_started without detail")
}
runID = message.RunDetail.ID
case protocol.RunEvent:
frame := message.RunEvent
if frame == nil {
t.Fatal("run_event without payload")
}
switch {
case frame.Step == nil && frame.Status == protocol.RunStatusQueued:
created = frame
case frame.Step != nil && frame.Step.Status == protocol.StepStatusRunning:
if stepStarted == nil {
stepStarted = frame
releaseOnce.Do(func() { close(release) })
}
case frame.Step != nil && frame.Step.Status == "completed":
stepFinished = frame
case frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled":
terminal = frame
}
}
}
if runID == 0 {
t.Fatal("run_started never carried the run id")
}
if created == nil || created.RunID != runID || created.Step != nil || created.CreatedAt == "" {
t.Fatalf("created frame = %+v", created)
}
if stepStarted == nil || stepStarted.RunID != runID || stepStarted.Status != protocol.RunStatusRunning || stepStarted.CurrentStage != "one" || stepStarted.Step.Sequence != 1 || stepStarted.Step.Stage != "one" {
t.Fatalf("step-started frame = %+v", stepStarted)
}
// Every frame carries the run's pipeline identity and stored title, so a
// client that has never held the run can build its list row from the frame.
for name, frame := range map[string]*protocol.RunEventFrame{"created": created, "step-started": stepStarted} {
if frame.PipelineID != ids[0] || frame.Name == "" || frame.Title == "" || frame.Version != 1 {
t.Fatalf("%s frame identity = %+v, want pipeline %d with name/title and version 1", name, frame, ids[0])
}
}
if stepFinished == nil || stepFinished.RunID != runID || stepFinished.Step.Status != "completed" || stepFinished.Step.Outcome != "success" || stepFinished.Step.Target != protocol.SuccessEnd || stepFinished.CompletedSteps != 1 {
t.Fatalf("step-finished frame = %+v", stepFinished)
}
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
t.Fatalf("terminal frame = %+v", terminal)
}
for _, frame := range []*protocol.RunEventFrame{created, stepStarted, stepFinished, terminal} {
if frame.NotifyMode != "off" {
t.Fatalf("silent mode lost in event: %+v", frame)
}
}
}
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
// already-existing run receives its current state as a run_event frame before
// the subscribe ack, so a reconnecting client re-syncs without polling.
func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
handler.runner.wg.Wait()
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: workspace})
first := client.read()
if first.Type != protocol.RunEvent || first.RunEvent == nil {
t.Fatalf("first frame after subscribe = %+v, want a run_event snapshot", first)
}
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
t.Fatalf("snapshot frame = %+v", first.RunEvent)
}
if first.RunEvent.NotifyMode != "off" {
t.Fatalf("silent mode lost in snapshot: %+v", first.RunEvent)
}
// A snapshot for a run the client never saw also carries the identity its row
// needs, so a fresh subscribe renders it without a second read.
if first.RunEvent.PipelineID != ids[0] || first.RunEvent.Name == "" || first.RunEvent.Version != 1 {
t.Fatalf("snapshot identity = %+v, want pipeline %d version 1", first.RunEvent, ids[0])
}
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "snap" {
t.Fatalf("subscribe ack = %+v", ack)
}
}
// TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose
// snapshot holds more runs than the subscription's bounded buffer delivers the
// snapshot in full, then the ack, then a live delta for a run started
// afterwards. Before the snapshot fix, the subscription wrote its own snapshot
// through the bounded channel, so sendMessage's drop-a-slow-subscriber policy
// fired on the snapshot itself: the subscription closed part-way through, the
// client saw at most runEventSubscriberBuffer frames, and it never received the
// ack or a single later delta.
func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
const seeded = runEventSubscriberBuffer + 8
// Seed more finished runs than the buffer holds, each in its own workspace so
// no two runs share a tree, then wait for every one to settle. A blank-scope
// subscribe sees them all.
seededIDs := make([]int64, 0, seeded)
for i := 0; i < seeded; i++ {
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
seededIDs = append(seededIDs, run.ID)
}
handler.runner.wg.Wait()
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""})
snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second)
defer cancelSnapshot()
seen := map[int64]bool{}
var ack *protocol.Message
for ack == nil || len(seen) < seeded {
var message protocol.Message
if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil {
t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err)
}
switch message.Type {
case protocol.RunEvent:
if message.RunEvent != nil {
seen[message.RunEvent.RunID] = true
}
case protocol.Subscribe:
got := message
ack = &got
// The snapshot precedes the ack: when the ack is read every snapshot
// frame must already have been, or a frame was reordered after it.
if len(seen) != seeded {
t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded)
}
}
}
if ack.ID != "snap" {
t.Fatalf("subscribe ack = %+v", *ack)
}
for _, id := range seededIDs {
if !seen[id] {
t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded)
}
}
// The subscription must outlive its own snapshot: a run started after it must
// still deliver a delta, through to its terminal frame.
late := eventWorkspace(t)
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}})
deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second)
defer cancelDelta()
var terminal *protocol.RunEventFrame
for terminal == nil {
var message protocol.Message
if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil {
t.Fatalf("no terminal delta after the snapshot: %v", err)
}
if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late {
continue
}
switch message.RunEvent.Status {
case "succeeded", "failed", "canceled":
terminal = message.RunEvent
}
}
if terminal.RunID == 0 || terminal.Outcome != "success" {
t.Fatalf("late terminal frame = %+v", terminal)
}
}
// TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's
// identity from an earlier transition when a later delta omits it, so the merged
// frame a client actually receives still carries what its row needs.
func TestMergeRunEventFrameKeepsIdentity(t *testing.T) {
into := &protocol.RunEventFrame{RunID: 1, PipelineID: 4, Name: "release", Title: "Ship it", Version: 3, Status: protocol.RunStatusQueued}
mergeRunEventFrame(into, &protocol.RunEventFrame{RunID: 1, Status: protocol.RunStatusRunning, CurrentStage: "build"})
if into.PipelineID != 4 || into.Name != "release" || into.Title != "Ship it" || into.Version != 3 || into.Status != protocol.RunStatusRunning {
t.Fatalf("merged identity lost: %+v", into)
}
}
// assertNoRunEvent reads for d, ignoring responses, and fails if any run_event
// frame arrives.
func assertNoRunEvent(t *testing.T, client *runEventClient, d time.Duration) {
t.Helper()
readCtx, cancel := context.WithTimeout(client.ctx, d)
defer cancel()
for {
var message protocol.Message
if err := wsjson.Read(readCtx, client.conn, &message); err != nil {
return
}
if message.Type == protocol.RunEvent {
t.Fatalf("client received an unexpected run_event: %+v", message.RunEvent)
}
}
}
// TestRunEventUnsubscribeAndNonSubscriberReceiveNothing proves an unsubscribed
// client and a client that never subscribed receive no run_event frames.
func TestRunEventUnsubscribeAndNonSubscriberReceiveNothing(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
subscribed := dialRunEvents(t, ctx, handler)
unsubscribedWorkspace := eventWorkspace(t)
subscribed.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: unsubscribedWorkspace})
if ack := subscribed.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
subscribed.send(protocol.Message{Type: protocol.Unsubscribe, ID: "unsub"})
if ack := subscribed.read(); ack.Type != protocol.Unsubscribe || ack.ID != "unsub" {
t.Fatalf("unsubscribe ack = %+v", ack)
}
subscribed.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-a", PipelineID: ids[0], Version: 1, RequestKey: "after-unsub", Workspace: unsubscribedWorkspace, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, subscribed, 400*time.Millisecond)
stranger := dialRunEvents(t, ctx, handler)
strangerWorkspace := eventWorkspace(t)
stranger.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-b", PipelineID: ids[0], Version: 1, RequestKey: "never-subscribed", Workspace: strangerWorkspace, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, stranger, 400*time.Millisecond)
}
// TestRunEventWorkspaceIsolation proves a subscriber to one workspace never
// receives a delta for a run in another workspace, and still receives its own.
func TestRunEventWorkspaceIsolation(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
watched := eventWorkspace(t)
other := eventWorkspace(t)
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: watched})
if ack := client.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
// The watched workspace still delivers its own run's terminal frame.
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "mine", PipelineID: ids[0], Version: 1, RequestKey: "watched-ws", Workspace: watched, Input: map[string]any{"prompt": "test"}})
var terminal *protocol.RunEventFrame
var runID int64
for terminal == nil {
message := client.read()
if message.Type == protocol.RunStarted && message.RunDetail != nil {
runID = message.RunDetail.ID
}
if message.Type == protocol.RunEvent && message.RunEvent != nil &&
(message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled") {
terminal = message.RunEvent
}
}
if runID == 0 || terminal.RunID != runID || terminal.Workspace != watched {
t.Fatalf("watched run terminal = %+v (run=%d), want workspace %q", terminal, runID, watched)
}
// A run in another workspace must not leak. This assertion reads until a
// bounded deadline, and that deadline closes the connection, so it runs last.
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "other", PipelineID: ids[0], Version: 1, RequestKey: "other-ws", Workspace: other, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, client, 400*time.Millisecond)
}
// TestRunEventAllWorkspacesSubscription proves the blank ("any workspace")
// subscribe scope — the one the app now asks for because its runs list is
// server-wide — delivers a run from every workspace, while an explicit
// workspace scope stays isolated (TestRunEventWorkspaceIsolation). It is what
// lets the app see a run executing in a lane worktree, a different workspace
// from the one the operator configured.
func TestRunEventAllWorkspacesSubscription(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
first := eventWorkspace(t)
second := eventWorkspace(t)
client := dialRunEvents(t, ctx, handler)
// The blank workspace is the "every workspace" scope; no run_ids narrows it.
client.send(protocol.Message{Type: protocol.Subscribe, ID: "all", Workspace: ""})
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "all" {
t.Fatalf("subscribe ack = %+v", ack)
}
// Subscribe first, then start a run in each workspace, so both are delivered
// as live deltas rather than only in the snapshot.
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "a", PipelineID: ids[0], Version: 1, RequestKey: "all-ws-a", Workspace: first, Input: map[string]any{"prompt": "test"}})
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "b", PipelineID: ids[0], Version: 1, RequestKey: "all-ws-b", Workspace: second, Input: map[string]any{"prompt": "test"}})
terminals := map[string]*protocol.RunEventFrame{}
deadline := time.Now().Add(8 * time.Second)
for len(terminals) < 2 && time.Now().Before(deadline) {
message := client.read()
if message.Type != protocol.RunEvent || message.RunEvent == nil {
continue
}
frame := message.RunEvent
if frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled" {
terminals[frame.Workspace] = frame
}
}
if terminals[first] == nil || terminals[second] == nil {
t.Fatalf("blank-scope subscription missed a workspace: got %v", terminals)
}
}
// TestRunEventCoalescingBoundsFrameCount records how many run_event frames a
// typical multi-stage run produces and asserts the count stays bounded: rapid
// transitions coalesce, and the terminal frame is never dropped.
func TestRunEventCoalescingBoundsFrameCount(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stages := []string{"a", "b", "c"}
_, handler, ids := eventStore(t, ctx, stages...)
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
if ack := client.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "coalesce", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
frames := 0
var terminal *protocol.RunEventFrame
for terminal == nil {
message := client.read()
if message.Type != protocol.RunEvent || message.RunEvent == nil {
continue
}
frames++
if message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled" {
terminal = message.RunEvent
}
}
bound := 2*len(stages) + 4
if frames > bound {
t.Fatalf("run produced %d run_event frames for %d stages, want at most %d", frames, len(stages), bound)
}
if terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.CompletedSteps != len(stages) {
t.Fatalf("terminal frame = %+v", terminal)
}
t.Logf("run_event frames for a %d-stage run: %d (bound %d)", len(stages), frames, bound)
}