package server import ( "context" "fmt" "net/http/httptest" "path/filepath" "strings" "sync" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" ) // runEventClient is a thin websocket client for the run-event tests: it dials // /connect, consumes the unsolicited welcome, and reads/writes protocol // envelopes. type runEventClient struct { t *testing.T ctx context.Context conn *websocket.Conn } func dialRunEvents(t *testing.T, ctx context.Context, handler *runHandler) *runEventClient { t.Helper() srv := httptest.NewServer(handler) t.Cleanup(srv.Close) conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) if err != nil { t.Fatal(err) } t.Cleanup(func() { conn.CloseNow() }) var welcome protocol.Message if err := wsjson.Read(ctx, conn, &welcome); err != nil { t.Fatal(err) } return &runEventClient{t: t, ctx: ctx, conn: conn} } func (c *runEventClient) send(message protocol.Message) { c.t.Helper() if err := wsjson.Write(c.ctx, c.conn, message); err != nil { c.t.Fatal(err) } } func (c *runEventClient) read() protocol.Message { c.t.Helper() var message protocol.Message if err := wsjson.Read(c.ctx, c.conn, &message); err != nil { c.t.Fatal(err) } return message } // eventStore starts a fresh store holding one pipeline template and a handler // wired to it, with the execute seam returning a deterministic success. func eventStore(t *testing.T, ctx context.Context, stages ...string) (*pipeline.Store, *runHandler, []int64) { t.Helper() store, err := pipeline.Open(filepath.Join(t.TempDir(), "events.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { store.Close() }) if _, err := store.Submit(ctx, "events", pipeline.Template(stages, true)); err != nil { t.Fatal(err) } items, err := store.List(ctx) if err != nil { t.Fatal(err) } var ids []int64 for _, item := range items { ids = append(ids, item.ID) } handler := Handler(ctx, store).(*runHandler) t.Cleanup(func() { WaitRuns(handler) }) return store, handler, ids } // eventWorkspace returns a temporary directory canonicalised the same way the // runner canonicalises a run workspace (EvalSymlinks). A subscription must use // this resolved path to match: t.TempDir() alone is a symlinked /var path on // macOS and would never equal the run's /private/var workspace, so no frame // would be delivered. func eventWorkspace(t *testing.T) string { t.Helper() dir, err := filepath.EvalSymlinks(t.TempDir()) if err != nil { t.Fatal(err) } return dir } // TestRunEventWebSocketPushesTransitions starts a run, subscribes to its // workspace, and proves the client observes the created, queued->running / // step-started, step-finished (with the right outcome and target) and terminal // frames, all carrying the run id. func TestRunEventWebSocketPushesTransitions(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() _, handler, ids := eventStore(t, ctx, "one") workspace := eventWorkspace(t) release := make(chan struct{}) var releaseOnce sync.Once handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { select { case <-release: case <-ctx.Done(): return stage.Result{}, ctx.Err() } return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } client := dialRunEvents(t, ctx, handler) client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace}) if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" { t.Fatalf("subscribe ack = %+v", ack) } client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}}) var runID int64 var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame for terminal == nil { message := client.read() switch message.Type { case protocol.RunStarted: if message.RunDetail == nil { t.Fatal("run_started without detail") } runID = message.RunDetail.ID case protocol.RunEvent: frame := message.RunEvent if frame == nil { t.Fatal("run_event without payload") } switch { case frame.Step == nil && frame.Status == protocol.RunStatusQueued: created = frame case frame.Step != nil && frame.Step.Status == protocol.StepStatusRunning: if stepStarted == nil { stepStarted = frame releaseOnce.Do(func() { close(release) }) } case frame.Step != nil && frame.Step.Status == "completed": stepFinished = frame case frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled": terminal = frame } } } if runID == 0 { t.Fatal("run_started never carried the run id") } if created == nil || created.RunID != runID || created.Step != nil || created.CreatedAt == "" { t.Fatalf("created frame = %+v", created) } if stepStarted == nil || stepStarted.RunID != runID || stepStarted.Status != protocol.RunStatusRunning || stepStarted.CurrentStage != "one" || stepStarted.Step.Sequence != 1 || stepStarted.Step.Stage != "one" { t.Fatalf("step-started frame = %+v", stepStarted) } // Every frame carries the run's pipeline identity and stored title, so a // client that has never held the run can build its list row from the frame. for name, frame := range map[string]*protocol.RunEventFrame{"created": created, "step-started": stepStarted} { if frame.PipelineID != ids[0] || frame.Name == "" || frame.Title == "" || frame.Version != 1 { t.Fatalf("%s frame identity = %+v, want pipeline %d with name/title and version 1", name, frame, ids[0]) } } if stepFinished == nil || stepFinished.RunID != runID || stepFinished.Step.Status != "completed" || stepFinished.Step.Outcome != "success" || stepFinished.Step.Target != protocol.SuccessEnd || stepFinished.CompletedSteps != 1 { t.Fatalf("step-finished frame = %+v", stepFinished) } if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" { t.Fatalf("terminal frame = %+v", terminal) } for _, frame := range []*protocol.RunEventFrame{created, stepStarted, stepFinished, terminal} { if frame.NotifyMode != "off" { t.Fatalf("silent mode lost in event: %+v", frame) } } } // TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an // already-existing run receives its current state as a run_event frame before // the subscribe ack, so a reconnecting client re-syncs without polling. func TestRunEventSubscribeSendsSnapshot(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _, handler, ids := eventStore(t, ctx, "one") handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } workspace := eventWorkspace(t) run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "test"}}) if err != nil { t.Fatal(err) } handler.runner.wg.Wait() client := dialRunEvents(t, ctx, handler) client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: workspace}) first := client.read() if first.Type != protocol.RunEvent || first.RunEvent == nil { t.Fatalf("first frame after subscribe = %+v, want a run_event snapshot", first) } if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" { t.Fatalf("snapshot frame = %+v", first.RunEvent) } if first.RunEvent.NotifyMode != "off" { t.Fatalf("silent mode lost in snapshot: %+v", first.RunEvent) } // A snapshot for a run the client never saw also carries the identity its row // needs, so a fresh subscribe renders it without a second read. if first.RunEvent.PipelineID != ids[0] || first.RunEvent.Name == "" || first.RunEvent.Version != 1 { t.Fatalf("snapshot identity = %+v, want pipeline %d version 1", first.RunEvent, ids[0]) } if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "snap" { t.Fatalf("subscribe ack = %+v", ack) } } // TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose // snapshot holds more runs than the subscription's bounded buffer delivers the // snapshot in full, then the ack, then a live delta for a run started // afterwards. Before the snapshot fix, the subscription wrote its own snapshot // through the bounded channel, so sendMessage's drop-a-slow-subscriber policy // fired on the snapshot itself: the subscription closed part-way through, the // client saw at most runEventSubscriberBuffer frames, and it never received the // ack or a single later delta. func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, handler, ids := eventStore(t, ctx, "one") handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } const seeded = runEventSubscriberBuffer + 8 // Seed more finished runs than the buffer holds, each in its own workspace so // no two runs share a tree, then wait for every one to settle. A blank-scope // subscribe sees them all. seededIDs := make([]int64, 0, seeded) for i := 0; i < seeded; i++ { run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}}) if err != nil { t.Fatal(err) } seededIDs = append(seededIDs, run.ID) } handler.runner.wg.Wait() client := dialRunEvents(t, ctx, handler) client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""}) snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second) defer cancelSnapshot() seen := map[int64]bool{} var ack *protocol.Message for ack == nil || len(seen) < seeded { var message protocol.Message if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil { t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err) } switch message.Type { case protocol.RunEvent: if message.RunEvent != nil { seen[message.RunEvent.RunID] = true } case protocol.Subscribe: got := message ack = &got // The snapshot precedes the ack: when the ack is read every snapshot // frame must already have been, or a frame was reordered after it. if len(seen) != seeded { t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded) } } } if ack.ID != "snap" { t.Fatalf("subscribe ack = %+v", *ack) } for _, id := range seededIDs { if !seen[id] { t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded) } } // The subscription must outlive its own snapshot: a run started after it must // still deliver a delta, through to its terminal frame. late := eventWorkspace(t) client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}}) deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second) defer cancelDelta() var terminal *protocol.RunEventFrame for terminal == nil { var message protocol.Message if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil { t.Fatalf("no terminal delta after the snapshot: %v", err) } if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late { continue } switch message.RunEvent.Status { case "succeeded", "failed", "canceled": terminal = message.RunEvent } } if terminal.RunID == 0 || terminal.Outcome != "success" { t.Fatalf("late terminal frame = %+v", terminal) } } // TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's // identity from an earlier transition when a later delta omits it, so the merged // frame a client actually receives still carries what its row needs. func TestMergeRunEventFrameKeepsIdentity(t *testing.T) { into := &protocol.RunEventFrame{RunID: 1, PipelineID: 4, Name: "release", Title: "Ship it", Version: 3, Status: protocol.RunStatusQueued} mergeRunEventFrame(into, &protocol.RunEventFrame{RunID: 1, Status: protocol.RunStatusRunning, CurrentStage: "build"}) if into.PipelineID != 4 || into.Name != "release" || into.Title != "Ship it" || into.Version != 3 || into.Status != protocol.RunStatusRunning { t.Fatalf("merged identity lost: %+v", into) } } // assertNoRunEvent reads for d, ignoring responses, and fails if any run_event // frame arrives. func assertNoRunEvent(t *testing.T, client *runEventClient, d time.Duration) { t.Helper() readCtx, cancel := context.WithTimeout(client.ctx, d) defer cancel() for { var message protocol.Message if err := wsjson.Read(readCtx, client.conn, &message); err != nil { return } if message.Type == protocol.RunEvent { t.Fatalf("client received an unexpected run_event: %+v", message.RunEvent) } } } // TestRunEventUnsubscribeAndNonSubscriberReceiveNothing proves an unsubscribed // client and a client that never subscribed receive no run_event frames. func TestRunEventUnsubscribeAndNonSubscriberReceiveNothing(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _, handler, ids := eventStore(t, ctx, "one") handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } subscribed := dialRunEvents(t, ctx, handler) unsubscribedWorkspace := eventWorkspace(t) subscribed.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: unsubscribedWorkspace}) if ack := subscribed.read(); ack.Type != protocol.Subscribe { t.Fatalf("subscribe ack = %+v", ack) } subscribed.send(protocol.Message{Type: protocol.Unsubscribe, ID: "unsub"}) if ack := subscribed.read(); ack.Type != protocol.Unsubscribe || ack.ID != "unsub" { t.Fatalf("unsubscribe ack = %+v", ack) } subscribed.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-a", PipelineID: ids[0], Version: 1, RequestKey: "after-unsub", Workspace: unsubscribedWorkspace, Input: map[string]any{"prompt": "test"}}) handler.runner.wg.Wait() assertNoRunEvent(t, subscribed, 400*time.Millisecond) stranger := dialRunEvents(t, ctx, handler) strangerWorkspace := eventWorkspace(t) stranger.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-b", PipelineID: ids[0], Version: 1, RequestKey: "never-subscribed", Workspace: strangerWorkspace, Input: map[string]any{"prompt": "test"}}) handler.runner.wg.Wait() assertNoRunEvent(t, stranger, 400*time.Millisecond) } // TestRunEventWorkspaceIsolation proves a subscriber to one workspace never // receives a delta for a run in another workspace, and still receives its own. func TestRunEventWorkspaceIsolation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _, handler, ids := eventStore(t, ctx, "one") handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } watched := eventWorkspace(t) other := eventWorkspace(t) client := dialRunEvents(t, ctx, handler) client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: watched}) if ack := client.read(); ack.Type != protocol.Subscribe { t.Fatalf("subscribe ack = %+v", ack) } // The watched workspace still delivers its own run's terminal frame. client.send(protocol.Message{Type: protocol.RunPipeline, ID: "mine", PipelineID: ids[0], Version: 1, RequestKey: "watched-ws", Workspace: watched, Input: map[string]any{"prompt": "test"}}) var terminal *protocol.RunEventFrame var runID int64 for terminal == nil { message := client.read() if message.Type == protocol.RunStarted && message.RunDetail != nil { runID = message.RunDetail.ID } if message.Type == protocol.RunEvent && message.RunEvent != nil && (message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled") { terminal = message.RunEvent } } if runID == 0 || terminal.RunID != runID || terminal.Workspace != watched { t.Fatalf("watched run terminal = %+v (run=%d), want workspace %q", terminal, runID, watched) } // A run in another workspace must not leak. This assertion reads until a // bounded deadline, and that deadline closes the connection, so it runs last. client.send(protocol.Message{Type: protocol.RunPipeline, ID: "other", PipelineID: ids[0], Version: 1, RequestKey: "other-ws", Workspace: other, Input: map[string]any{"prompt": "test"}}) handler.runner.wg.Wait() assertNoRunEvent(t, client, 400*time.Millisecond) } // TestRunEventAllWorkspacesSubscription proves the blank ("any workspace") // subscribe scope — the one the app now asks for because its runs list is // server-wide — delivers a run from every workspace, while an explicit // workspace scope stays isolated (TestRunEventWorkspaceIsolation). It is what // lets the app see a run executing in a lane worktree, a different workspace // from the one the operator configured. func TestRunEventAllWorkspacesSubscription(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() _, handler, ids := eventStore(t, ctx, "one") handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } first := eventWorkspace(t) second := eventWorkspace(t) client := dialRunEvents(t, ctx, handler) // The blank workspace is the "every workspace" scope; no run_ids narrows it. client.send(protocol.Message{Type: protocol.Subscribe, ID: "all", Workspace: ""}) if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "all" { t.Fatalf("subscribe ack = %+v", ack) } // Subscribe first, then start a run in each workspace, so both are delivered // as live deltas rather than only in the snapshot. client.send(protocol.Message{Type: protocol.RunPipeline, ID: "a", PipelineID: ids[0], Version: 1, RequestKey: "all-ws-a", Workspace: first, Input: map[string]any{"prompt": "test"}}) client.send(protocol.Message{Type: protocol.RunPipeline, ID: "b", PipelineID: ids[0], Version: 1, RequestKey: "all-ws-b", Workspace: second, Input: map[string]any{"prompt": "test"}}) terminals := map[string]*protocol.RunEventFrame{} deadline := time.Now().Add(8 * time.Second) for len(terminals) < 2 && time.Now().Before(deadline) { message := client.read() if message.Type != protocol.RunEvent || message.RunEvent == nil { continue } frame := message.RunEvent if frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled" { terminals[frame.Workspace] = frame } } if terminals[first] == nil || terminals[second] == nil { t.Fatalf("blank-scope subscription missed a workspace: got %v", terminals) } } // TestRunEventCoalescingBoundsFrameCount records how many run_event frames a // typical multi-stage run produces and asserts the count stays bounded: rapid // transitions coalesce, and the terminal frame is never dropped. func TestRunEventCoalescingBoundsFrameCount(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() stages := []string{"a", "b", "c"} _, handler, ids := eventStore(t, ctx, stages...) handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } workspace := eventWorkspace(t) client := dialRunEvents(t, ctx, handler) client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace}) if ack := client.read(); ack.Type != protocol.Subscribe { t.Fatalf("subscribe ack = %+v", ack) } client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "coalesce", Workspace: workspace, Input: map[string]any{"prompt": "test"}}) frames := 0 var terminal *protocol.RunEventFrame for terminal == nil { message := client.read() if message.Type != protocol.RunEvent || message.RunEvent == nil { continue } frames++ if message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled" { terminal = message.RunEvent } } bound := 2*len(stages) + 4 if frames > bound { t.Fatalf("run produced %d run_event frames for %d stages, want at most %d", frames, len(stages), bound) } if terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.CompletedSteps != len(stages) { t.Fatalf("terminal frame = %+v", terminal) } t.Logf("run_event frames for a %d-stage run: %d (bound %d)", len(stages), frames, bound) }