cff6f4dfac
handleRunEvents seeded a new subscription by pushing every matching run through the subscription channel (runEventSubscriberBuffer = 32) and only then acked. sendMessage is a non-blocking send that closes the subscription when the channel is full, so a snapshot larger than the buffer dropped the subscription it was seeding: the remaining snapshot frames hit the closed-done early return, the ack was never sent, and no later delta could arrive. The connection stayed open, so the client could not tell it was no longer subscribed. This was the app's only subscription - AppViewModel subscribes with a blank workspace, matching every run - so the app's live state froze. The subscription now starts in a snapshot phase. The subscribing connection itself writes the snapshot frames via writeSnapshot (one direct write per frame, outside the bounded channel, so the drop policy cannot fire on them) and the ack, and ready then moves any deltas held meanwhile into the channel in order and starts the writer goroutine. sendMessage holds deltas in a bounded pending queue while snapshotting instead of closing, so the drop-a-slow-subscriber policy still applies, but only to deltas. Ordering is snapshot, ack, then deltas; a write failure or a listRuns error closes the subscription instead of leaving it registered. Numbers reported by this run: the probe against the deployed server saw blank-scope frames=2 ack=0 (scoped frames=1 ack=0) because the snapshot was ~252 frames into a 32-frame buffer; the scratch-server probe with the fix saw 40 snapshot frames, ack=1 and a delta after a run transition. Guarded now by TestRunEventSubscribeSnapshotLargerThanBuffer (seeds 40 finished runs, asserts every snapshot frame, the ack strictly after them, and a terminal delta for a run started after subscribing), which fails against the previous runevents.go. Checked for staleness: the packaged templates and the packaged authoring skill references internal/author/skill/references/ carry no run-event subscription contract (their "snapshot" mentions are the workspace-change snapshots in operations.md), and README carries none either, so none of them needs updating. No UI surface changed. Checks run before committing: gofmt -l . prints nothing, go build ./..., go vet ./..., go test ./... (all packages ok), go test -race ./internal/server (ok).
486 lines
21 KiB
Go
486 lines
21 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
)
|
|
|
|
// runEventClient is a thin websocket client for the run-event tests: it dials
|
|
// /connect, consumes the unsolicited welcome, and reads/writes protocol
|
|
// envelopes.
|
|
type runEventClient struct {
|
|
t *testing.T
|
|
ctx context.Context
|
|
conn *websocket.Conn
|
|
}
|
|
|
|
func dialRunEvents(t *testing.T, ctx context.Context, handler *runHandler) *runEventClient {
|
|
t.Helper()
|
|
srv := httptest.NewServer(handler)
|
|
t.Cleanup(srv.Close)
|
|
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { conn.CloseNow() })
|
|
var welcome protocol.Message
|
|
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return &runEventClient{t: t, ctx: ctx, conn: conn}
|
|
}
|
|
|
|
func (c *runEventClient) send(message protocol.Message) {
|
|
c.t.Helper()
|
|
if err := wsjson.Write(c.ctx, c.conn, message); err != nil {
|
|
c.t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (c *runEventClient) read() protocol.Message {
|
|
c.t.Helper()
|
|
var message protocol.Message
|
|
if err := wsjson.Read(c.ctx, c.conn, &message); err != nil {
|
|
c.t.Fatal(err)
|
|
}
|
|
return message
|
|
}
|
|
|
|
// eventStore starts a fresh store holding one pipeline template and a handler
|
|
// wired to it, with the execute seam returning a deterministic success.
|
|
func eventStore(t *testing.T, ctx context.Context, stages ...string) (*pipeline.Store, *runHandler, []int64) {
|
|
t.Helper()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "events.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { store.Close() })
|
|
if _, err := store.Submit(ctx, "events", pipeline.Template(stages, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var ids []int64
|
|
for _, item := range items {
|
|
ids = append(ids, item.ID)
|
|
}
|
|
handler := Handler(ctx, store).(*runHandler)
|
|
t.Cleanup(func() { WaitRuns(handler) })
|
|
return store, handler, ids
|
|
}
|
|
|
|
// eventWorkspace returns a temporary directory canonicalised the same way the
|
|
// runner canonicalises a run workspace (EvalSymlinks). A subscription must use
|
|
// this resolved path to match: t.TempDir() alone is a symlinked /var path on
|
|
// macOS and would never equal the run's /private/var workspace, so no frame
|
|
// would be delivered.
|
|
func eventWorkspace(t *testing.T) string {
|
|
t.Helper()
|
|
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return dir
|
|
}
|
|
|
|
// TestRunEventWebSocketPushesTransitions starts a run, subscribes to its
|
|
// workspace, and proves the client observes the created, queued->running /
|
|
// step-started, step-finished (with the right outcome and target) and terminal
|
|
// frames, all carrying the run id.
|
|
func TestRunEventWebSocketPushesTransitions(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
workspace := eventWorkspace(t)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
select {
|
|
case <-release:
|
|
case <-ctx.Done():
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
|
|
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
|
|
|
|
var runID int64
|
|
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
|
|
for terminal == nil {
|
|
message := client.read()
|
|
switch message.Type {
|
|
case protocol.RunStarted:
|
|
if message.RunDetail == nil {
|
|
t.Fatal("run_started without detail")
|
|
}
|
|
runID = message.RunDetail.ID
|
|
case protocol.RunEvent:
|
|
frame := message.RunEvent
|
|
if frame == nil {
|
|
t.Fatal("run_event without payload")
|
|
}
|
|
switch {
|
|
case frame.Step == nil && frame.Status == protocol.RunStatusQueued:
|
|
created = frame
|
|
case frame.Step != nil && frame.Step.Status == protocol.StepStatusRunning:
|
|
if stepStarted == nil {
|
|
stepStarted = frame
|
|
releaseOnce.Do(func() { close(release) })
|
|
}
|
|
case frame.Step != nil && frame.Step.Status == "completed":
|
|
stepFinished = frame
|
|
case frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled":
|
|
terminal = frame
|
|
}
|
|
}
|
|
}
|
|
if runID == 0 {
|
|
t.Fatal("run_started never carried the run id")
|
|
}
|
|
if created == nil || created.RunID != runID || created.Step != nil || created.CreatedAt == "" {
|
|
t.Fatalf("created frame = %+v", created)
|
|
}
|
|
if stepStarted == nil || stepStarted.RunID != runID || stepStarted.Status != protocol.RunStatusRunning || stepStarted.CurrentStage != "one" || stepStarted.Step.Sequence != 1 || stepStarted.Step.Stage != "one" {
|
|
t.Fatalf("step-started frame = %+v", stepStarted)
|
|
}
|
|
// Every frame carries the run's pipeline identity and stored title, so a
|
|
// client that has never held the run can build its list row from the frame.
|
|
for name, frame := range map[string]*protocol.RunEventFrame{"created": created, "step-started": stepStarted} {
|
|
if frame.PipelineID != ids[0] || frame.Name == "" || frame.Title == "" || frame.Version != 1 {
|
|
t.Fatalf("%s frame identity = %+v, want pipeline %d with name/title and version 1", name, frame, ids[0])
|
|
}
|
|
}
|
|
if stepFinished == nil || stepFinished.RunID != runID || stepFinished.Step.Status != "completed" || stepFinished.Step.Outcome != "success" || stepFinished.Step.Target != protocol.SuccessEnd || stepFinished.CompletedSteps != 1 {
|
|
t.Fatalf("step-finished frame = %+v", stepFinished)
|
|
}
|
|
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
|
|
t.Fatalf("terminal frame = %+v", terminal)
|
|
}
|
|
}
|
|
|
|
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
|
|
// already-existing run receives its current state as a run_event frame before
|
|
// the subscribe ack, so a reconnecting client re-syncs without polling.
|
|
func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
workspace := eventWorkspace(t)
|
|
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
handler.runner.wg.Wait()
|
|
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: workspace})
|
|
first := client.read()
|
|
if first.Type != protocol.RunEvent || first.RunEvent == nil {
|
|
t.Fatalf("first frame after subscribe = %+v, want a run_event snapshot", first)
|
|
}
|
|
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
|
|
t.Fatalf("snapshot frame = %+v", first.RunEvent)
|
|
}
|
|
// A snapshot for a run the client never saw also carries the identity its row
|
|
// needs, so a fresh subscribe renders it without a second read.
|
|
if first.RunEvent.PipelineID != ids[0] || first.RunEvent.Name == "" || first.RunEvent.Version != 1 {
|
|
t.Fatalf("snapshot identity = %+v, want pipeline %d version 1", first.RunEvent, ids[0])
|
|
}
|
|
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "snap" {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
}
|
|
|
|
// TestRunEventSubscribeSnapshotLargerThanBuffer proves a subscribe whose
|
|
// snapshot holds more runs than the subscription's bounded buffer delivers the
|
|
// snapshot in full, then the ack, then a live delta for a run started
|
|
// afterwards. Before the snapshot fix, the subscription wrote its own snapshot
|
|
// through the bounded channel, so sendMessage's drop-a-slow-subscriber policy
|
|
// fired on the snapshot itself: the subscription closed part-way through, the
|
|
// client saw at most runEventSubscriberBuffer frames, and it never received the
|
|
// ack or a single later delta.
|
|
func TestRunEventSubscribeSnapshotLargerThanBuffer(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
const seeded = runEventSubscriberBuffer + 8
|
|
// Seed more finished runs than the buffer holds, each in its own workspace so
|
|
// no two runs share a tree, then wait for every one to settle. A blank-scope
|
|
// subscribe sees them all.
|
|
seededIDs := make([]int64, 0, seeded)
|
|
for i := 0; i < seeded; i++ {
|
|
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: fmt.Sprintf("snapshot-%d", i), Workspace: eventWorkspace(t), Input: map[string]any{"prompt": "test"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seededIDs = append(seededIDs, run.ID)
|
|
}
|
|
handler.runner.wg.Wait()
|
|
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: ""})
|
|
snapshotCtx, cancelSnapshot := context.WithTimeout(ctx, 8*time.Second)
|
|
defer cancelSnapshot()
|
|
seen := map[int64]bool{}
|
|
var ack *protocol.Message
|
|
for ack == nil || len(seen) < seeded {
|
|
var message protocol.Message
|
|
if err := wsjson.Read(snapshotCtx, client.conn, &message); err != nil {
|
|
t.Fatalf("snapshot stopped after %d of %d frames (ack=%v): %v", len(seen), seeded, ack != nil, err)
|
|
}
|
|
switch message.Type {
|
|
case protocol.RunEvent:
|
|
if message.RunEvent != nil {
|
|
seen[message.RunEvent.RunID] = true
|
|
}
|
|
case protocol.Subscribe:
|
|
got := message
|
|
ack = &got
|
|
// The snapshot precedes the ack: when the ack is read every snapshot
|
|
// frame must already have been, or a frame was reordered after it.
|
|
if len(seen) != seeded {
|
|
t.Fatalf("ack arrived after %d of %d snapshot frames", len(seen), seeded)
|
|
}
|
|
}
|
|
}
|
|
if ack.ID != "snap" {
|
|
t.Fatalf("subscribe ack = %+v", *ack)
|
|
}
|
|
for _, id := range seededIDs {
|
|
if !seen[id] {
|
|
t.Fatalf("snapshot omitted run %d: got %d of %d frames", id, len(seen), seeded)
|
|
}
|
|
}
|
|
|
|
// The subscription must outlive its own snapshot: a run started after it must
|
|
// still deliver a delta, through to its terminal frame.
|
|
late := eventWorkspace(t)
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "late", PipelineID: ids[0], Version: 1, RequestKey: "snapshot-late", Workspace: late, Input: map[string]any{"prompt": "test"}})
|
|
deltaCtx, cancelDelta := context.WithTimeout(ctx, 8*time.Second)
|
|
defer cancelDelta()
|
|
var terminal *protocol.RunEventFrame
|
|
for terminal == nil {
|
|
var message protocol.Message
|
|
if err := wsjson.Read(deltaCtx, client.conn, &message); err != nil {
|
|
t.Fatalf("no terminal delta after the snapshot: %v", err)
|
|
}
|
|
if message.Type != protocol.RunEvent || message.RunEvent == nil || message.RunEvent.Workspace != late {
|
|
continue
|
|
}
|
|
switch message.RunEvent.Status {
|
|
case "succeeded", "failed", "canceled":
|
|
terminal = message.RunEvent
|
|
}
|
|
}
|
|
if terminal.RunID == 0 || terminal.Outcome != "success" {
|
|
t.Fatalf("late terminal frame = %+v", terminal)
|
|
}
|
|
}
|
|
|
|
// TestMergeRunEventFrameKeepsIdentity proves a coalesced frame keeps the run's
|
|
// identity from an earlier transition when a later delta omits it, so the merged
|
|
// frame a client actually receives still carries what its row needs.
|
|
func TestMergeRunEventFrameKeepsIdentity(t *testing.T) {
|
|
into := &protocol.RunEventFrame{RunID: 1, PipelineID: 4, Name: "release", Title: "Ship it", Version: 3, Status: protocol.RunStatusQueued}
|
|
mergeRunEventFrame(into, &protocol.RunEventFrame{RunID: 1, Status: protocol.RunStatusRunning, CurrentStage: "build"})
|
|
if into.PipelineID != 4 || into.Name != "release" || into.Title != "Ship it" || into.Version != 3 || into.Status != protocol.RunStatusRunning {
|
|
t.Fatalf("merged identity lost: %+v", into)
|
|
}
|
|
}
|
|
|
|
// assertNoRunEvent reads for d, ignoring responses, and fails if any run_event
|
|
// frame arrives.
|
|
func assertNoRunEvent(t *testing.T, client *runEventClient, d time.Duration) {
|
|
t.Helper()
|
|
readCtx, cancel := context.WithTimeout(client.ctx, d)
|
|
defer cancel()
|
|
for {
|
|
var message protocol.Message
|
|
if err := wsjson.Read(readCtx, client.conn, &message); err != nil {
|
|
return
|
|
}
|
|
if message.Type == protocol.RunEvent {
|
|
t.Fatalf("client received an unexpected run_event: %+v", message.RunEvent)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRunEventUnsubscribeAndNonSubscriberReceiveNothing proves an unsubscribed
|
|
// client and a client that never subscribed receive no run_event frames.
|
|
func TestRunEventUnsubscribeAndNonSubscriberReceiveNothing(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
|
|
subscribed := dialRunEvents(t, ctx, handler)
|
|
unsubscribedWorkspace := eventWorkspace(t)
|
|
subscribed.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: unsubscribedWorkspace})
|
|
if ack := subscribed.read(); ack.Type != protocol.Subscribe {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
subscribed.send(protocol.Message{Type: protocol.Unsubscribe, ID: "unsub"})
|
|
if ack := subscribed.read(); ack.Type != protocol.Unsubscribe || ack.ID != "unsub" {
|
|
t.Fatalf("unsubscribe ack = %+v", ack)
|
|
}
|
|
subscribed.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-a", PipelineID: ids[0], Version: 1, RequestKey: "after-unsub", Workspace: unsubscribedWorkspace, Input: map[string]any{"prompt": "test"}})
|
|
handler.runner.wg.Wait()
|
|
assertNoRunEvent(t, subscribed, 400*time.Millisecond)
|
|
|
|
stranger := dialRunEvents(t, ctx, handler)
|
|
strangerWorkspace := eventWorkspace(t)
|
|
stranger.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-b", PipelineID: ids[0], Version: 1, RequestKey: "never-subscribed", Workspace: strangerWorkspace, Input: map[string]any{"prompt": "test"}})
|
|
handler.runner.wg.Wait()
|
|
assertNoRunEvent(t, stranger, 400*time.Millisecond)
|
|
}
|
|
|
|
// TestRunEventWorkspaceIsolation proves a subscriber to one workspace never
|
|
// receives a delta for a run in another workspace, and still receives its own.
|
|
func TestRunEventWorkspaceIsolation(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
watched := eventWorkspace(t)
|
|
other := eventWorkspace(t)
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: watched})
|
|
if ack := client.read(); ack.Type != protocol.Subscribe {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
// The watched workspace still delivers its own run's terminal frame.
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "mine", PipelineID: ids[0], Version: 1, RequestKey: "watched-ws", Workspace: watched, Input: map[string]any{"prompt": "test"}})
|
|
var terminal *protocol.RunEventFrame
|
|
var runID int64
|
|
for terminal == nil {
|
|
message := client.read()
|
|
if message.Type == protocol.RunStarted && message.RunDetail != nil {
|
|
runID = message.RunDetail.ID
|
|
}
|
|
if message.Type == protocol.RunEvent && message.RunEvent != nil &&
|
|
(message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled") {
|
|
terminal = message.RunEvent
|
|
}
|
|
}
|
|
if runID == 0 || terminal.RunID != runID || terminal.Workspace != watched {
|
|
t.Fatalf("watched run terminal = %+v (run=%d), want workspace %q", terminal, runID, watched)
|
|
}
|
|
// A run in another workspace must not leak. This assertion reads until a
|
|
// bounded deadline, and that deadline closes the connection, so it runs last.
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "other", PipelineID: ids[0], Version: 1, RequestKey: "other-ws", Workspace: other, Input: map[string]any{"prompt": "test"}})
|
|
handler.runner.wg.Wait()
|
|
assertNoRunEvent(t, client, 400*time.Millisecond)
|
|
}
|
|
|
|
// TestRunEventAllWorkspacesSubscription proves the blank ("any workspace")
|
|
// subscribe scope — the one the app now asks for because its runs list is
|
|
// server-wide — delivers a run from every workspace, while an explicit
|
|
// workspace scope stays isolated (TestRunEventWorkspaceIsolation). It is what
|
|
// lets the app see a run executing in a lane worktree, a different workspace
|
|
// from the one the operator configured.
|
|
func TestRunEventAllWorkspacesSubscription(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
first := eventWorkspace(t)
|
|
second := eventWorkspace(t)
|
|
client := dialRunEvents(t, ctx, handler)
|
|
// The blank workspace is the "every workspace" scope; no run_ids narrows it.
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "all", Workspace: ""})
|
|
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "all" {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
// Subscribe first, then start a run in each workspace, so both are delivered
|
|
// as live deltas rather than only in the snapshot.
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "a", PipelineID: ids[0], Version: 1, RequestKey: "all-ws-a", Workspace: first, Input: map[string]any{"prompt": "test"}})
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "b", PipelineID: ids[0], Version: 1, RequestKey: "all-ws-b", Workspace: second, Input: map[string]any{"prompt": "test"}})
|
|
|
|
terminals := map[string]*protocol.RunEventFrame{}
|
|
deadline := time.Now().Add(8 * time.Second)
|
|
for len(terminals) < 2 && time.Now().Before(deadline) {
|
|
message := client.read()
|
|
if message.Type != protocol.RunEvent || message.RunEvent == nil {
|
|
continue
|
|
}
|
|
frame := message.RunEvent
|
|
if frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled" {
|
|
terminals[frame.Workspace] = frame
|
|
}
|
|
}
|
|
if terminals[first] == nil || terminals[second] == nil {
|
|
t.Fatalf("blank-scope subscription missed a workspace: got %v", terminals)
|
|
}
|
|
}
|
|
|
|
// TestRunEventCoalescingBoundsFrameCount records how many run_event frames a
|
|
// typical multi-stage run produces and asserts the count stays bounded: rapid
|
|
// transitions coalesce, and the terminal frame is never dropped.
|
|
func TestRunEventCoalescingBoundsFrameCount(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
stages := []string{"a", "b", "c"}
|
|
_, handler, ids := eventStore(t, ctx, stages...)
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
workspace := eventWorkspace(t)
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
|
|
if ack := client.read(); ack.Type != protocol.Subscribe {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "coalesce", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
|
|
frames := 0
|
|
var terminal *protocol.RunEventFrame
|
|
for terminal == nil {
|
|
message := client.read()
|
|
if message.Type != protocol.RunEvent || message.RunEvent == nil {
|
|
continue
|
|
}
|
|
frames++
|
|
if message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled" {
|
|
terminal = message.RunEvent
|
|
}
|
|
}
|
|
bound := 2*len(stages) + 4
|
|
if frames > bound {
|
|
t.Fatalf("run produced %d run_event frames for %d stages, want at most %d", frames, len(stages), bound)
|
|
}
|
|
if terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.CompletedSteps != len(stages) {
|
|
t.Fatalf("terminal frame = %+v", terminal)
|
|
}
|
|
t.Logf("run_event frames for a %d-stage run: %d (bound %d)", len(stages), frames, bound)
|
|
}
|