8f3036bd3b
Add a server-pushed run_event frame so a client no longer has to poll once a second to notice run changes. - protocol: RunEventFrame/RunStepDelta compact delta (run id, workspace, status, outcome, current stage, completed-step count, timestamps and an optional single step delta), plus subscribe/unsubscribe/run_event types and the optional RunEvent/RunIDs message fields. All fields optional so older peers ignore them. - server: runEventHub with per-connection subscriptions scoped to a workspace (and optional explicit run ids), one bounded writer goroutine per subscription, slow-subscriber drop, per-client subscription cap and a ~220ms per-run coalescer that flushes pending deltas before a terminal frame. - runner: emit deltas on created/queued, queued->running and step start, human waiting, step finish (outcome, target, diagnostic, artifacts and completed count), terminal finish, cancel of a waiting run and decide/resume; emitRun is a no-op without a hub. - websocket: subscribe seeds current state before the ack, unsubscribe and connection teardown drop subscriptions, and pushed frames reuse the connection write closure. - tests: protocol golden-JSON wire shape plus websocket coverage for the started-run frame sequence, subscribe current-state snapshot, silent unsubscribed/never-subscribed clients, workspace isolation and a bounded coalescing frame count. - docs: note subscribe/unsubscribe/run_event in docs/cli.md; TUI and Android still poll in this release.
330 lines
13 KiB
Go
330 lines
13 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
)
|
|
|
|
// runEventClient is a thin websocket client for the run-event tests: it dials
|
|
// /connect, consumes the unsolicited welcome, and reads/writes protocol
|
|
// envelopes.
|
|
type runEventClient struct {
|
|
t *testing.T
|
|
ctx context.Context
|
|
conn *websocket.Conn
|
|
}
|
|
|
|
func dialRunEvents(t *testing.T, ctx context.Context, handler *runHandler) *runEventClient {
|
|
t.Helper()
|
|
srv := httptest.NewServer(handler)
|
|
t.Cleanup(srv.Close)
|
|
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { conn.CloseNow() })
|
|
var welcome protocol.Message
|
|
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return &runEventClient{t: t, ctx: ctx, conn: conn}
|
|
}
|
|
|
|
func (c *runEventClient) send(message protocol.Message) {
|
|
c.t.Helper()
|
|
if err := wsjson.Write(c.ctx, c.conn, message); err != nil {
|
|
c.t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (c *runEventClient) read() protocol.Message {
|
|
c.t.Helper()
|
|
var message protocol.Message
|
|
if err := wsjson.Read(c.ctx, c.conn, &message); err != nil {
|
|
c.t.Fatal(err)
|
|
}
|
|
return message
|
|
}
|
|
|
|
// eventStore starts a fresh store holding one pipeline template and a handler
|
|
// wired to it, with the execute seam returning a deterministic success.
|
|
func eventStore(t *testing.T, ctx context.Context, stages ...string) (*pipeline.Store, *runHandler, []int64) {
|
|
t.Helper()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "events.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { store.Close() })
|
|
if _, err := store.Submit(ctx, "events", pipeline.Template(stages, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var ids []int64
|
|
for _, item := range items {
|
|
ids = append(ids, item.ID)
|
|
}
|
|
handler := Handler(ctx, store).(*runHandler)
|
|
t.Cleanup(func() { WaitRuns(handler) })
|
|
return store, handler, ids
|
|
}
|
|
|
|
// eventWorkspace returns a temporary directory canonicalised the same way the
|
|
// runner canonicalises a run workspace (EvalSymlinks). A subscription must use
|
|
// this resolved path to match: t.TempDir() alone is a symlinked /var path on
|
|
// macOS and would never equal the run's /private/var workspace, so no frame
|
|
// would be delivered.
|
|
func eventWorkspace(t *testing.T) string {
|
|
t.Helper()
|
|
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return dir
|
|
}
|
|
|
|
// TestRunEventWebSocketPushesTransitions starts a run, subscribes to its
|
|
// workspace, and proves the client observes the created, queued->running /
|
|
// step-started, step-finished (with the right outcome and target) and terminal
|
|
// frames, all carrying the run id.
|
|
func TestRunEventWebSocketPushesTransitions(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
workspace := eventWorkspace(t)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
select {
|
|
case <-release:
|
|
case <-ctx.Done():
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
|
|
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
|
|
|
|
var runID int64
|
|
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
|
|
for terminal == nil {
|
|
message := client.read()
|
|
switch message.Type {
|
|
case protocol.RunStarted:
|
|
if message.RunDetail == nil {
|
|
t.Fatal("run_started without detail")
|
|
}
|
|
runID = message.RunDetail.ID
|
|
case protocol.RunEvent:
|
|
frame := message.RunEvent
|
|
if frame == nil {
|
|
t.Fatal("run_event without payload")
|
|
}
|
|
switch {
|
|
case frame.Step == nil && frame.Status == protocol.RunStatusQueued:
|
|
created = frame
|
|
case frame.Step != nil && frame.Step.Status == protocol.StepStatusRunning:
|
|
if stepStarted == nil {
|
|
stepStarted = frame
|
|
releaseOnce.Do(func() { close(release) })
|
|
}
|
|
case frame.Step != nil && frame.Step.Status == "completed":
|
|
stepFinished = frame
|
|
case frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled":
|
|
terminal = frame
|
|
}
|
|
}
|
|
}
|
|
if runID == 0 {
|
|
t.Fatal("run_started never carried the run id")
|
|
}
|
|
if created == nil || created.RunID != runID || created.Step != nil || created.CreatedAt == "" {
|
|
t.Fatalf("created frame = %+v", created)
|
|
}
|
|
if stepStarted == nil || stepStarted.RunID != runID || stepStarted.Status != protocol.RunStatusRunning || stepStarted.CurrentStage != "one" || stepStarted.Step.Sequence != 1 || stepStarted.Step.Stage != "one" {
|
|
t.Fatalf("step-started frame = %+v", stepStarted)
|
|
}
|
|
if stepFinished == nil || stepFinished.RunID != runID || stepFinished.Step.Status != "completed" || stepFinished.Step.Outcome != "success" || stepFinished.Step.Target != protocol.SuccessEnd || stepFinished.CompletedSteps != 1 {
|
|
t.Fatalf("step-finished frame = %+v", stepFinished)
|
|
}
|
|
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
|
|
t.Fatalf("terminal frame = %+v", terminal)
|
|
}
|
|
}
|
|
|
|
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
|
|
// already-existing run receives its current state as a run_event frame before
|
|
// the subscribe ack, so a reconnecting client re-syncs without polling.
|
|
func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
workspace := eventWorkspace(t)
|
|
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
handler.runner.wg.Wait()
|
|
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: workspace})
|
|
first := client.read()
|
|
if first.Type != protocol.RunEvent || first.RunEvent == nil {
|
|
t.Fatalf("first frame after subscribe = %+v, want a run_event snapshot", first)
|
|
}
|
|
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
|
|
t.Fatalf("snapshot frame = %+v", first.RunEvent)
|
|
}
|
|
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "snap" {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
}
|
|
|
|
// assertNoRunEvent reads for d, ignoring responses, and fails if any run_event
|
|
// frame arrives.
|
|
func assertNoRunEvent(t *testing.T, client *runEventClient, d time.Duration) {
|
|
t.Helper()
|
|
readCtx, cancel := context.WithTimeout(client.ctx, d)
|
|
defer cancel()
|
|
for {
|
|
var message protocol.Message
|
|
if err := wsjson.Read(readCtx, client.conn, &message); err != nil {
|
|
return
|
|
}
|
|
if message.Type == protocol.RunEvent {
|
|
t.Fatalf("client received an unexpected run_event: %+v", message.RunEvent)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRunEventUnsubscribeAndNonSubscriberReceiveNothing proves an unsubscribed
|
|
// client and a client that never subscribed receive no run_event frames.
|
|
func TestRunEventUnsubscribeAndNonSubscriberReceiveNothing(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
|
|
subscribed := dialRunEvents(t, ctx, handler)
|
|
unsubscribedWorkspace := eventWorkspace(t)
|
|
subscribed.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: unsubscribedWorkspace})
|
|
if ack := subscribed.read(); ack.Type != protocol.Subscribe {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
subscribed.send(protocol.Message{Type: protocol.Unsubscribe, ID: "unsub"})
|
|
if ack := subscribed.read(); ack.Type != protocol.Unsubscribe || ack.ID != "unsub" {
|
|
t.Fatalf("unsubscribe ack = %+v", ack)
|
|
}
|
|
subscribed.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-a", PipelineID: ids[0], Version: 1, RequestKey: "after-unsub", Workspace: unsubscribedWorkspace, Input: map[string]any{"prompt": "test"}})
|
|
handler.runner.wg.Wait()
|
|
assertNoRunEvent(t, subscribed, 400*time.Millisecond)
|
|
|
|
stranger := dialRunEvents(t, ctx, handler)
|
|
strangerWorkspace := eventWorkspace(t)
|
|
stranger.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-b", PipelineID: ids[0], Version: 1, RequestKey: "never-subscribed", Workspace: strangerWorkspace, Input: map[string]any{"prompt": "test"}})
|
|
handler.runner.wg.Wait()
|
|
assertNoRunEvent(t, stranger, 400*time.Millisecond)
|
|
}
|
|
|
|
// TestRunEventWorkspaceIsolation proves a subscriber to one workspace never
|
|
// receives a delta for a run in another workspace, and still receives its own.
|
|
func TestRunEventWorkspaceIsolation(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
_, handler, ids := eventStore(t, ctx, "one")
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
watched := eventWorkspace(t)
|
|
other := eventWorkspace(t)
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: watched})
|
|
if ack := client.read(); ack.Type != protocol.Subscribe {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
// The watched workspace still delivers its own run's terminal frame.
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "mine", PipelineID: ids[0], Version: 1, RequestKey: "watched-ws", Workspace: watched, Input: map[string]any{"prompt": "test"}})
|
|
var terminal *protocol.RunEventFrame
|
|
var runID int64
|
|
for terminal == nil {
|
|
message := client.read()
|
|
if message.Type == protocol.RunStarted && message.RunDetail != nil {
|
|
runID = message.RunDetail.ID
|
|
}
|
|
if message.Type == protocol.RunEvent && message.RunEvent != nil &&
|
|
(message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled") {
|
|
terminal = message.RunEvent
|
|
}
|
|
}
|
|
if runID == 0 || terminal.RunID != runID || terminal.Workspace != watched {
|
|
t.Fatalf("watched run terminal = %+v (run=%d), want workspace %q", terminal, runID, watched)
|
|
}
|
|
// A run in another workspace must not leak. This assertion reads until a
|
|
// bounded deadline, and that deadline closes the connection, so it runs last.
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "other", PipelineID: ids[0], Version: 1, RequestKey: "other-ws", Workspace: other, Input: map[string]any{"prompt": "test"}})
|
|
handler.runner.wg.Wait()
|
|
assertNoRunEvent(t, client, 400*time.Millisecond)
|
|
}
|
|
|
|
// TestRunEventCoalescingBoundsFrameCount records how many run_event frames a
|
|
// typical multi-stage run produces and asserts the count stays bounded: rapid
|
|
// transitions coalesce, and the terminal frame is never dropped.
|
|
func TestRunEventCoalescingBoundsFrameCount(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
stages := []string{"a", "b", "c"}
|
|
_, handler, ids := eventStore(t, ctx, stages...)
|
|
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
workspace := eventWorkspace(t)
|
|
client := dialRunEvents(t, ctx, handler)
|
|
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
|
|
if ack := client.read(); ack.Type != protocol.Subscribe {
|
|
t.Fatalf("subscribe ack = %+v", ack)
|
|
}
|
|
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "coalesce", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
|
|
frames := 0
|
|
var terminal *protocol.RunEventFrame
|
|
for terminal == nil {
|
|
message := client.read()
|
|
if message.Type != protocol.RunEvent || message.RunEvent == nil {
|
|
continue
|
|
}
|
|
frames++
|
|
if message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled" {
|
|
terminal = message.RunEvent
|
|
}
|
|
}
|
|
bound := 2*len(stages) + 4
|
|
if frames > bound {
|
|
t.Fatalf("run produced %d run_event frames for %d stages, want at most %d", frames, len(stages), bound)
|
|
}
|
|
if terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.CompletedSteps != len(stages) {
|
|
t.Fatalf("terminal frame = %+v", terminal)
|
|
}
|
|
t.Logf("run_event frames for a %d-stage run: %d (bound %d)", len(stages), frames, bound)
|
|
}
|