diff --git a/docs/cli.md b/docs/cli.md index c7ffe15c..9d1e2db0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -348,6 +348,16 @@ or Tab to return to details. When not focused, the graph follows the latest visi or selected stage. Existing polling refreshes the graph once per second, and running durations show elapsed time rather than β€œin progress.” +The server can also push run changes. A client sends `subscribe` scoped to a +workspace (and optionally to explicit run ids); the server first returns the +current state of the matching runs, then a compact `run_event` delta on each +transition β€” which run changed, its status, outcome, current stage, completed-step +count and timestamps, and, when a single step changed, that step's sequence, stage, +status, outcome, target and diagnostic. It is a delta, never the whole run, and +rapid changes are coalesced to at most a few frames per second per run. +`unsubscribe` or disconnecting stops the frames. This lets a client drop the +once-per-second poll; the TUI and Android app still poll in this release. + In the run inspector, `n`/`p` select the next/previous recorded execution visit, including repeated visits in loops, from any pane. Selection stops at the last visit or overview; it does not wrap. The selected stage has a full-row highlight; diff --git a/internal/protocol/message.go b/internal/protocol/message.go index dbac24f6..080cdc7d 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -164,4 +164,9 @@ type Message struct { ChatEvent *ChatEventFrame `json:"chat_event,omitempty"` ChatPermission *ChatPermissionFrame `json:"chat_permission,omitempty"` ChatReply *ChatReplyRequest `json:"chat_reply,omitempty"` + // RunEvent is the server-pushed run state delta (see runevents.go) and + // RunIDs the optional subscribe scope, so a client asks for the runs it + // displays. Both are optional so older peers ignore them. + RunEvent *RunEventFrame `json:"run_event,omitempty"` + RunIDs []int64 `json:"run_ids,omitempty"` } diff --git a/internal/protocol/runevents.go b/internal/protocol/runevents.go new file mode 100644 index 00000000..ab1796ff --- /dev/null +++ b/internal/protocol/runevents.go @@ -0,0 +1,48 @@ +package protocol + +// Server-pushed run state. A client sends subscribe (scoped to the workspace and +// optionally to explicit run ids) to ask for the runs it displays: the server +// first returns the current state of those runs so a reconnecting client +// re-syncs, then pushes a compact run_event delta on every meaningful +// transition. It sends unsubscribe to stop. Like chat_event, run_event is a +// server-pushed frame with no matching request, so both fields are optional and +// older peers ignore them. +const ( + Subscribe = "subscribe" + Unsubscribe = "unsubscribe" + RunEvent = "run_event" +) + +// RunEventFrame is the server-pushed run_event delta. It carries only the values +// that changed, never the whole run: the run's identity, status, outcome, current +// stage, completed-step count and timestamps, plus at most one changed step. +// Every field is optional so an older peer ignores what it does not know and a +// client can fold the frame into its cached run. +type RunEventFrame struct { + RunID int64 `json:"run_id"` + Workspace string `json:"workspace,omitempty"` + Status string `json:"status,omitempty"` + Outcome string `json:"outcome,omitempty"` + CurrentStage string `json:"current_stage,omitempty"` + CompletedSteps int `json:"completed_steps,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + FinishedAt string `json:"finished_at,omitempty"` + // Artifacts is the number of files recorded for the referenced step, so a + // client can surface artifact capture without reading the step. + Artifacts int `json:"artifacts,omitempty"` + // Step is present when a single step changed (started, finished or waiting). + Step *RunStepDelta `json:"step,omitempty"` +} + +// RunStepDelta is the compact change of one step within a RunEventFrame. It is a +// delta, not a RunStep: it omits the input/output, transcript, file changes and +// artifact bytes a full step carries. +type RunStepDelta struct { + Sequence int `json:"sequence"` + Stage string `json:"stage,omitempty"` + Status string `json:"status,omitempty"` + Outcome string `json:"outcome,omitempty"` + Target string `json:"target,omitempty"` + Diagnostic string `json:"diagnostic,omitempty"` + Artifacts int `json:"artifacts,omitempty"` +} diff --git a/internal/protocol/runevents_test.go b/internal/protocol/runevents_test.go new file mode 100644 index 00000000..812d9786 --- /dev/null +++ b/internal/protocol/runevents_test.go @@ -0,0 +1,65 @@ +package protocol + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +// TestRunEventGoldenJSON pins the exact wire shape of the pushed run_event frame +// so a JSON tag change cannot silently alter what a client parses. +func TestRunEventGoldenJSON(t *testing.T) { + frame := RunEventFrame{ + RunID: 7, Workspace: "/work", Status: RunStatusRunning, CurrentStage: "build", + CompletedSteps: 1, CreatedAt: "2026-01-02T03:04:05Z", Artifacts: 2, + Step: &RunStepDelta{Sequence: 2, Stage: "build", Status: "completed", Outcome: "success", Target: SuccessEnd, Artifacts: 2}, + } + encoded, err := json.Marshal(frame) + if err != nil { + t.Fatal(err) + } + want := `{"run_id":7,"workspace":"/work","status":"running","current_stage":"build","completed_steps":1,"created_at":"2026-01-02T03:04:05Z","artifacts":2,"step":{"sequence":2,"stage":"build","status":"completed","outcome":"success","target":"#success","artifacts":2}}` + if string(encoded) != want { + t.Fatalf("golden mismatch\n got %s\nwant %s", encoded, want) + } + var decoded RunEventFrame + if err := json.Unmarshal([]byte(want), &decoded); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(decoded, frame) { + t.Fatalf("golden did not decode back: %+v", decoded) + } +} + +// TestRunEventEnvelopeRoundTrip proves the new optional fields survive the shared +// Message envelope and that an unused payload is omitted from the wire. +func TestRunEventEnvelopeRoundTrip(t *testing.T) { + message := Message{ + Type: RunEvent, + ID: "r1", + RunEvent: &RunEventFrame{RunID: 3, Workspace: "/w", Status: RunStatusQueued, CreatedAt: "2026-01-02T03:04:05Z"}, + RunIDs: []int64{3, 4}, + } + encoded, err := json.Marshal(message) + if err != nil { + t.Fatal(err) + } + var decoded Message + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(decoded.RunEvent, message.RunEvent) || !reflect.DeepEqual(decoded.RunIDs, message.RunIDs) { + t.Fatalf("run event fields lost: %+v", decoded) + } + + minimal, err := json.Marshal(Message{Type: Subscribe, ID: "s1", Workspace: "/w"}) + if err != nil { + t.Fatal(err) + } + for _, absent := range []string{"run_event", "run_ids"} { + if strings.Contains(string(minimal), absent) { + t.Fatalf("absent %q must be omitted: %s", absent, minimal) + } + } +} diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 32278b8b..4c229b9c 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -21,8 +21,10 @@ const ( // the run and its step persist these statuses and the decision path resumes // them. Store, server, CLI and TUI read the one pair below, never a literal. const ( + RunStatusQueued = "queued" RunStatusWaiting = "waiting" RunStatusRunning = "running" + StepStatusRunning = "running" StepStatusWaiting = "waiting" // NotifyWaiting is the notification event a human stage emits when it parks // the run. It is the same value the `on=` policy accepts. diff --git a/internal/server/run_event_websocket_test.go b/internal/server/run_event_websocket_test.go new file mode 100644 index 00000000..008320b4 --- /dev/null +++ b/internal/server/run_event_websocket_test.go @@ -0,0 +1,329 @@ +package server + +import ( + "context" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" +) + +// runEventClient is a thin websocket client for the run-event tests: it dials +// /connect, consumes the unsolicited welcome, and reads/writes protocol +// envelopes. +type runEventClient struct { + t *testing.T + ctx context.Context + conn *websocket.Conn +} + +func dialRunEvents(t *testing.T, ctx context.Context, handler *runHandler) *runEventClient { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.CloseNow() }) + var welcome protocol.Message + if err := wsjson.Read(ctx, conn, &welcome); err != nil { + t.Fatal(err) + } + return &runEventClient{t: t, ctx: ctx, conn: conn} +} + +func (c *runEventClient) send(message protocol.Message) { + c.t.Helper() + if err := wsjson.Write(c.ctx, c.conn, message); err != nil { + c.t.Fatal(err) + } +} + +func (c *runEventClient) read() protocol.Message { + c.t.Helper() + var message protocol.Message + if err := wsjson.Read(c.ctx, c.conn, &message); err != nil { + c.t.Fatal(err) + } + return message +} + +// eventStore starts a fresh store holding one pipeline template and a handler +// wired to it, with the execute seam returning a deterministic success. +func eventStore(t *testing.T, ctx context.Context, stages ...string) (*pipeline.Store, *runHandler, []int64) { + t.Helper() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "events.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + if _, err := store.Submit(ctx, "events", pipeline.Template(stages, true)); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + var ids []int64 + for _, item := range items { + ids = append(ids, item.ID) + } + handler := Handler(ctx, store).(*runHandler) + t.Cleanup(func() { WaitRuns(handler) }) + return store, handler, ids +} + +// eventWorkspace returns a temporary directory canonicalised the same way the +// runner canonicalises a run workspace (EvalSymlinks). A subscription must use +// this resolved path to match: t.TempDir() alone is a symlinked /var path on +// macOS and would never equal the run's /private/var workspace, so no frame +// would be delivered. +func eventWorkspace(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} + +// TestRunEventWebSocketPushesTransitions starts a run, subscribes to its +// workspace, and proves the client observes the created, queued->running / +// step-started, step-finished (with the right outcome and target) and terminal +// frames, all carrying the run id. +func TestRunEventWebSocketPushesTransitions(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _, handler, ids := eventStore(t, ctx, "one") + workspace := eventWorkspace(t) + release := make(chan struct{}) + var releaseOnce sync.Once + handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + client := dialRunEvents(t, ctx, handler) + client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace}) + if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" { + t.Fatalf("subscribe ack = %+v", ack) + } + client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}}) + + var runID int64 + var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame + for terminal == nil { + message := client.read() + switch message.Type { + case protocol.RunStarted: + if message.RunDetail == nil { + t.Fatal("run_started without detail") + } + runID = message.RunDetail.ID + case protocol.RunEvent: + frame := message.RunEvent + if frame == nil { + t.Fatal("run_event without payload") + } + switch { + case frame.Step == nil && frame.Status == protocol.RunStatusQueued: + created = frame + case frame.Step != nil && frame.Step.Status == protocol.StepStatusRunning: + if stepStarted == nil { + stepStarted = frame + releaseOnce.Do(func() { close(release) }) + } + case frame.Step != nil && frame.Step.Status == "completed": + stepFinished = frame + case frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled": + terminal = frame + } + } + } + if runID == 0 { + t.Fatal("run_started never carried the run id") + } + if created == nil || created.RunID != runID || created.Step != nil || created.CreatedAt == "" { + t.Fatalf("created frame = %+v", created) + } + if stepStarted == nil || stepStarted.RunID != runID || stepStarted.Status != protocol.RunStatusRunning || stepStarted.CurrentStage != "one" || stepStarted.Step.Sequence != 1 || stepStarted.Step.Stage != "one" { + t.Fatalf("step-started frame = %+v", stepStarted) + } + if stepFinished == nil || stepFinished.RunID != runID || stepFinished.Step.Status != "completed" || stepFinished.Step.Outcome != "success" || stepFinished.Step.Target != protocol.SuccessEnd || stepFinished.CompletedSteps != 1 { + t.Fatalf("step-finished frame = %+v", stepFinished) + } + if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" { + t.Fatalf("terminal frame = %+v", terminal) + } +} + +// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an +// already-existing run receives its current state as a run_event frame before +// the subscribe ack, so a reconnecting client re-syncs without polling. +func TestRunEventSubscribeSendsSnapshot(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, handler, ids := eventStore(t, ctx, "one") + handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + workspace := eventWorkspace(t) + run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}}) + if err != nil { + t.Fatal(err) + } + handler.runner.wg.Wait() + + client := dialRunEvents(t, ctx, handler) + client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: workspace}) + first := client.read() + if first.Type != protocol.RunEvent || first.RunEvent == nil { + t.Fatalf("first frame after subscribe = %+v, want a run_event snapshot", first) + } + if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" { + t.Fatalf("snapshot frame = %+v", first.RunEvent) + } + if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "snap" { + t.Fatalf("subscribe ack = %+v", ack) + } +} + +// assertNoRunEvent reads for d, ignoring responses, and fails if any run_event +// frame arrives. +func assertNoRunEvent(t *testing.T, client *runEventClient, d time.Duration) { + t.Helper() + readCtx, cancel := context.WithTimeout(client.ctx, d) + defer cancel() + for { + var message protocol.Message + if err := wsjson.Read(readCtx, client.conn, &message); err != nil { + return + } + if message.Type == protocol.RunEvent { + t.Fatalf("client received an unexpected run_event: %+v", message.RunEvent) + } + } +} + +// TestRunEventUnsubscribeAndNonSubscriberReceiveNothing proves an unsubscribed +// client and a client that never subscribed receive no run_event frames. +func TestRunEventUnsubscribeAndNonSubscriberReceiveNothing(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, handler, ids := eventStore(t, ctx, "one") + handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + + subscribed := dialRunEvents(t, ctx, handler) + unsubscribedWorkspace := eventWorkspace(t) + subscribed.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: unsubscribedWorkspace}) + if ack := subscribed.read(); ack.Type != protocol.Subscribe { + t.Fatalf("subscribe ack = %+v", ack) + } + subscribed.send(protocol.Message{Type: protocol.Unsubscribe, ID: "unsub"}) + if ack := subscribed.read(); ack.Type != protocol.Unsubscribe || ack.ID != "unsub" { + t.Fatalf("unsubscribe ack = %+v", ack) + } + subscribed.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-a", PipelineID: ids[0], Version: 1, RequestKey: "after-unsub", Workspace: unsubscribedWorkspace, Input: map[string]any{"prompt": "test"}}) + handler.runner.wg.Wait() + assertNoRunEvent(t, subscribed, 400*time.Millisecond) + + stranger := dialRunEvents(t, ctx, handler) + strangerWorkspace := eventWorkspace(t) + stranger.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-b", PipelineID: ids[0], Version: 1, RequestKey: "never-subscribed", Workspace: strangerWorkspace, Input: map[string]any{"prompt": "test"}}) + handler.runner.wg.Wait() + assertNoRunEvent(t, stranger, 400*time.Millisecond) +} + +// TestRunEventWorkspaceIsolation proves a subscriber to one workspace never +// receives a delta for a run in another workspace, and still receives its own. +func TestRunEventWorkspaceIsolation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, handler, ids := eventStore(t, ctx, "one") + handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + watched := eventWorkspace(t) + other := eventWorkspace(t) + client := dialRunEvents(t, ctx, handler) + client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: watched}) + if ack := client.read(); ack.Type != protocol.Subscribe { + t.Fatalf("subscribe ack = %+v", ack) + } + // The watched workspace still delivers its own run's terminal frame. + client.send(protocol.Message{Type: protocol.RunPipeline, ID: "mine", PipelineID: ids[0], Version: 1, RequestKey: "watched-ws", Workspace: watched, Input: map[string]any{"prompt": "test"}}) + var terminal *protocol.RunEventFrame + var runID int64 + for terminal == nil { + message := client.read() + if message.Type == protocol.RunStarted && message.RunDetail != nil { + runID = message.RunDetail.ID + } + if message.Type == protocol.RunEvent && message.RunEvent != nil && + (message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled") { + terminal = message.RunEvent + } + } + if runID == 0 || terminal.RunID != runID || terminal.Workspace != watched { + t.Fatalf("watched run terminal = %+v (run=%d), want workspace %q", terminal, runID, watched) + } + // A run in another workspace must not leak. This assertion reads until a + // bounded deadline, and that deadline closes the connection, so it runs last. + client.send(protocol.Message{Type: protocol.RunPipeline, ID: "other", PipelineID: ids[0], Version: 1, RequestKey: "other-ws", Workspace: other, Input: map[string]any{"prompt": "test"}}) + handler.runner.wg.Wait() + assertNoRunEvent(t, client, 400*time.Millisecond) +} + +// TestRunEventCoalescingBoundsFrameCount records how many run_event frames a +// typical multi-stage run produces and asserts the count stays bounded: rapid +// transitions coalesce, and the terminal frame is never dropped. +func TestRunEventCoalescingBoundsFrameCount(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + stages := []string{"a", "b", "c"} + _, handler, ids := eventStore(t, ctx, stages...) + handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + workspace := eventWorkspace(t) + client := dialRunEvents(t, ctx, handler) + client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace}) + if ack := client.read(); ack.Type != protocol.Subscribe { + t.Fatalf("subscribe ack = %+v", ack) + } + client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "coalesce", Workspace: workspace, Input: map[string]any{"prompt": "test"}}) + frames := 0 + var terminal *protocol.RunEventFrame + for terminal == nil { + message := client.read() + if message.Type != protocol.RunEvent || message.RunEvent == nil { + continue + } + frames++ + if message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled" { + terminal = message.RunEvent + } + } + bound := 2*len(stages) + 4 + if frames > bound { + t.Fatalf("run produced %d run_event frames for %d stages, want at most %d", frames, len(stages), bound) + } + if terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.CompletedSteps != len(stages) { + t.Fatalf("terminal frame = %+v", terminal) + } + t.Logf("run_event frames for a %d-stage run: %d (bound %d)", len(stages), frames, bound) +} diff --git a/internal/server/runevents.go b/internal/server/runevents.go new file mode 100644 index 00000000..e767d4fa --- /dev/null +++ b/internal/server/runevents.go @@ -0,0 +1,429 @@ +package server + +import ( + "sync" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// The run-event hub pushes compact run state deltas to subscribed connections so +// a client no longer has to poll. It mirrors the chat hub's shape: subscriptions +// live on the hub, keyed by connection, and every write goes through the +// connection-scoped write closure. Unlike chat, a run transition has no session +// to stream from, so deltas are emitted by the runner and coalesced here. +const ( + // runEventFlushInterval throttles a busy run's pushes to at most a few frames + // per second. A terminal delta is flushed immediately regardless, so the final + // state is never delayed or dropped. + runEventFlushInterval = 220 * time.Millisecond + // runEventSubscriberBuffer bounds one subscription's pending frames. A + // subscriber that falls this far behind is dropped instead of blocking the + // producer or other subscribers; it re-syncs from a fresh subscribe snapshot. + runEventSubscriberBuffer = 32 + // runEventMaxSubscriptionsPerClient bounds one connection's subscriptions so a + // client cannot grow the registry without limit. + runEventMaxSubscriptionsPerClient = 16 + // runEventMaxCoalescers bounds the per-run coalescer registry; idle + // coalescers are evicted once it is exceeded. + runEventMaxCoalescers = 512 +) + +// runEventHub tracks the connections subscribed to run deltas and one coalescer +// per active run. All shared maps are guarded by mu. Nothing here performs a +// database read: the snapshot is supplied by the caller and every delta carries +// values already computed at the emit site. +type runEventHub struct { + mu sync.Mutex + subs map[*runSubscription]struct{} + byClient map[string]map[*runSubscription]struct{} + runs map[int64]*runCoalescer + now func() time.Time + interval time.Duration +} + +// runCoalescer merges a run's rapid field updates into the latest pending delta +// and flushes at most once per interval. A terminal delta bypasses the timer. +type runCoalescer struct { + runID int64 + workspace string + pending *protocol.RunEventFrame + timer *time.Timer + lastFlush time.Time + updated time.Time +} + +// runSubscription is one connection's watch over a workspace (and optionally +// explicit run ids). Frames are handed to a dedicated writer goroutine so a slow +// or broken subscriber never stalls the producer, and its buffered channel +// bounds how far behind it may fall. +type runSubscription struct { + hub *runEventHub + clientID string + workspace string + runIDs map[int64]struct{} + ch chan protocol.Message + done chan struct{} + once sync.Once + write func(protocol.Message) error +} + +func newRunEventHub() *runEventHub { + return &runEventHub{ + subs: map[*runSubscription]struct{}{}, + byClient: map[string]map[*runSubscription]struct{}{}, + runs: map[int64]*runCoalescer{}, + now: time.Now, + interval: runEventFlushInterval, + } +} + +func (h *runEventHub) clock() time.Time { + if h.now != nil { + return h.now() + } + return time.Now() +} + +// subscribe registers one connection's watch and starts its writer goroutine. It +// returns nil when the connection already holds the per-client subscription cap. +func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription { + if h == nil || write == nil { + return nil + } + h.mu.Lock() + if len(h.byClient[clientID]) >= runEventMaxSubscriptionsPerClient { + h.mu.Unlock() + return nil + } + sub := &runSubscription{ + hub: h, + clientID: clientID, + workspace: workspace, + ch: make(chan protocol.Message, runEventSubscriberBuffer), + done: make(chan struct{}), + write: write, + } + for _, id := range runIDs { + if id <= 0 { + continue + } + if sub.runIDs == nil { + sub.runIDs = map[int64]struct{}{} + } + sub.runIDs[id] = struct{}{} + } + h.subs[sub] = struct{}{} + if h.byClient[clientID] == nil { + h.byClient[clientID] = map[*runSubscription]struct{}{} + } + h.byClient[clientID][sub] = struct{}{} + h.mu.Unlock() + go sub.run() + return sub +} + +// unsubscribeClient drops every subscription held by one connection; the read +// loop calls it when the connection ends so nothing leaks. +func (h *runEventHub) unsubscribeClient(clientID string) { + if h == nil { + return + } + h.mu.Lock() + var subs []*runSubscription + for sub := range h.byClient[clientID] { + subs = append(subs, sub) + } + h.mu.Unlock() + for _, sub := range subs { + sub.close() + } +} + +func (h *runEventHub) remove(sub *runSubscription) { + h.mu.Lock() + delete(h.subs, sub) + if set := h.byClient[sub.clientID]; set != nil { + delete(set, sub) + if len(set) == 0 { + delete(h.byClient, sub.clientID) + } + } + h.releaseIdleLocked() + h.mu.Unlock() +} + +// emit folds one transition into its run's coalescer and schedules the flush. +// A terminal transition flushes any pending delta first, then the terminal frame +// immediately, so a client always observes the final state as its own frame. +func (h *runEventHub) emit(frame protocol.RunEventFrame, terminal bool) { + if h == nil || frame.RunID <= 0 { + return + } + h.mu.Lock() + if !h.anySubscriberLocked(frame.Workspace) { + h.mu.Unlock() + return + } + c := h.runs[frame.RunID] + if c == nil { + h.evictIdleLocked() + c = &runCoalescer{runID: frame.RunID} + h.runs[frame.RunID] = c + } + c.workspace = frame.Workspace + now := h.clock() + c.updated = now + var out []protocol.RunEventFrame + if terminal { + if c.pending != nil { + out = append(out, *c.pending) + c.pending = nil + } + out = append(out, frame) + h.stopTimerLocked(c) + c.lastFlush = now + } else { + if c.pending == nil { + pending := frame + c.pending = &pending + } else { + mergeRunEventFrame(c.pending, &frame) + } + if c.timer == nil { + if wait := h.interval - now.Sub(c.lastFlush); wait <= 0 { + out = append(out, *c.pending) + c.pending = nil + c.lastFlush = now + } else { + c.timer = time.AfterFunc(wait, func() { h.flushRun(frame.RunID) }) + } + } + } + h.mu.Unlock() + for _, f := range out { + h.deliver(f) + } +} + +func (h *runEventHub) flushRun(runID int64) { + h.mu.Lock() + c := h.runs[runID] + if c == nil { + h.mu.Unlock() + return + } + c.timer = nil + var out []protocol.RunEventFrame + if c.pending != nil { + out = append(out, *c.pending) + c.pending = nil + c.lastFlush = h.clock() + c.updated = c.lastFlush + } + h.mu.Unlock() + for _, f := range out { + h.deliver(f) + } +} + +// deliver fans one frame out to the matching subscriptions. It collects the +// targets under the lock and hands them off outside it, so a slow write cannot +// hold the hub and stall the producer. +func (h *runEventHub) deliver(frame protocol.RunEventFrame) { + h.mu.Lock() + targets := make([]*runSubscription, 0, len(h.subs)) + for sub := range h.subs { + if sub.matches(frame.Workspace, frame.RunID) { + targets = append(targets, sub) + } + } + h.mu.Unlock() + for _, sub := range targets { + sub.send(frame) + } +} + +func (h *runEventHub) anySubscriberLocked(workspace string) bool { + for sub := range h.subs { + if sub.workspace == "" || sub.workspace == workspace { + return true + } + } + return false +} + +// releaseIdleLocked drops every coalescer whose workspace no longer has a +// subscriber, stopping its timer, so the registry is bounded and released on +// unsubscribe/disconnect. +func (h *runEventHub) releaseIdleLocked() { + for id, c := range h.runs { + if h.anySubscriberLocked(c.workspace) { + continue + } + h.stopTimerLocked(c) + delete(h.runs, id) + } +} + +// evictIdleLocked keeps the coalescer registry bounded by dropping the +// least-recently-updated idle coalescer once the cap is reached. It never evicts +// a coalescer with pending work or a running timer. +func (h *runEventHub) evictIdleLocked() { + for len(h.runs) >= runEventMaxCoalescers { + var oldest *runCoalescer + for _, c := range h.runs { + if c.pending != nil || c.timer != nil { + continue + } + if oldest == nil || c.updated.Before(oldest.updated) { + oldest = c + } + } + if oldest == nil { + return + } + h.stopTimerLocked(oldest) + delete(h.runs, oldest.runID) + } +} + +func (h *runEventHub) stopTimerLocked(c *runCoalescer) { + if c.timer != nil { + c.timer.Stop() + c.timer = nil + } +} + +func (s *runSubscription) matches(workspace string, runID int64) bool { + if s.workspace != "" && s.workspace != workspace { + return false + } + if len(s.runIDs) > 0 { + _, ok := s.runIDs[runID] + return ok + } + return true +} + +func (s *runSubscription) run() { + for { + select { + case <-s.done: + return + case msg := <-s.ch: + if err := s.write(msg); err != nil { + s.close() + return + } + } + } +} + +func (s *runSubscription) send(frame protocol.RunEventFrame) { + s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) +} + +func (s *runSubscription) sendMessage(msg protocol.Message) { + select { + case <-s.done: + return + default: + } + select { + case s.ch <- msg: + default: + // The subscriber is not draining: drop it rather than block the + // producer. Its next subscribe re-syncs from a fresh snapshot. + s.close() + } +} + +// close unregisters the subscription and stops its writer goroutine exactly +// once. It is safe to call from the writer goroutine, from unsubscribe and from +// the disconnect path. +func (s *runSubscription) close() { + s.once.Do(func() { + close(s.done) + s.hub.remove(s) + }) +} + +func mergeRunEventFrame(into, in *protocol.RunEventFrame) { + if in.Workspace != "" { + into.Workspace = in.Workspace + } + if in.Status != "" { + into.Status = in.Status + } + if in.Outcome != "" { + into.Outcome = in.Outcome + } + if in.CurrentStage != "" { + into.CurrentStage = in.CurrentStage + } + if in.CompletedSteps > into.CompletedSteps { + into.CompletedSteps = in.CompletedSteps + } + if in.CreatedAt != "" { + into.CreatedAt = in.CreatedAt + } + if in.FinishedAt != "" { + into.FinishedAt = in.FinishedAt + } + if in.Artifacts > into.Artifacts { + into.Artifacts = in.Artifacts + } + if in.Step != nil { + into.Step = in.Step + } +} + +// handleRunEvents dispatches one subscribe/unsubscribe request and reports +// whether it was a run-event type (so the caller never falls through to the +// generic unknown-command response). Subscribe registers the watch, sends the +// current state of the watched runs through the subscription, then acks; the +// snapshot therefore precedes every later delta. +func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool { + switch request.Type { + case protocol.Subscribe: + if runs == nil || events == nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "run events unavailable: no store"}) + return true + } + sub := events.subscribe(clientID, request.Workspace, request.RunIDs, write) + if sub == nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "too many run subscriptions on this connection"}) + return true + } + // One bounded read seeds the subscription. It reuses listRuns, the same + // workspace-scoped read the polling clients use, instead of a new query. + summaries, err := runs.listRuns() + if err != nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) + return true + } + for _, summary := range summaries { + if !sub.matches(summary.Workspace, summary.ID) { + continue + } + sub.send(protocol.RunEventFrame{ + RunID: summary.ID, + Workspace: summary.Workspace, + Status: summary.Status, + Outcome: summary.Outcome, + CurrentStage: summary.CurrentStage, + CompletedSteps: summary.CompletedSteps, + CreatedAt: summary.CreatedAt, + FinishedAt: summary.FinishedAt, + }) + } + sub.sendMessage(protocol.Message{Type: protocol.Subscribe, ID: request.ID}) + return true + case protocol.Unsubscribe: + events.unsubscribeClient(clientID) + _ = write(protocol.Message{Type: protocol.Unsubscribe, ID: request.ID}) + return true + } + return false +} diff --git a/internal/server/runner.go b/internal/server/runner.go index 304f89b0..53059da6 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -43,7 +43,10 @@ type runner struct { // verdict generates the bounded one-line reason for a completed stage visit // from its output map. When nil the store derives the reason at read time with // no model call. It is best-effort: any failure must never fail the stage. - verdict func(context.Context, map[string]any) string + verdict func(context.Context, map[string]any) string + // events pushes run state deltas to subscribed clients. When nil (tests that + // build a runner directly) emits are dropped, so nothing else changes. + events *runEventHub wg sync.WaitGroup schedulerWG sync.WaitGroup } @@ -241,6 +244,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru return nil, err } if created { + // Emit the created delta before launching: the queued frame reaches + // subscribers before the running/step-started transition, so a client + // sees the run appear and then advance. + r.emitRun(protocol.RunEventFrame{ + RunID: id, + Workspace: workspace, + Status: protocol.RunStatusQueued, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + }, false) r.launch(id, workspace, func(ctx context.Context) { r.run(ctx, id, d, workspace, request) }) @@ -376,7 +388,25 @@ func (r *runner) cancel(id int64) error { } // A run that is waiting on a person has no live goroutine, so cancel it at // the persistence layer instead. This keeps cancel working across a restart. - return r.store.CancelWaiting(r.ctx, id) + // The single follow-up read seeds the terminal delta; cancel is a cold + // operator action, not a per-transition hot path. + if err := r.store.CancelWaiting(r.ctx, id); err != nil { + return err + } + if r.events != nil { + if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil { + r.emitRun(protocol.RunEventFrame{ + RunID: id, + Workspace: detail.Workspace, + Status: detail.Status, + Outcome: detail.Outcome, + CurrentStage: detail.CurrentStage, + CompletedSteps: detail.CompletedSteps, + FinishedAt: detail.FinishedAt, + }, true) + } + } + return nil } // decide records an operator's choice for a waiting human stage and resumes the @@ -415,6 +445,23 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { break } } + // The decision closes the waiting step and returns the run to running; + // report that before the resumed loop emits its own deltas. + r.emitRun(protocol.RunEventFrame{ + RunID: detail.ID, + Workspace: detail.Workspace, + Status: protocol.RunStatusRunning, + CurrentStage: step.Stage, + CompletedSteps: completedSteps(detail.Steps), + Step: &protocol.RunStepDelta{ + Sequence: step.Sequence, + Stage: step.Stage, + Status: step.Status, + Outcome: step.Outcome, + Target: step.Target, + Diagnostic: step.Diagnostic, + }, + }, false) r.launch(detail.ID, detail.Workspace, func(ctx context.Context) { r.resume(ctx, detail.ID, detail, detail.Workspace, step) }) @@ -426,7 +473,7 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) { original, copyErr := copyValues(stage.Values(request.Input)) if copyErr != nil { - r.finishRunState(id, "failed", "error", copyErr.Error(), stage.Values(request.Input)) + r.finishRunState(id, workspace, "failed", "error", copyErr.Error(), 0, "", stage.Values(request.Input)) return } input := stage.Values(request.Input) @@ -462,7 +509,7 @@ func terminalFinish(target, stageKey, outcome, diagnostic string) (status, final func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) { // The decision may route straight to a terminal, which has no stage to run. if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal { - r.finishRunState(id, status, outcome, diagnostic, stage.Values(step.Output)) + r.finishRunState(id, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Stage, stage.Values(step.Output)) return } state := rebuildRunState(d, step) @@ -478,6 +525,9 @@ type runState struct { visits map[string]int memory map[string]memoryValue input stage.Values + // completed counts the run's finished steps, so a delta can report the + // running total without re-reading the store. + completed int } func newRunState(key string, input stage.Values) *runState { @@ -498,22 +548,54 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS if values, err := copyValues(stage.Values(step.Output)); err == nil { state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } + state.completed++ } } return state } -func (r *runner) finishRunState(id int64, status, outcome, diagnostic string, input stage.Values) { +// emitRun pushes one run delta to the subscribed-client hub when one is wired, +// and does nothing otherwise (a runner built directly in a test has no hub). +// terminal marks a final transition, which the hub flushes without coalescing. +func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) { + if r.events == nil { + return + } + r.events.emit(frame, terminal) +} + +func (r *runner) finishRunState(id int64, workspace, status, outcome, diagnostic string, completed int, currentStage string, input stage.Values) { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil { log.Printf("finish run %d: %v", id, err) } + r.emitRun(protocol.RunEventFrame{ + RunID: id, + Workspace: workspace, + Status: status, + Outcome: outcome, + CurrentStage: currentStage, + CompletedSteps: completed, + FinishedAt: time.Now().UTC().Format(time.RFC3339), + }, true) +} + +// completedSteps counts the completed steps of a run detail; resume uses it to +// seed the terminal delta without another store read. +func completedSteps(steps []protocol.RunStep) int { + n := 0 + for _, step := range steps { + if step.Status == "completed" { + n++ + } + } + return n } func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) { finish := func(status, outcome, diagnostic string) { - r.finishRunState(id, status, outcome, diagnostic, state.input) + r.finishRunState(id, workspace, status, outcome, diagnostic, state.completed, state.key, state.input) } defer func() { if recovered := recover(); recovered != nil { @@ -604,6 +686,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na finish("failed", "error", err.Error()) return } + // queued -> running: report the new current stage and the step start. + r.emitRun(protocol.RunEventFrame{ + RunID: id, + Workspace: workspace, + Status: protocol.RunStatusRunning, + CurrentStage: key, + CompletedSteps: state.completed, + Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, + }, false) if isHuman { // A human step is O(1): one BeginStep plus one MarkWaiting, no // filediff capture, no artifacts directory and no workspace mutation. @@ -617,6 +708,16 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na return } waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence} + // A human step parks the run without a live goroutine; the delta is + // coalesced by the hub's timer, which keeps running after this returns. + r.emitRun(protocol.RunEventFrame{ + RunID: id, + Workspace: workspace, + Status: protocol.RunStatusWaiting, + CurrentStage: key, + CompletedSteps: state.completed, + Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting}, + }, false) r.notifyStep(id, name, workspace, selected.Notify, waitStep) return } @@ -799,7 +900,27 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na return } state.memory[key] = memoryValue{output: values, sequence: sequence} + state.completed++ } + // Step finished: report its outcome, routing target, diagnostic and the + // artifacts recorded for the visit, plus the run's running step count. + r.emitRun(protocol.RunEventFrame{ + RunID: id, + Workspace: workspace, + Status: protocol.RunStatusRunning, + CurrentStage: key, + CompletedSteps: state.completed, + Artifacts: len(step.Artifacts), + Step: &protocol.RunStepDelta{ + Sequence: step.Sequence, + Stage: step.Stage, + Status: step.Status, + Outcome: step.Outcome, + Target: step.Target, + Diagnostic: step.Diagnostic, + Artifacts: len(step.Artifacts), + }, + }, false) r.notifyStep(id, name, workspace, selected.Notify, step) state.input = stage.Values(step.Output) if useAgent || errors.Is(err, stage.ErrStageCleanup) { diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 2c6389ec..c668c9b3 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -55,11 +55,15 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin build := buildinfo.Current() started := time.Now() hub := newChatHub() + events := newRunEventHub() mux := http.NewServeMux() var runs *runner var runErr error if len(stores) > 0 { runs, runErr = newRunner(ctx, stores[0]) + if runErr == nil { + runs.events = events + } if runErr == nil && options.ArtifactsRoot != "" { runs.artifactsRoot = options.ArtifactsRoot } @@ -131,6 +135,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin // clientID is this connection's identity for chat session claims: a // second client is refused a session another connection holds. clientID := chatClientToken() + // Drop every run subscription this connection holds when the read loop + // returns, so no subscriber, goroutine or timer outlives the connection. + defer events.unsubscribeClient(clientID) if len(stores) > 0 { items, err := stores[0].List(ctx) if err != nil { @@ -150,6 +157,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if handleChat(connCtx, request, options.Chat, hub, owned, clientID, write) { continue } + if handleRunEvents(request, events, runs, clientID, write) { + continue + } response := protocol.Message{Type: protocol.Pong, ID: request.ID} if request.Type == protocol.List || request.Type == protocol.Submit { response.NotificationEndpoints = notificationEndpoints()