Push run state to subscribed websocket clients

Add a server-pushed run_event frame so a client no longer has to poll
once a second to notice run changes.

- protocol: RunEventFrame/RunStepDelta compact delta (run id, workspace,
  status, outcome, current stage, completed-step count, timestamps and an
  optional single step delta), plus subscribe/unsubscribe/run_event types
  and the optional RunEvent/RunIDs message fields. All fields optional so
  older peers ignore them.
- server: runEventHub with per-connection subscriptions scoped to a
  workspace (and optional explicit run ids), one bounded writer goroutine
  per subscription, slow-subscriber drop, per-client subscription cap and
  a ~220ms per-run coalescer that flushes pending deltas before a terminal
  frame.
- runner: emit deltas on created/queued, queued->running and step start,
  human waiting, step finish (outcome, target, diagnostic, artifacts and
  completed count), terminal finish, cancel of a waiting run and
  decide/resume; emitRun is a no-op without a hub.
- websocket: subscribe seeds current state before the ack, unsubscribe and
  connection teardown drop subscriptions, and pushed frames reuse the
  connection write closure.
- tests: protocol golden-JSON wire shape plus websocket coverage for the
  started-run frame sequence, subscribe current-state snapshot, silent
  unsubscribed/never-subscribed clients, workspace isolation and a
  bounded coalescing frame count.
- docs: note subscribe/unsubscribe/run_event in docs/cli.md; TUI and
  Android still poll in this release.
This commit is contained in:
2026-10-17 00:43:05 +01:00
parent 889d811858
commit 8f3036bd3b
9 changed files with 1025 additions and 6 deletions
+10
View File
@@ -348,6 +348,16 @@ or Tab to return to details. When not focused, the graph follows the latest visi
or selected stage. Existing polling refreshes the graph once per second, and
running durations show elapsed time rather than “in progress.”
The server can also push run changes. A client sends `subscribe` scoped to a
workspace (and optionally to explicit run ids); the server first returns the
current state of the matching runs, then a compact `run_event` delta on each
transition — which run changed, its status, outcome, current stage, completed-step
count and timestamps, and, when a single step changed, that step's sequence, stage,
status, outcome, target and diagnostic. It is a delta, never the whole run, and
rapid changes are coalesced to at most a few frames per second per run.
`unsubscribe` or disconnecting stops the frames. This lets a client drop the
once-per-second poll; the TUI and Android app still poll in this release.
In the run inspector, `n`/`p` select the next/previous recorded execution visit,
including repeated visits in loops, from any pane. Selection stops at the last
visit or overview; it does not wrap. The selected stage has a full-row highlight;
+5
View File
@@ -164,4 +164,9 @@ type Message struct {
ChatEvent *ChatEventFrame `json:"chat_event,omitempty"`
ChatPermission *ChatPermissionFrame `json:"chat_permission,omitempty"`
ChatReply *ChatReplyRequest `json:"chat_reply,omitempty"`
// RunEvent is the server-pushed run state delta (see runevents.go) and
// RunIDs the optional subscribe scope, so a client asks for the runs it
// displays. Both are optional so older peers ignore them.
RunEvent *RunEventFrame `json:"run_event,omitempty"`
RunIDs []int64 `json:"run_ids,omitempty"`
}
+48
View File
@@ -0,0 +1,48 @@
package protocol
// Server-pushed run state. A client sends subscribe (scoped to the workspace and
// optionally to explicit run ids) to ask for the runs it displays: the server
// first returns the current state of those runs so a reconnecting client
// re-syncs, then pushes a compact run_event delta on every meaningful
// transition. It sends unsubscribe to stop. Like chat_event, run_event is a
// server-pushed frame with no matching request, so both fields are optional and
// older peers ignore them.
const (
Subscribe = "subscribe"
Unsubscribe = "unsubscribe"
RunEvent = "run_event"
)
// RunEventFrame is the server-pushed run_event delta. It carries only the values
// that changed, never the whole run: the run's identity, status, outcome, current
// stage, completed-step count and timestamps, plus at most one changed step.
// Every field is optional so an older peer ignores what it does not know and a
// client can fold the frame into its cached run.
type RunEventFrame struct {
RunID int64 `json:"run_id"`
Workspace string `json:"workspace,omitempty"`
Status string `json:"status,omitempty"`
Outcome string `json:"outcome,omitempty"`
CurrentStage string `json:"current_stage,omitempty"`
CompletedSteps int `json:"completed_steps,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
FinishedAt string `json:"finished_at,omitempty"`
// Artifacts is the number of files recorded for the referenced step, so a
// client can surface artifact capture without reading the step.
Artifacts int `json:"artifacts,omitempty"`
// Step is present when a single step changed (started, finished or waiting).
Step *RunStepDelta `json:"step,omitempty"`
}
// RunStepDelta is the compact change of one step within a RunEventFrame. It is a
// delta, not a RunStep: it omits the input/output, transcript, file changes and
// artifact bytes a full step carries.
type RunStepDelta struct {
Sequence int `json:"sequence"`
Stage string `json:"stage,omitempty"`
Status string `json:"status,omitempty"`
Outcome string `json:"outcome,omitempty"`
Target string `json:"target,omitempty"`
Diagnostic string `json:"diagnostic,omitempty"`
Artifacts int `json:"artifacts,omitempty"`
}
+65
View File
@@ -0,0 +1,65 @@
package protocol
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
// TestRunEventGoldenJSON pins the exact wire shape of the pushed run_event frame
// so a JSON tag change cannot silently alter what a client parses.
func TestRunEventGoldenJSON(t *testing.T) {
frame := RunEventFrame{
RunID: 7, Workspace: "/work", Status: RunStatusRunning, CurrentStage: "build",
CompletedSteps: 1, CreatedAt: "2026-01-02T03:04:05Z", Artifacts: 2,
Step: &RunStepDelta{Sequence: 2, Stage: "build", Status: "completed", Outcome: "success", Target: SuccessEnd, Artifacts: 2},
}
encoded, err := json.Marshal(frame)
if err != nil {
t.Fatal(err)
}
want := `{"run_id":7,"workspace":"/work","status":"running","current_stage":"build","completed_steps":1,"created_at":"2026-01-02T03:04:05Z","artifacts":2,"step":{"sequence":2,"stage":"build","status":"completed","outcome":"success","target":"#success","artifacts":2}}`
if string(encoded) != want {
t.Fatalf("golden mismatch\n got %s\nwant %s", encoded, want)
}
var decoded RunEventFrame
if err := json.Unmarshal([]byte(want), &decoded); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(decoded, frame) {
t.Fatalf("golden did not decode back: %+v", decoded)
}
}
// TestRunEventEnvelopeRoundTrip proves the new optional fields survive the shared
// Message envelope and that an unused payload is omitted from the wire.
func TestRunEventEnvelopeRoundTrip(t *testing.T) {
message := Message{
Type: RunEvent,
ID: "r1",
RunEvent: &RunEventFrame{RunID: 3, Workspace: "/w", Status: RunStatusQueued, CreatedAt: "2026-01-02T03:04:05Z"},
RunIDs: []int64{3, 4},
}
encoded, err := json.Marshal(message)
if err != nil {
t.Fatal(err)
}
var decoded Message
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(decoded.RunEvent, message.RunEvent) || !reflect.DeepEqual(decoded.RunIDs, message.RunIDs) {
t.Fatalf("run event fields lost: %+v", decoded)
}
minimal, err := json.Marshal(Message{Type: Subscribe, ID: "s1", Workspace: "/w"})
if err != nil {
t.Fatal(err)
}
for _, absent := range []string{"run_event", "run_ids"} {
if strings.Contains(string(minimal), absent) {
t.Fatalf("absent %q must be omitted: %s", absent, minimal)
}
}
}
+2
View File
@@ -21,8 +21,10 @@ const (
// the run and its step persist these statuses and the decision path resumes
// them. Store, server, CLI and TUI read the one pair below, never a literal.
const (
RunStatusQueued = "queued"
RunStatusWaiting = "waiting"
RunStatusRunning = "running"
StepStatusRunning = "running"
StepStatusWaiting = "waiting"
// NotifyWaiting is the notification event a human stage emits when it parks
// the run. It is the same value the `on=` policy accepts.
+329
View File
@@ -0,0 +1,329 @@
package server
import (
"context"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
// runEventClient is a thin websocket client for the run-event tests: it dials
// /connect, consumes the unsolicited welcome, and reads/writes protocol
// envelopes.
type runEventClient struct {
t *testing.T
ctx context.Context
conn *websocket.Conn
}
func dialRunEvents(t *testing.T, ctx context.Context, handler *runHandler) *runEventClient {
t.Helper()
srv := httptest.NewServer(handler)
t.Cleanup(srv.Close)
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { conn.CloseNow() })
var welcome protocol.Message
if err := wsjson.Read(ctx, conn, &welcome); err != nil {
t.Fatal(err)
}
return &runEventClient{t: t, ctx: ctx, conn: conn}
}
func (c *runEventClient) send(message protocol.Message) {
c.t.Helper()
if err := wsjson.Write(c.ctx, c.conn, message); err != nil {
c.t.Fatal(err)
}
}
func (c *runEventClient) read() protocol.Message {
c.t.Helper()
var message protocol.Message
if err := wsjson.Read(c.ctx, c.conn, &message); err != nil {
c.t.Fatal(err)
}
return message
}
// eventStore starts a fresh store holding one pipeline template and a handler
// wired to it, with the execute seam returning a deterministic success.
func eventStore(t *testing.T, ctx context.Context, stages ...string) (*pipeline.Store, *runHandler, []int64) {
t.Helper()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "events.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { store.Close() })
if _, err := store.Submit(ctx, "events", pipeline.Template(stages, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
var ids []int64
for _, item := range items {
ids = append(ids, item.ID)
}
handler := Handler(ctx, store).(*runHandler)
t.Cleanup(func() { WaitRuns(handler) })
return store, handler, ids
}
// eventWorkspace returns a temporary directory canonicalised the same way the
// runner canonicalises a run workspace (EvalSymlinks). A subscription must use
// this resolved path to match: t.TempDir() alone is a symlinked /var path on
// macOS and would never equal the run's /private/var workspace, so no frame
// would be delivered.
func eventWorkspace(t *testing.T) string {
t.Helper()
dir, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
return dir
}
// TestRunEventWebSocketPushesTransitions starts a run, subscribes to its
// workspace, and proves the client observes the created, queued->running /
// step-started, step-finished (with the right outcome and target) and terminal
// frames, all carrying the run id.
func TestRunEventWebSocketPushesTransitions(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
workspace := eventWorkspace(t)
release := make(chan struct{})
var releaseOnce sync.Once
handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "sub" {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "evt-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
var runID int64
var created, stepStarted, stepFinished, terminal *protocol.RunEventFrame
for terminal == nil {
message := client.read()
switch message.Type {
case protocol.RunStarted:
if message.RunDetail == nil {
t.Fatal("run_started without detail")
}
runID = message.RunDetail.ID
case protocol.RunEvent:
frame := message.RunEvent
if frame == nil {
t.Fatal("run_event without payload")
}
switch {
case frame.Step == nil && frame.Status == protocol.RunStatusQueued:
created = frame
case frame.Step != nil && frame.Step.Status == protocol.StepStatusRunning:
if stepStarted == nil {
stepStarted = frame
releaseOnce.Do(func() { close(release) })
}
case frame.Step != nil && frame.Step.Status == "completed":
stepFinished = frame
case frame.Status == "succeeded" || frame.Status == "failed" || frame.Status == "canceled":
terminal = frame
}
}
}
if runID == 0 {
t.Fatal("run_started never carried the run id")
}
if created == nil || created.RunID != runID || created.Step != nil || created.CreatedAt == "" {
t.Fatalf("created frame = %+v", created)
}
if stepStarted == nil || stepStarted.RunID != runID || stepStarted.Status != protocol.RunStatusRunning || stepStarted.CurrentStage != "one" || stepStarted.Step.Sequence != 1 || stepStarted.Step.Stage != "one" {
t.Fatalf("step-started frame = %+v", stepStarted)
}
if stepFinished == nil || stepFinished.RunID != runID || stepFinished.Step.Status != "completed" || stepFinished.Step.Outcome != "success" || stepFinished.Step.Target != protocol.SuccessEnd || stepFinished.CompletedSteps != 1 {
t.Fatalf("step-finished frame = %+v", stepFinished)
}
if terminal.RunID != runID || terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.FinishedAt == "" {
t.Fatalf("terminal frame = %+v", terminal)
}
}
// TestRunEventSubscribeSendsSnapshot proves a client that subscribes to an
// already-existing run receives its current state as a run_event frame before
// the subscribe ack, so a reconnecting client re-syncs without polling.
func TestRunEventSubscribeSendsSnapshot(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
run, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "snap-1", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
if err != nil {
t.Fatal(err)
}
handler.runner.wg.Wait()
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "snap", Workspace: workspace})
first := client.read()
if first.Type != protocol.RunEvent || first.RunEvent == nil {
t.Fatalf("first frame after subscribe = %+v, want a run_event snapshot", first)
}
if first.RunEvent.RunID != run.ID || first.RunEvent.Status != "succeeded" || first.RunEvent.CompletedSteps != 1 || first.RunEvent.FinishedAt == "" {
t.Fatalf("snapshot frame = %+v", first.RunEvent)
}
if ack := client.read(); ack.Type != protocol.Subscribe || ack.ID != "snap" {
t.Fatalf("subscribe ack = %+v", ack)
}
}
// assertNoRunEvent reads for d, ignoring responses, and fails if any run_event
// frame arrives.
func assertNoRunEvent(t *testing.T, client *runEventClient, d time.Duration) {
t.Helper()
readCtx, cancel := context.WithTimeout(client.ctx, d)
defer cancel()
for {
var message protocol.Message
if err := wsjson.Read(readCtx, client.conn, &message); err != nil {
return
}
if message.Type == protocol.RunEvent {
t.Fatalf("client received an unexpected run_event: %+v", message.RunEvent)
}
}
}
// TestRunEventUnsubscribeAndNonSubscriberReceiveNothing proves an unsubscribed
// client and a client that never subscribed receive no run_event frames.
func TestRunEventUnsubscribeAndNonSubscriberReceiveNothing(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
subscribed := dialRunEvents(t, ctx, handler)
unsubscribedWorkspace := eventWorkspace(t)
subscribed.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: unsubscribedWorkspace})
if ack := subscribed.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
subscribed.send(protocol.Message{Type: protocol.Unsubscribe, ID: "unsub"})
if ack := subscribed.read(); ack.Type != protocol.Unsubscribe || ack.ID != "unsub" {
t.Fatalf("unsubscribe ack = %+v", ack)
}
subscribed.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-a", PipelineID: ids[0], Version: 1, RequestKey: "after-unsub", Workspace: unsubscribedWorkspace, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, subscribed, 400*time.Millisecond)
stranger := dialRunEvents(t, ctx, handler)
strangerWorkspace := eventWorkspace(t)
stranger.send(protocol.Message{Type: protocol.RunPipeline, ID: "start-b", PipelineID: ids[0], Version: 1, RequestKey: "never-subscribed", Workspace: strangerWorkspace, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, stranger, 400*time.Millisecond)
}
// TestRunEventWorkspaceIsolation proves a subscriber to one workspace never
// receives a delta for a run in another workspace, and still receives its own.
func TestRunEventWorkspaceIsolation(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, handler, ids := eventStore(t, ctx, "one")
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
watched := eventWorkspace(t)
other := eventWorkspace(t)
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: watched})
if ack := client.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
// The watched workspace still delivers its own run's terminal frame.
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "mine", PipelineID: ids[0], Version: 1, RequestKey: "watched-ws", Workspace: watched, Input: map[string]any{"prompt": "test"}})
var terminal *protocol.RunEventFrame
var runID int64
for terminal == nil {
message := client.read()
if message.Type == protocol.RunStarted && message.RunDetail != nil {
runID = message.RunDetail.ID
}
if message.Type == protocol.RunEvent && message.RunEvent != nil &&
(message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled") {
terminal = message.RunEvent
}
}
if runID == 0 || terminal.RunID != runID || terminal.Workspace != watched {
t.Fatalf("watched run terminal = %+v (run=%d), want workspace %q", terminal, runID, watched)
}
// A run in another workspace must not leak. This assertion reads until a
// bounded deadline, and that deadline closes the connection, so it runs last.
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "other", PipelineID: ids[0], Version: 1, RequestKey: "other-ws", Workspace: other, Input: map[string]any{"prompt": "test"}})
handler.runner.wg.Wait()
assertNoRunEvent(t, client, 400*time.Millisecond)
}
// TestRunEventCoalescingBoundsFrameCount records how many run_event frames a
// typical multi-stage run produces and asserts the count stays bounded: rapid
// transitions coalesce, and the terminal frame is never dropped.
func TestRunEventCoalescingBoundsFrameCount(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stages := []string{"a", "b", "c"}
_, handler, ids := eventStore(t, ctx, stages...)
handler.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
client := dialRunEvents(t, ctx, handler)
client.send(protocol.Message{Type: protocol.Subscribe, ID: "sub", Workspace: workspace})
if ack := client.read(); ack.Type != protocol.Subscribe {
t.Fatalf("subscribe ack = %+v", ack)
}
client.send(protocol.Message{Type: protocol.RunPipeline, ID: "start", PipelineID: ids[0], Version: 1, RequestKey: "coalesce", Workspace: workspace, Input: map[string]any{"prompt": "test"}})
frames := 0
var terminal *protocol.RunEventFrame
for terminal == nil {
message := client.read()
if message.Type != protocol.RunEvent || message.RunEvent == nil {
continue
}
frames++
if message.RunEvent.Status == "succeeded" || message.RunEvent.Status == "failed" || message.RunEvent.Status == "canceled" {
terminal = message.RunEvent
}
}
bound := 2*len(stages) + 4
if frames > bound {
t.Fatalf("run produced %d run_event frames for %d stages, want at most %d", frames, len(stages), bound)
}
if terminal.Status != "succeeded" || terminal.Outcome != "success" || terminal.CompletedSteps != len(stages) {
t.Fatalf("terminal frame = %+v", terminal)
}
t.Logf("run_event frames for a %d-stage run: %d (bound %d)", len(stages), frames, bound)
}
+429
View File
@@ -0,0 +1,429 @@
package server
import (
"sync"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// The run-event hub pushes compact run state deltas to subscribed connections so
// a client no longer has to poll. It mirrors the chat hub's shape: subscriptions
// live on the hub, keyed by connection, and every write goes through the
// connection-scoped write closure. Unlike chat, a run transition has no session
// to stream from, so deltas are emitted by the runner and coalesced here.
const (
// runEventFlushInterval throttles a busy run's pushes to at most a few frames
// per second. A terminal delta is flushed immediately regardless, so the final
// state is never delayed or dropped.
runEventFlushInterval = 220 * time.Millisecond
// runEventSubscriberBuffer bounds one subscription's pending frames. A
// subscriber that falls this far behind is dropped instead of blocking the
// producer or other subscribers; it re-syncs from a fresh subscribe snapshot.
runEventSubscriberBuffer = 32
// runEventMaxSubscriptionsPerClient bounds one connection's subscriptions so a
// client cannot grow the registry without limit.
runEventMaxSubscriptionsPerClient = 16
// runEventMaxCoalescers bounds the per-run coalescer registry; idle
// coalescers are evicted once it is exceeded.
runEventMaxCoalescers = 512
)
// runEventHub tracks the connections subscribed to run deltas and one coalescer
// per active run. All shared maps are guarded by mu. Nothing here performs a
// database read: the snapshot is supplied by the caller and every delta carries
// values already computed at the emit site.
type runEventHub struct {
mu sync.Mutex
subs map[*runSubscription]struct{}
byClient map[string]map[*runSubscription]struct{}
runs map[int64]*runCoalescer
now func() time.Time
interval time.Duration
}
// runCoalescer merges a run's rapid field updates into the latest pending delta
// and flushes at most once per interval. A terminal delta bypasses the timer.
type runCoalescer struct {
runID int64
workspace string
pending *protocol.RunEventFrame
timer *time.Timer
lastFlush time.Time
updated time.Time
}
// runSubscription is one connection's watch over a workspace (and optionally
// explicit run ids). Frames are handed to a dedicated writer goroutine so a slow
// or broken subscriber never stalls the producer, and its buffered channel
// bounds how far behind it may fall.
type runSubscription struct {
hub *runEventHub
clientID string
workspace string
runIDs map[int64]struct{}
ch chan protocol.Message
done chan struct{}
once sync.Once
write func(protocol.Message) error
}
func newRunEventHub() *runEventHub {
return &runEventHub{
subs: map[*runSubscription]struct{}{},
byClient: map[string]map[*runSubscription]struct{}{},
runs: map[int64]*runCoalescer{},
now: time.Now,
interval: runEventFlushInterval,
}
}
func (h *runEventHub) clock() time.Time {
if h.now != nil {
return h.now()
}
return time.Now()
}
// subscribe registers one connection's watch and starts its writer goroutine. It
// returns nil when the connection already holds the per-client subscription cap.
func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription {
if h == nil || write == nil {
return nil
}
h.mu.Lock()
if len(h.byClient[clientID]) >= runEventMaxSubscriptionsPerClient {
h.mu.Unlock()
return nil
}
sub := &runSubscription{
hub: h,
clientID: clientID,
workspace: workspace,
ch: make(chan protocol.Message, runEventSubscriberBuffer),
done: make(chan struct{}),
write: write,
}
for _, id := range runIDs {
if id <= 0 {
continue
}
if sub.runIDs == nil {
sub.runIDs = map[int64]struct{}{}
}
sub.runIDs[id] = struct{}{}
}
h.subs[sub] = struct{}{}
if h.byClient[clientID] == nil {
h.byClient[clientID] = map[*runSubscription]struct{}{}
}
h.byClient[clientID][sub] = struct{}{}
h.mu.Unlock()
go sub.run()
return sub
}
// unsubscribeClient drops every subscription held by one connection; the read
// loop calls it when the connection ends so nothing leaks.
func (h *runEventHub) unsubscribeClient(clientID string) {
if h == nil {
return
}
h.mu.Lock()
var subs []*runSubscription
for sub := range h.byClient[clientID] {
subs = append(subs, sub)
}
h.mu.Unlock()
for _, sub := range subs {
sub.close()
}
}
func (h *runEventHub) remove(sub *runSubscription) {
h.mu.Lock()
delete(h.subs, sub)
if set := h.byClient[sub.clientID]; set != nil {
delete(set, sub)
if len(set) == 0 {
delete(h.byClient, sub.clientID)
}
}
h.releaseIdleLocked()
h.mu.Unlock()
}
// emit folds one transition into its run's coalescer and schedules the flush.
// A terminal transition flushes any pending delta first, then the terminal frame
// immediately, so a client always observes the final state as its own frame.
func (h *runEventHub) emit(frame protocol.RunEventFrame, terminal bool) {
if h == nil || frame.RunID <= 0 {
return
}
h.mu.Lock()
if !h.anySubscriberLocked(frame.Workspace) {
h.mu.Unlock()
return
}
c := h.runs[frame.RunID]
if c == nil {
h.evictIdleLocked()
c = &runCoalescer{runID: frame.RunID}
h.runs[frame.RunID] = c
}
c.workspace = frame.Workspace
now := h.clock()
c.updated = now
var out []protocol.RunEventFrame
if terminal {
if c.pending != nil {
out = append(out, *c.pending)
c.pending = nil
}
out = append(out, frame)
h.stopTimerLocked(c)
c.lastFlush = now
} else {
if c.pending == nil {
pending := frame
c.pending = &pending
} else {
mergeRunEventFrame(c.pending, &frame)
}
if c.timer == nil {
if wait := h.interval - now.Sub(c.lastFlush); wait <= 0 {
out = append(out, *c.pending)
c.pending = nil
c.lastFlush = now
} else {
c.timer = time.AfterFunc(wait, func() { h.flushRun(frame.RunID) })
}
}
}
h.mu.Unlock()
for _, f := range out {
h.deliver(f)
}
}
func (h *runEventHub) flushRun(runID int64) {
h.mu.Lock()
c := h.runs[runID]
if c == nil {
h.mu.Unlock()
return
}
c.timer = nil
var out []protocol.RunEventFrame
if c.pending != nil {
out = append(out, *c.pending)
c.pending = nil
c.lastFlush = h.clock()
c.updated = c.lastFlush
}
h.mu.Unlock()
for _, f := range out {
h.deliver(f)
}
}
// deliver fans one frame out to the matching subscriptions. It collects the
// targets under the lock and hands them off outside it, so a slow write cannot
// hold the hub and stall the producer.
func (h *runEventHub) deliver(frame protocol.RunEventFrame) {
h.mu.Lock()
targets := make([]*runSubscription, 0, len(h.subs))
for sub := range h.subs {
if sub.matches(frame.Workspace, frame.RunID) {
targets = append(targets, sub)
}
}
h.mu.Unlock()
for _, sub := range targets {
sub.send(frame)
}
}
func (h *runEventHub) anySubscriberLocked(workspace string) bool {
for sub := range h.subs {
if sub.workspace == "" || sub.workspace == workspace {
return true
}
}
return false
}
// releaseIdleLocked drops every coalescer whose workspace no longer has a
// subscriber, stopping its timer, so the registry is bounded and released on
// unsubscribe/disconnect.
func (h *runEventHub) releaseIdleLocked() {
for id, c := range h.runs {
if h.anySubscriberLocked(c.workspace) {
continue
}
h.stopTimerLocked(c)
delete(h.runs, id)
}
}
// evictIdleLocked keeps the coalescer registry bounded by dropping the
// least-recently-updated idle coalescer once the cap is reached. It never evicts
// a coalescer with pending work or a running timer.
func (h *runEventHub) evictIdleLocked() {
for len(h.runs) >= runEventMaxCoalescers {
var oldest *runCoalescer
for _, c := range h.runs {
if c.pending != nil || c.timer != nil {
continue
}
if oldest == nil || c.updated.Before(oldest.updated) {
oldest = c
}
}
if oldest == nil {
return
}
h.stopTimerLocked(oldest)
delete(h.runs, oldest.runID)
}
}
func (h *runEventHub) stopTimerLocked(c *runCoalescer) {
if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (s *runSubscription) matches(workspace string, runID int64) bool {
if s.workspace != "" && s.workspace != workspace {
return false
}
if len(s.runIDs) > 0 {
_, ok := s.runIDs[runID]
return ok
}
return true
}
func (s *runSubscription) run() {
for {
select {
case <-s.done:
return
case msg := <-s.ch:
if err := s.write(msg); err != nil {
s.close()
return
}
}
}
}
func (s *runSubscription) send(frame protocol.RunEventFrame) {
s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame})
}
func (s *runSubscription) sendMessage(msg protocol.Message) {
select {
case <-s.done:
return
default:
}
select {
case s.ch <- msg:
default:
// The subscriber is not draining: drop it rather than block the
// producer. Its next subscribe re-syncs from a fresh snapshot.
s.close()
}
}
// close unregisters the subscription and stops its writer goroutine exactly
// once. It is safe to call from the writer goroutine, from unsubscribe and from
// the disconnect path.
func (s *runSubscription) close() {
s.once.Do(func() {
close(s.done)
s.hub.remove(s)
})
}
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.Workspace != "" {
into.Workspace = in.Workspace
}
if in.Status != "" {
into.Status = in.Status
}
if in.Outcome != "" {
into.Outcome = in.Outcome
}
if in.CurrentStage != "" {
into.CurrentStage = in.CurrentStage
}
if in.CompletedSteps > into.CompletedSteps {
into.CompletedSteps = in.CompletedSteps
}
if in.CreatedAt != "" {
into.CreatedAt = in.CreatedAt
}
if in.FinishedAt != "" {
into.FinishedAt = in.FinishedAt
}
if in.Artifacts > into.Artifacts {
into.Artifacts = in.Artifacts
}
if in.Step != nil {
into.Step = in.Step
}
}
// handleRunEvents dispatches one subscribe/unsubscribe request and reports
// whether it was a run-event type (so the caller never falls through to the
// generic unknown-command response). Subscribe registers the watch, sends the
// current state of the watched runs through the subscription, then acks; the
// snapshot therefore precedes every later delta.
func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool {
switch request.Type {
case protocol.Subscribe:
if runs == nil || events == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "run events unavailable: no store"})
return true
}
sub := events.subscribe(clientID, request.Workspace, request.RunIDs, write)
if sub == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "too many run subscriptions on this connection"})
return true
}
// One bounded read seeds the subscription. It reuses listRuns, the same
// workspace-scoped read the polling clients use, instead of a new query.
summaries, err := runs.listRuns()
if err != nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return true
}
for _, summary := range summaries {
if !sub.matches(summary.Workspace, summary.ID) {
continue
}
sub.send(protocol.RunEventFrame{
RunID: summary.ID,
Workspace: summary.Workspace,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
})
}
sub.sendMessage(protocol.Message{Type: protocol.Subscribe, ID: request.ID})
return true
case protocol.Unsubscribe:
events.unsubscribeClient(clientID)
_ = write(protocol.Message{Type: protocol.Unsubscribe, ID: request.ID})
return true
}
return false
}
+127 -6
View File
@@ -43,7 +43,10 @@ type runner struct {
// verdict generates the bounded one-line reason for a completed stage visit
// from its output map. When nil the store derives the reason at read time with
// no model call. It is best-effort: any failure must never fail the stage.
verdict func(context.Context, map[string]any) string
verdict func(context.Context, map[string]any) string
// events pushes run state deltas to subscribed clients. When nil (tests that
// build a runner directly) emits are dropped, so nothing else changes.
events *runEventHub
wg sync.WaitGroup
schedulerWG sync.WaitGroup
}
@@ -241,6 +244,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
return nil, err
}
if created {
// Emit the created delta before launching: the queued frame reaches
// subscribers before the running/step-started transition, so a client
// sees the run appear and then advance.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusQueued,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}, false)
r.launch(id, workspace, func(ctx context.Context) {
r.run(ctx, id, d, workspace, request)
})
@@ -376,7 +388,25 @@ func (r *runner) cancel(id int64) error {
}
// A run that is waiting on a person has no live goroutine, so cancel it at
// the persistence layer instead. This keeps cancel working across a restart.
return r.store.CancelWaiting(r.ctx, id)
// The single follow-up read seeds the terminal delta; cancel is a cold
// operator action, not a per-transition hot path.
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
return err
}
if r.events != nil {
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: detail.Workspace,
Status: detail.Status,
Outcome: detail.Outcome,
CurrentStage: detail.CurrentStage,
CompletedSteps: detail.CompletedSteps,
FinishedAt: detail.FinishedAt,
}, true)
}
}
return nil
}
// decide records an operator's choice for a waiting human stage and resumes the
@@ -415,6 +445,23 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
break
}
}
// The decision closes the waiting step and returns the run to running;
// report that before the resumed loop emits its own deltas.
r.emitRun(protocol.RunEventFrame{
RunID: detail.ID,
Workspace: detail.Workspace,
Status: protocol.RunStatusRunning,
CurrentStage: step.Stage,
CompletedSteps: completedSteps(detail.Steps),
Step: &protocol.RunStepDelta{
Sequence: step.Sequence,
Stage: step.Stage,
Status: step.Status,
Outcome: step.Outcome,
Target: step.Target,
Diagnostic: step.Diagnostic,
},
}, false)
r.launch(detail.ID, detail.Workspace, func(ctx context.Context) {
r.resume(ctx, detail.ID, detail, detail.Workspace, step)
})
@@ -426,7 +473,7 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
r.finishRunState(id, "failed", "error", copyErr.Error(), stage.Values(request.Input))
r.finishRunState(id, workspace, "failed", "error", copyErr.Error(), 0, "", stage.Values(request.Input))
return
}
input := stage.Values(request.Input)
@@ -462,7 +509,7 @@ func terminalFinish(target, stageKey, outcome, diagnostic string) (status, final
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
// The decision may route straight to a terminal, which has no stage to run.
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
r.finishRunState(id, status, outcome, diagnostic, stage.Values(step.Output))
r.finishRunState(id, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Stage, stage.Values(step.Output))
return
}
state := rebuildRunState(d, step)
@@ -478,6 +525,9 @@ type runState struct {
visits map[string]int
memory map[string]memoryValue
input stage.Values
// completed counts the run's finished steps, so a delta can report the
// running total without re-reading the store.
completed int
}
func newRunState(key string, input stage.Values) *runState {
@@ -498,22 +548,54 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
state.completed++
}
}
return state
}
func (r *runner) finishRunState(id int64, status, outcome, diagnostic string, input stage.Values) {
// emitRun pushes one run delta to the subscribed-client hub when one is wired,
// and does nothing otherwise (a runner built directly in a test has no hub).
// terminal marks a final transition, which the hub flushes without coalescing.
func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
if r.events == nil {
return
}
r.events.emit(frame, terminal)
}
func (r *runner) finishRunState(id int64, workspace, status, outcome, diagnostic string, completed int, currentStage string, input stage.Values) {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
log.Printf("finish run %d: %v", id, err)
}
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: status,
Outcome: outcome,
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
}, true)
}
// completedSteps counts the completed steps of a run detail; resume uses it to
// seed the terminal delta without another store read.
func completedSteps(steps []protocol.RunStep) int {
n := 0
for _, step := range steps {
if step.Status == "completed" {
n++
}
}
return n
}
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
finish := func(status, outcome, diagnostic string) {
r.finishRunState(id, status, outcome, diagnostic, state.input)
r.finishRunState(id, workspace, status, outcome, diagnostic, state.completed, state.key, state.input)
}
defer func() {
if recovered := recover(); recovered != nil {
@@ -604,6 +686,15 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
finish("failed", "error", err.Error())
return
}
// queued -> running: report the new current stage and the step start.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
}, false)
if isHuman {
// A human step is O(1): one BeginStep plus one MarkWaiting, no
// filediff capture, no artifacts directory and no workspace mutation.
@@ -617,6 +708,16 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
return
}
waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence}
// A human step parks the run without a live goroutine; the delta is
// coalesced by the hub's timer, which keeps running after this returns.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusWaiting,
CurrentStage: key,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
}, false)
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
return
}
@@ -799,7 +900,27 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
return
}
state.memory[key] = memoryValue{output: values, sequence: sequence}
state.completed++
}
// Step finished: report its outcome, routing target, diagnostic and the
// artifacts recorded for the visit, plus the run's running step count.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
CompletedSteps: state.completed,
Artifacts: len(step.Artifacts),
Step: &protocol.RunStepDelta{
Sequence: step.Sequence,
Stage: step.Stage,
Status: step.Status,
Outcome: step.Outcome,
Target: step.Target,
Diagnostic: step.Diagnostic,
Artifacts: len(step.Artifacts),
},
}, false)
r.notifyStep(id, name, workspace, selected.Notify, step)
state.input = stage.Values(step.Output)
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
+10
View File
@@ -55,11 +55,15 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
build := buildinfo.Current()
started := time.Now()
hub := newChatHub()
events := newRunEventHub()
mux := http.NewServeMux()
var runs *runner
var runErr error
if len(stores) > 0 {
runs, runErr = newRunner(ctx, stores[0])
if runErr == nil {
runs.events = events
}
if runErr == nil && options.ArtifactsRoot != "" {
runs.artifactsRoot = options.ArtifactsRoot
}
@@ -131,6 +135,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
// clientID is this connection's identity for chat session claims: a
// second client is refused a session another connection holds.
clientID := chatClientToken()
// Drop every run subscription this connection holds when the read loop
// returns, so no subscriber, goroutine or timer outlives the connection.
defer events.unsubscribeClient(clientID)
if len(stores) > 0 {
items, err := stores[0].List(ctx)
if err != nil {
@@ -150,6 +157,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
if handleChat(connCtx, request, options.Chat, hub, owned, clientID, write) {
continue
}
if handleRunEvents(request, events, runs, clientID, write) {
continue
}
response := protocol.Message{Type: protocol.Pong, ID: request.ID}
if request.Type == protocol.List || request.Type == protocol.Submit {
response.NotificationEndpoints = notificationEndpoints()