8f3036bd3b
Add a server-pushed run_event frame so a client no longer has to poll once a second to notice run changes. - protocol: RunEventFrame/RunStepDelta compact delta (run id, workspace, status, outcome, current stage, completed-step count, timestamps and an optional single step delta), plus subscribe/unsubscribe/run_event types and the optional RunEvent/RunIDs message fields. All fields optional so older peers ignore them. - server: runEventHub with per-connection subscriptions scoped to a workspace (and optional explicit run ids), one bounded writer goroutine per subscription, slow-subscriber drop, per-client subscription cap and a ~220ms per-run coalescer that flushes pending deltas before a terminal frame. - runner: emit deltas on created/queued, queued->running and step start, human waiting, step finish (outcome, target, diagnostic, artifacts and completed count), terminal finish, cancel of a waiting run and decide/resume; emitRun is a no-op without a hub. - websocket: subscribe seeds current state before the ack, unsubscribe and connection teardown drop subscriptions, and pushed frames reuse the connection write closure. - tests: protocol golden-JSON wire shape plus websocket coverage for the started-run frame sequence, subscribe current-state snapshot, silent unsubscribed/never-subscribed clients, workspace isolation and a bounded coalescing frame count. - docs: note subscribe/unsubscribe/run_event in docs/cli.md; TUI and Android still poll in this release.
430 lines
12 KiB
Go
430 lines
12 KiB
Go
package server
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// The run-event hub pushes compact run state deltas to subscribed connections so
|
|
// a client no longer has to poll. It mirrors the chat hub's shape: subscriptions
|
|
// live on the hub, keyed by connection, and every write goes through the
|
|
// connection-scoped write closure. Unlike chat, a run transition has no session
|
|
// to stream from, so deltas are emitted by the runner and coalesced here.
|
|
const (
|
|
// runEventFlushInterval throttles a busy run's pushes to at most a few frames
|
|
// per second. A terminal delta is flushed immediately regardless, so the final
|
|
// state is never delayed or dropped.
|
|
runEventFlushInterval = 220 * time.Millisecond
|
|
// runEventSubscriberBuffer bounds one subscription's pending frames. A
|
|
// subscriber that falls this far behind is dropped instead of blocking the
|
|
// producer or other subscribers; it re-syncs from a fresh subscribe snapshot.
|
|
runEventSubscriberBuffer = 32
|
|
// runEventMaxSubscriptionsPerClient bounds one connection's subscriptions so a
|
|
// client cannot grow the registry without limit.
|
|
runEventMaxSubscriptionsPerClient = 16
|
|
// runEventMaxCoalescers bounds the per-run coalescer registry; idle
|
|
// coalescers are evicted once it is exceeded.
|
|
runEventMaxCoalescers = 512
|
|
)
|
|
|
|
// runEventHub tracks the connections subscribed to run deltas and one coalescer
|
|
// per active run. All shared maps are guarded by mu. Nothing here performs a
|
|
// database read: the snapshot is supplied by the caller and every delta carries
|
|
// values already computed at the emit site.
|
|
type runEventHub struct {
|
|
mu sync.Mutex
|
|
subs map[*runSubscription]struct{}
|
|
byClient map[string]map[*runSubscription]struct{}
|
|
runs map[int64]*runCoalescer
|
|
now func() time.Time
|
|
interval time.Duration
|
|
}
|
|
|
|
// runCoalescer merges a run's rapid field updates into the latest pending delta
|
|
// and flushes at most once per interval. A terminal delta bypasses the timer.
|
|
type runCoalescer struct {
|
|
runID int64
|
|
workspace string
|
|
pending *protocol.RunEventFrame
|
|
timer *time.Timer
|
|
lastFlush time.Time
|
|
updated time.Time
|
|
}
|
|
|
|
// runSubscription is one connection's watch over a workspace (and optionally
|
|
// explicit run ids). Frames are handed to a dedicated writer goroutine so a slow
|
|
// or broken subscriber never stalls the producer, and its buffered channel
|
|
// bounds how far behind it may fall.
|
|
type runSubscription struct {
|
|
hub *runEventHub
|
|
clientID string
|
|
workspace string
|
|
runIDs map[int64]struct{}
|
|
ch chan protocol.Message
|
|
done chan struct{}
|
|
once sync.Once
|
|
write func(protocol.Message) error
|
|
}
|
|
|
|
func newRunEventHub() *runEventHub {
|
|
return &runEventHub{
|
|
subs: map[*runSubscription]struct{}{},
|
|
byClient: map[string]map[*runSubscription]struct{}{},
|
|
runs: map[int64]*runCoalescer{},
|
|
now: time.Now,
|
|
interval: runEventFlushInterval,
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) clock() time.Time {
|
|
if h.now != nil {
|
|
return h.now()
|
|
}
|
|
return time.Now()
|
|
}
|
|
|
|
// subscribe registers one connection's watch and starts its writer goroutine. It
|
|
// returns nil when the connection already holds the per-client subscription cap.
|
|
func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription {
|
|
if h == nil || write == nil {
|
|
return nil
|
|
}
|
|
h.mu.Lock()
|
|
if len(h.byClient[clientID]) >= runEventMaxSubscriptionsPerClient {
|
|
h.mu.Unlock()
|
|
return nil
|
|
}
|
|
sub := &runSubscription{
|
|
hub: h,
|
|
clientID: clientID,
|
|
workspace: workspace,
|
|
ch: make(chan protocol.Message, runEventSubscriberBuffer),
|
|
done: make(chan struct{}),
|
|
write: write,
|
|
}
|
|
for _, id := range runIDs {
|
|
if id <= 0 {
|
|
continue
|
|
}
|
|
if sub.runIDs == nil {
|
|
sub.runIDs = map[int64]struct{}{}
|
|
}
|
|
sub.runIDs[id] = struct{}{}
|
|
}
|
|
h.subs[sub] = struct{}{}
|
|
if h.byClient[clientID] == nil {
|
|
h.byClient[clientID] = map[*runSubscription]struct{}{}
|
|
}
|
|
h.byClient[clientID][sub] = struct{}{}
|
|
h.mu.Unlock()
|
|
go sub.run()
|
|
return sub
|
|
}
|
|
|
|
// unsubscribeClient drops every subscription held by one connection; the read
|
|
// loop calls it when the connection ends so nothing leaks.
|
|
func (h *runEventHub) unsubscribeClient(clientID string) {
|
|
if h == nil {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
var subs []*runSubscription
|
|
for sub := range h.byClient[clientID] {
|
|
subs = append(subs, sub)
|
|
}
|
|
h.mu.Unlock()
|
|
for _, sub := range subs {
|
|
sub.close()
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) remove(sub *runSubscription) {
|
|
h.mu.Lock()
|
|
delete(h.subs, sub)
|
|
if set := h.byClient[sub.clientID]; set != nil {
|
|
delete(set, sub)
|
|
if len(set) == 0 {
|
|
delete(h.byClient, sub.clientID)
|
|
}
|
|
}
|
|
h.releaseIdleLocked()
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
// emit folds one transition into its run's coalescer and schedules the flush.
|
|
// A terminal transition flushes any pending delta first, then the terminal frame
|
|
// immediately, so a client always observes the final state as its own frame.
|
|
func (h *runEventHub) emit(frame protocol.RunEventFrame, terminal bool) {
|
|
if h == nil || frame.RunID <= 0 {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
if !h.anySubscriberLocked(frame.Workspace) {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
c := h.runs[frame.RunID]
|
|
if c == nil {
|
|
h.evictIdleLocked()
|
|
c = &runCoalescer{runID: frame.RunID}
|
|
h.runs[frame.RunID] = c
|
|
}
|
|
c.workspace = frame.Workspace
|
|
now := h.clock()
|
|
c.updated = now
|
|
var out []protocol.RunEventFrame
|
|
if terminal {
|
|
if c.pending != nil {
|
|
out = append(out, *c.pending)
|
|
c.pending = nil
|
|
}
|
|
out = append(out, frame)
|
|
h.stopTimerLocked(c)
|
|
c.lastFlush = now
|
|
} else {
|
|
if c.pending == nil {
|
|
pending := frame
|
|
c.pending = &pending
|
|
} else {
|
|
mergeRunEventFrame(c.pending, &frame)
|
|
}
|
|
if c.timer == nil {
|
|
if wait := h.interval - now.Sub(c.lastFlush); wait <= 0 {
|
|
out = append(out, *c.pending)
|
|
c.pending = nil
|
|
c.lastFlush = now
|
|
} else {
|
|
c.timer = time.AfterFunc(wait, func() { h.flushRun(frame.RunID) })
|
|
}
|
|
}
|
|
}
|
|
h.mu.Unlock()
|
|
for _, f := range out {
|
|
h.deliver(f)
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) flushRun(runID int64) {
|
|
h.mu.Lock()
|
|
c := h.runs[runID]
|
|
if c == nil {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
c.timer = nil
|
|
var out []protocol.RunEventFrame
|
|
if c.pending != nil {
|
|
out = append(out, *c.pending)
|
|
c.pending = nil
|
|
c.lastFlush = h.clock()
|
|
c.updated = c.lastFlush
|
|
}
|
|
h.mu.Unlock()
|
|
for _, f := range out {
|
|
h.deliver(f)
|
|
}
|
|
}
|
|
|
|
// deliver fans one frame out to the matching subscriptions. It collects the
|
|
// targets under the lock and hands them off outside it, so a slow write cannot
|
|
// hold the hub and stall the producer.
|
|
func (h *runEventHub) deliver(frame protocol.RunEventFrame) {
|
|
h.mu.Lock()
|
|
targets := make([]*runSubscription, 0, len(h.subs))
|
|
for sub := range h.subs {
|
|
if sub.matches(frame.Workspace, frame.RunID) {
|
|
targets = append(targets, sub)
|
|
}
|
|
}
|
|
h.mu.Unlock()
|
|
for _, sub := range targets {
|
|
sub.send(frame)
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) anySubscriberLocked(workspace string) bool {
|
|
for sub := range h.subs {
|
|
if sub.workspace == "" || sub.workspace == workspace {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// releaseIdleLocked drops every coalescer whose workspace no longer has a
|
|
// subscriber, stopping its timer, so the registry is bounded and released on
|
|
// unsubscribe/disconnect.
|
|
func (h *runEventHub) releaseIdleLocked() {
|
|
for id, c := range h.runs {
|
|
if h.anySubscriberLocked(c.workspace) {
|
|
continue
|
|
}
|
|
h.stopTimerLocked(c)
|
|
delete(h.runs, id)
|
|
}
|
|
}
|
|
|
|
// evictIdleLocked keeps the coalescer registry bounded by dropping the
|
|
// least-recently-updated idle coalescer once the cap is reached. It never evicts
|
|
// a coalescer with pending work or a running timer.
|
|
func (h *runEventHub) evictIdleLocked() {
|
|
for len(h.runs) >= runEventMaxCoalescers {
|
|
var oldest *runCoalescer
|
|
for _, c := range h.runs {
|
|
if c.pending != nil || c.timer != nil {
|
|
continue
|
|
}
|
|
if oldest == nil || c.updated.Before(oldest.updated) {
|
|
oldest = c
|
|
}
|
|
}
|
|
if oldest == nil {
|
|
return
|
|
}
|
|
h.stopTimerLocked(oldest)
|
|
delete(h.runs, oldest.runID)
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) stopTimerLocked(c *runCoalescer) {
|
|
if c.timer != nil {
|
|
c.timer.Stop()
|
|
c.timer = nil
|
|
}
|
|
}
|
|
|
|
func (s *runSubscription) matches(workspace string, runID int64) bool {
|
|
if s.workspace != "" && s.workspace != workspace {
|
|
return false
|
|
}
|
|
if len(s.runIDs) > 0 {
|
|
_, ok := s.runIDs[runID]
|
|
return ok
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (s *runSubscription) run() {
|
|
for {
|
|
select {
|
|
case <-s.done:
|
|
return
|
|
case msg := <-s.ch:
|
|
if err := s.write(msg); err != nil {
|
|
s.close()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *runSubscription) send(frame protocol.RunEventFrame) {
|
|
s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame})
|
|
}
|
|
|
|
func (s *runSubscription) sendMessage(msg protocol.Message) {
|
|
select {
|
|
case <-s.done:
|
|
return
|
|
default:
|
|
}
|
|
select {
|
|
case s.ch <- msg:
|
|
default:
|
|
// The subscriber is not draining: drop it rather than block the
|
|
// producer. Its next subscribe re-syncs from a fresh snapshot.
|
|
s.close()
|
|
}
|
|
}
|
|
|
|
// close unregisters the subscription and stops its writer goroutine exactly
|
|
// once. It is safe to call from the writer goroutine, from unsubscribe and from
|
|
// the disconnect path.
|
|
func (s *runSubscription) close() {
|
|
s.once.Do(func() {
|
|
close(s.done)
|
|
s.hub.remove(s)
|
|
})
|
|
}
|
|
|
|
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
|
if in.Workspace != "" {
|
|
into.Workspace = in.Workspace
|
|
}
|
|
if in.Status != "" {
|
|
into.Status = in.Status
|
|
}
|
|
if in.Outcome != "" {
|
|
into.Outcome = in.Outcome
|
|
}
|
|
if in.CurrentStage != "" {
|
|
into.CurrentStage = in.CurrentStage
|
|
}
|
|
if in.CompletedSteps > into.CompletedSteps {
|
|
into.CompletedSteps = in.CompletedSteps
|
|
}
|
|
if in.CreatedAt != "" {
|
|
into.CreatedAt = in.CreatedAt
|
|
}
|
|
if in.FinishedAt != "" {
|
|
into.FinishedAt = in.FinishedAt
|
|
}
|
|
if in.Artifacts > into.Artifacts {
|
|
into.Artifacts = in.Artifacts
|
|
}
|
|
if in.Step != nil {
|
|
into.Step = in.Step
|
|
}
|
|
}
|
|
|
|
// handleRunEvents dispatches one subscribe/unsubscribe request and reports
|
|
// whether it was a run-event type (so the caller never falls through to the
|
|
// generic unknown-command response). Subscribe registers the watch, sends the
|
|
// current state of the watched runs through the subscription, then acks; the
|
|
// snapshot therefore precedes every later delta.
|
|
func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool {
|
|
switch request.Type {
|
|
case protocol.Subscribe:
|
|
if runs == nil || events == nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "run events unavailable: no store"})
|
|
return true
|
|
}
|
|
sub := events.subscribe(clientID, request.Workspace, request.RunIDs, write)
|
|
if sub == nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "too many run subscriptions on this connection"})
|
|
return true
|
|
}
|
|
// One bounded read seeds the subscription. It reuses listRuns, the same
|
|
// workspace-scoped read the polling clients use, instead of a new query.
|
|
summaries, err := runs.listRuns()
|
|
if err != nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
|
return true
|
|
}
|
|
for _, summary := range summaries {
|
|
if !sub.matches(summary.Workspace, summary.ID) {
|
|
continue
|
|
}
|
|
sub.send(protocol.RunEventFrame{
|
|
RunID: summary.ID,
|
|
Workspace: summary.Workspace,
|
|
Status: summary.Status,
|
|
Outcome: summary.Outcome,
|
|
CurrentStage: summary.CurrentStage,
|
|
CompletedSteps: summary.CompletedSteps,
|
|
CreatedAt: summary.CreatedAt,
|
|
FinishedAt: summary.FinishedAt,
|
|
})
|
|
}
|
|
sub.sendMessage(protocol.Message{Type: protocol.Subscribe, ID: request.ID})
|
|
return true
|
|
case protocol.Unsubscribe:
|
|
events.unsubscribeClient(clientID)
|
|
_ = write(protocol.Message{Type: protocol.Unsubscribe, ID: request.ID})
|
|
return true
|
|
}
|
|
return false
|
|
}
|