Files
pipelines/internal/server/runevents.go
T
alan cff6f4dfac server: write a subscribe snapshot in full before its ack
handleRunEvents seeded a new subscription by pushing every matching run
through the subscription channel (runEventSubscriberBuffer = 32) and only
then acked. sendMessage is a non-blocking send that closes the
subscription when the channel is full, so a snapshot larger than the
buffer dropped the subscription it was seeding: the remaining snapshot
frames hit the closed-done early return, the ack was never sent, and no
later delta could arrive. The connection stayed open, so the client could
not tell it was no longer subscribed. This was the app's only
subscription - AppViewModel subscribes with a blank workspace, matching
every run - so the app's live state froze.

The subscription now starts in a snapshot phase. The subscribing
connection itself writes the snapshot frames via writeSnapshot (one
direct write per frame, outside the bounded channel, so the drop policy
cannot fire on them) and the ack, and ready then moves any deltas held
meanwhile into the channel in order and starts the writer goroutine.
sendMessage holds deltas in a bounded pending queue while snapshotting
instead of closing, so the drop-a-slow-subscriber policy still applies,
but only to deltas. Ordering is snapshot, ack, then deltas; a write
failure or a listRuns error closes the subscription instead of leaving
it registered.

Numbers reported by this run: the probe against the deployed server saw
blank-scope frames=2 ack=0 (scoped frames=1 ack=0) because the snapshot
was ~252 frames into a 32-frame buffer; the scratch-server probe with the
fix saw 40 snapshot frames, ack=1 and a delta after a run transition.
Guarded now by TestRunEventSubscribeSnapshotLargerThanBuffer (seeds 40
finished runs, asserts every snapshot frame, the ack strictly after them,
and a terminal delta for a run started after subscribing), which fails
against the previous runevents.go.

Checked for staleness: the packaged templates and the packaged authoring
skill references internal/author/skill/references/ carry no run-event
subscription contract (their "snapshot" mentions are the workspace-change
snapshots in operations.md), and README carries none either, so none of
them needs updating. No UI surface changed.

Checks run before committing: gofmt -l . prints nothing, go build ./...,
go vet ./..., go test ./... (all packages ok), go test -race
./internal/server (ok).
2026-10-18 02:23:58 +01:00

536 lines
15 KiB
Go

package server
import (
"sync"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// The run-event hub pushes compact run state deltas to subscribed connections so
// a client no longer has to poll. It mirrors the chat hub's shape: subscriptions
// live on the hub, keyed by connection, and every write goes through the
// connection-scoped write closure. Unlike chat, a run transition has no session
// to stream from, so deltas are emitted by the runner and coalesced here.
const (
// runEventFlushInterval throttles a busy run's pushes to at most a few frames
// per second. A terminal delta is flushed immediately regardless, so the final
// state is never delayed or dropped.
runEventFlushInterval = 220 * time.Millisecond
// runEventSubscriberBuffer bounds one subscription's pending frames. A
// subscriber that falls this far behind is dropped instead of blocking the
// producer or other subscribers; it re-syncs from a fresh subscribe snapshot.
runEventSubscriberBuffer = 32
// runEventMaxSubscriptionsPerClient bounds one connection's subscriptions so a
// client cannot grow the registry without limit.
runEventMaxSubscriptionsPerClient = 16
// runEventMaxCoalescers bounds the per-run coalescer registry; idle
// coalescers are evicted once it is exceeded.
runEventMaxCoalescers = 512
)
// runEventHub tracks the connections subscribed to run deltas and one coalescer
// per active run. All shared maps are guarded by mu. Nothing here performs a
// database read: the snapshot is supplied by the caller and every delta carries
// values already computed at the emit site.
type runEventHub struct {
mu sync.Mutex
subs map[*runSubscription]struct{}
byClient map[string]map[*runSubscription]struct{}
runs map[int64]*runCoalescer
now func() time.Time
interval time.Duration
}
// runCoalescer merges a run's rapid field updates into the latest pending delta
// and flushes at most once per interval. A terminal delta bypasses the timer.
type runCoalescer struct {
runID int64
workspace string
pending *protocol.RunEventFrame
timer *time.Timer
lastFlush time.Time
updated time.Time
}
// runSubscription is one connection's watch over a workspace (and optionally
// explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow
// or broken subscriber never stalls the producer, and its buffered channel
// bounds how far behind it may fall.
//
// A subscription starts in its snapshot phase: the subscribing connection
// writes the snapshot and the ack itself (writeSnapshot) and only then starts
// the writer goroutine (ready). mu guards that phase, so the drop-a-slow-
// subscriber policy can never fire on the snapshot or lose its ack: while
// snapshotting a delta is held in pending instead of the bounded channel, and
// nothing writes to the channel until ready has released the held frames.
type runSubscription struct {
hub *runEventHub
clientID string
workspace string
runIDs map[int64]struct{}
ch chan protocol.Message
done chan struct{}
once sync.Once
write func(protocol.Message) error
mu sync.Mutex
snapshotting bool
pending []protocol.Message
}
func newRunEventHub() *runEventHub {
return &runEventHub{
subs: map[*runSubscription]struct{}{},
byClient: map[string]map[*runSubscription]struct{}{},
runs: map[int64]*runCoalescer{},
now: time.Now,
interval: runEventFlushInterval,
}
}
func (h *runEventHub) clock() time.Time {
if h.now != nil {
return h.now()
}
return time.Now()
}
// subscribe registers one connection's watch and returns it in its snapshot
// phase: the caller delivers the snapshot and the ack with writeSnapshot and
// then calls ready, which releases the deltas emitted meanwhile and starts the
// writer goroutine. It returns nil when the connection already holds the
// per-client subscription cap.
func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription {
if h == nil || write == nil {
return nil
}
h.mu.Lock()
if len(h.byClient[clientID]) >= runEventMaxSubscriptionsPerClient {
h.mu.Unlock()
return nil
}
sub := &runSubscription{
hub: h,
clientID: clientID,
workspace: workspace,
ch: make(chan protocol.Message, runEventSubscriberBuffer),
done: make(chan struct{}),
write: write,
snapshotting: true,
}
for _, id := range runIDs {
if id <= 0 {
continue
}
if sub.runIDs == nil {
sub.runIDs = map[int64]struct{}{}
}
sub.runIDs[id] = struct{}{}
}
h.subs[sub] = struct{}{}
if h.byClient[clientID] == nil {
h.byClient[clientID] = map[*runSubscription]struct{}{}
}
h.byClient[clientID][sub] = struct{}{}
h.mu.Unlock()
// The writer goroutine starts only in ready, after the snapshot and the ack
// are on the wire, so a delta can never overtake them.
return sub
}
// unsubscribeClient drops every subscription held by one connection; the read
// loop calls it when the connection ends so nothing leaks.
func (h *runEventHub) unsubscribeClient(clientID string) {
if h == nil {
return
}
h.mu.Lock()
var subs []*runSubscription
for sub := range h.byClient[clientID] {
subs = append(subs, sub)
}
h.mu.Unlock()
for _, sub := range subs {
sub.close()
}
}
func (h *runEventHub) remove(sub *runSubscription) {
h.mu.Lock()
delete(h.subs, sub)
if set := h.byClient[sub.clientID]; set != nil {
delete(set, sub)
if len(set) == 0 {
delete(h.byClient, sub.clientID)
}
}
h.releaseIdleLocked()
h.mu.Unlock()
}
// emit folds one transition into its run's coalescer and schedules the flush.
// A terminal transition flushes any pending delta first, then the terminal frame
// immediately, so a client always observes the final state as its own frame.
func (h *runEventHub) emit(frame protocol.RunEventFrame, terminal bool) {
if h == nil || frame.RunID <= 0 {
return
}
h.mu.Lock()
if !h.anySubscriberLocked(frame.Workspace) {
h.mu.Unlock()
return
}
c := h.runs[frame.RunID]
if c == nil {
h.evictIdleLocked()
c = &runCoalescer{runID: frame.RunID}
h.runs[frame.RunID] = c
}
c.workspace = frame.Workspace
now := h.clock()
c.updated = now
var out []protocol.RunEventFrame
if terminal {
if c.pending != nil {
out = append(out, *c.pending)
c.pending = nil
}
out = append(out, frame)
h.stopTimerLocked(c)
c.lastFlush = now
} else {
if c.pending == nil {
pending := frame
c.pending = &pending
} else {
mergeRunEventFrame(c.pending, &frame)
}
if c.timer == nil {
if wait := h.interval - now.Sub(c.lastFlush); wait <= 0 {
out = append(out, *c.pending)
c.pending = nil
c.lastFlush = now
} else {
c.timer = time.AfterFunc(wait, func() { h.flushRun(frame.RunID) })
}
}
}
h.mu.Unlock()
for _, f := range out {
h.deliver(f)
}
}
func (h *runEventHub) flushRun(runID int64) {
h.mu.Lock()
c := h.runs[runID]
if c == nil {
h.mu.Unlock()
return
}
c.timer = nil
var out []protocol.RunEventFrame
if c.pending != nil {
out = append(out, *c.pending)
c.pending = nil
c.lastFlush = h.clock()
c.updated = c.lastFlush
}
h.mu.Unlock()
for _, f := range out {
h.deliver(f)
}
}
// deliver fans one frame out to the matching subscriptions. It collects the
// targets under the lock and hands them off outside it, so a slow write cannot
// hold the hub and stall the producer.
func (h *runEventHub) deliver(frame protocol.RunEventFrame) {
h.mu.Lock()
targets := make([]*runSubscription, 0, len(h.subs))
for sub := range h.subs {
if sub.matches(frame.Workspace, frame.RunID) {
targets = append(targets, sub)
}
}
h.mu.Unlock()
for _, sub := range targets {
sub.send(frame)
}
}
func (h *runEventHub) anySubscriberLocked(workspace string) bool {
for sub := range h.subs {
if sub.workspace == "" || sub.workspace == workspace {
return true
}
}
return false
}
// releaseIdleLocked drops every coalescer whose workspace no longer has a
// subscriber, stopping its timer, so the registry is bounded and released on
// unsubscribe/disconnect.
func (h *runEventHub) releaseIdleLocked() {
for id, c := range h.runs {
if h.anySubscriberLocked(c.workspace) {
continue
}
h.stopTimerLocked(c)
delete(h.runs, id)
}
}
// evictIdleLocked keeps the coalescer registry bounded by dropping the
// least-recently-updated idle coalescer once the cap is reached. It never evicts
// a coalescer with pending work or a running timer.
func (h *runEventHub) evictIdleLocked() {
for len(h.runs) >= runEventMaxCoalescers {
var oldest *runCoalescer
for _, c := range h.runs {
if c.pending != nil || c.timer != nil {
continue
}
if oldest == nil || c.updated.Before(oldest.updated) {
oldest = c
}
}
if oldest == nil {
return
}
h.stopTimerLocked(oldest)
delete(h.runs, oldest.runID)
}
}
func (h *runEventHub) stopTimerLocked(c *runCoalescer) {
if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (s *runSubscription) matches(workspace string, runID int64) bool {
if s.workspace != "" && s.workspace != workspace {
return false
}
if len(s.runIDs) > 0 {
_, ok := s.runIDs[runID]
return ok
}
return true
}
func (s *runSubscription) run() {
for {
select {
case <-s.done:
return
case msg := <-s.ch:
if err := s.write(msg); err != nil {
s.close()
return
}
}
}
}
// writeSnapshot writes one frame of the subscription's snapshot directly on the
// subscribing connection. It runs only during the snapshot phase, before the
// writer goroutine exists, so it cannot reorder against a delta and is never
// subject to the bounded channel's drop policy. It returns false when the frame
// could not be written, having closed the subscription; the caller must then
// stop writing the snapshot.
func (s *runSubscription) writeSnapshot(msg protocol.Message) bool {
select {
case <-s.done:
return false
default:
}
if err := s.write(msg); err != nil {
s.close()
return false
}
return true
}
// ready ends the snapshot phase: the deltas held while the snapshot was written
// are moved into the channel in order and the writer goroutine starts. The
// channel is empty until then (sendMessage queues into pending) and pending is
// bounded by the channel's capacity, so the move cannot block. A subscription
// closed during its snapshot starts no goroutine at all.
func (s *runSubscription) ready() {
s.mu.Lock()
select {
case <-s.done:
s.mu.Unlock()
return
default:
}
for _, msg := range s.pending {
s.ch <- msg
}
s.pending = nil
s.snapshotting = false
s.mu.Unlock()
go s.run()
}
func (s *runSubscription) send(frame protocol.RunEventFrame) {
s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame})
}
func (s *runSubscription) sendMessage(msg protocol.Message) {
select {
case <-s.done:
return
default:
}
s.mu.Lock()
if s.snapshotting {
// The snapshot is still being written on the subscribing connection and
// no writer goroutine is draining yet, so closing here would truncate a
// snapshot in flight and swallow its ack. Hold the delta instead, bounded
// the same as the channel: the policy can only ever affect deltas, since
// the snapshot and the ack are written directly by the subscriber.
if len(s.pending) >= runEventSubscriberBuffer {
s.mu.Unlock()
s.close()
return
}
s.pending = append(s.pending, msg)
s.mu.Unlock()
return
}
s.mu.Unlock()
select {
case s.ch <- msg:
default:
// The subscriber is not draining: drop it rather than block the
// producer. Its next subscribe re-syncs from a fresh snapshot.
s.close()
}
}
// close unregisters the subscription and stops its writer goroutine exactly
// once. It is safe to call from the writer goroutine, from unsubscribe and from
// the disconnect path.
func (s *runSubscription) close() {
s.once.Do(func() {
close(s.done)
s.hub.remove(s)
})
}
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.Workspace != "" {
into.Workspace = in.Workspace
}
// Identity is fixed for a run's life, so the first frame that carries it wins
// and a later delta never blanks it.
if into.PipelineID == 0 {
into.PipelineID = in.PipelineID
}
if into.Name == "" {
into.Name = in.Name
}
if into.Title == "" {
into.Title = in.Title
}
if into.Version == 0 {
into.Version = in.Version
}
if in.Status != "" {
into.Status = in.Status
}
if in.Outcome != "" {
into.Outcome = in.Outcome
}
if in.CurrentStage != "" {
into.CurrentStage = in.CurrentStage
}
if in.CompletedSteps > into.CompletedSteps {
into.CompletedSteps = in.CompletedSteps
}
if in.CreatedAt != "" {
into.CreatedAt = in.CreatedAt
}
if in.FinishedAt != "" {
into.FinishedAt = in.FinishedAt
}
if in.Artifacts > into.Artifacts {
into.Artifacts = in.Artifacts
}
if in.Step != nil {
into.Step = in.Step
}
}
// handleRunEvents dispatches one subscribe/unsubscribe request and reports
// whether it was a run-event type (so the caller never falls through to the
// generic unknown-command response). Subscribe registers the watch, writes the
// current state of the watched runs as its snapshot, acks, and only then lets
// deltas flow: the snapshot and the ack go straight out on the subscribing
// connection, so a snapshot larger than the subscription's buffer can no longer
// drop the subscription it is seeding, and the ordering is always snapshot,
// ack, deltas.
func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool {
switch request.Type {
case protocol.Subscribe:
if runs == nil || events == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "run events unavailable: no store"})
return true
}
sub := events.subscribe(clientID, request.Workspace, request.RunIDs, write)
if sub == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "too many run subscriptions on this connection"})
return true
}
// One bounded read seeds the subscription. It reuses listRuns, the same
// workspace-scoped read the polling clients use, instead of a new query.
summaries, err := runs.listRuns()
if err != nil {
sub.close()
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return true
}
for _, summary := range summaries {
if !sub.matches(summary.Workspace, summary.ID) {
continue
}
frame := protocol.RunEventFrame{
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
// read loop will stop on its next read error, so there is nothing
// left to reply to.
return true
}
}
if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil {
sub.close()
return true
}
sub.ready()
return true
case protocol.Unsubscribe:
events.unsubscribeClient(clientID)
_ = write(protocol.Message{Type: protocol.Unsubscribe, ID: request.ID})
return true
}
return false
}