package server import ( "sync" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) // The run-event hub pushes compact run state deltas to subscribed connections so // a client no longer has to poll. It mirrors the chat hub's shape: subscriptions // live on the hub, keyed by connection, and every write goes through the // connection-scoped write closure. Unlike chat, a run transition has no session // to stream from, so deltas are emitted by the runner and coalesced here. const ( // runEventFlushInterval throttles a busy run's pushes to at most a few frames // per second. A terminal delta is flushed immediately regardless, so the final // state is never delayed or dropped. runEventFlushInterval = 220 * time.Millisecond // runEventSubscriberBuffer bounds one subscription's pending frames. A // subscriber that falls this far behind is dropped instead of blocking the // producer or other subscribers; it re-syncs from a fresh subscribe snapshot. runEventSubscriberBuffer = 32 // runEventMaxSubscriptionsPerClient bounds one connection's subscriptions so a // client cannot grow the registry without limit. runEventMaxSubscriptionsPerClient = 16 // runEventMaxCoalescers bounds the per-run coalescer registry; idle // coalescers are evicted once it is exceeded. runEventMaxCoalescers = 512 ) // runEventHub tracks the connections subscribed to run deltas and one coalescer // per active run. All shared maps are guarded by mu. Nothing here performs a // database read: the snapshot is supplied by the caller and every delta carries // values already computed at the emit site. type runEventHub struct { mu sync.Mutex subs map[*runSubscription]struct{} byClient map[string]map[*runSubscription]struct{} runs map[int64]*runCoalescer now func() time.Time interval time.Duration } // runCoalescer merges a run's rapid field updates into the latest pending delta // and flushes at most once per interval. A terminal delta bypasses the timer. type runCoalescer struct { runID int64 workspace string pending *protocol.RunEventFrame timer *time.Timer lastFlush time.Time updated time.Time } // runSubscription is one connection's watch over a workspace (and optionally // explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow // or broken subscriber never stalls the producer, and its buffered channel // bounds how far behind it may fall. // // A subscription starts in its snapshot phase: the subscribing connection // writes the snapshot and the ack itself (writeSnapshot) and only then starts // the writer goroutine (ready). mu guards that phase, so the drop-a-slow- // subscriber policy can never fire on the snapshot or lose its ack: while // snapshotting a delta is held in pending instead of the bounded channel, and // nothing writes to the channel until ready has released the held frames. type runSubscription struct { hub *runEventHub clientID string workspace string runIDs map[int64]struct{} ch chan protocol.Message done chan struct{} once sync.Once write func(protocol.Message) error mu sync.Mutex snapshotting bool pending []protocol.Message } func newRunEventHub() *runEventHub { return &runEventHub{ subs: map[*runSubscription]struct{}{}, byClient: map[string]map[*runSubscription]struct{}{}, runs: map[int64]*runCoalescer{}, now: time.Now, interval: runEventFlushInterval, } } func (h *runEventHub) clock() time.Time { if h.now != nil { return h.now() } return time.Now() } // subscribe registers one connection's watch and returns it in its snapshot // phase: the caller delivers the snapshot and the ack with writeSnapshot and // then calls ready, which releases the deltas emitted meanwhile and starts the // writer goroutine. It returns nil when the connection already holds the // per-client subscription cap. func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription { if h == nil || write == nil { return nil } h.mu.Lock() if len(h.byClient[clientID]) >= runEventMaxSubscriptionsPerClient { h.mu.Unlock() return nil } sub := &runSubscription{ hub: h, clientID: clientID, workspace: workspace, ch: make(chan protocol.Message, runEventSubscriberBuffer), done: make(chan struct{}), write: write, snapshotting: true, } for _, id := range runIDs { if id <= 0 { continue } if sub.runIDs == nil { sub.runIDs = map[int64]struct{}{} } sub.runIDs[id] = struct{}{} } h.subs[sub] = struct{}{} if h.byClient[clientID] == nil { h.byClient[clientID] = map[*runSubscription]struct{}{} } h.byClient[clientID][sub] = struct{}{} h.mu.Unlock() // The writer goroutine starts only in ready, after the snapshot and the ack // are on the wire, so a delta can never overtake them. return sub } // unsubscribeClient drops every subscription held by one connection; the read // loop calls it when the connection ends so nothing leaks. func (h *runEventHub) unsubscribeClient(clientID string) { if h == nil { return } h.mu.Lock() var subs []*runSubscription for sub := range h.byClient[clientID] { subs = append(subs, sub) } h.mu.Unlock() for _, sub := range subs { sub.close() } } func (h *runEventHub) remove(sub *runSubscription) { h.mu.Lock() delete(h.subs, sub) if set := h.byClient[sub.clientID]; set != nil { delete(set, sub) if len(set) == 0 { delete(h.byClient, sub.clientID) } } h.releaseIdleLocked() h.mu.Unlock() } // emit folds one transition into its run's coalescer and schedules the flush. // A terminal transition flushes any pending delta first, then the terminal frame // immediately, so a client always observes the final state as its own frame. func (h *runEventHub) emit(frame protocol.RunEventFrame, terminal bool) { if step := frame.Step; step != nil { frame.GraphVisits = append(frame.GraphVisits, protocol.RunGraphVisit{Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target}) } if h == nil || frame.RunID <= 0 { return } h.mu.Lock() if !h.anySubscriberLocked(frame.Workspace) { h.mu.Unlock() return } c := h.runs[frame.RunID] if c == nil { h.evictIdleLocked() c = &runCoalescer{runID: frame.RunID} h.runs[frame.RunID] = c } c.workspace = frame.Workspace now := h.clock() c.updated = now var out []protocol.RunEventFrame if terminal { if c.pending != nil { out = append(out, *c.pending) c.pending = nil } out = append(out, frame) h.stopTimerLocked(c) c.lastFlush = now } else { if c.pending == nil { pending := frame c.pending = &pending } else { mergeRunEventFrame(c.pending, &frame) } if c.timer == nil { if wait := h.interval - now.Sub(c.lastFlush); wait <= 0 { out = append(out, *c.pending) c.pending = nil c.lastFlush = now } else { c.timer = time.AfterFunc(wait, func() { h.flushRun(frame.RunID) }) } } } h.mu.Unlock() for _, f := range out { h.deliver(f) } } func (h *runEventHub) flushRun(runID int64) { h.mu.Lock() c := h.runs[runID] if c == nil { h.mu.Unlock() return } c.timer = nil var out []protocol.RunEventFrame if c.pending != nil { out = append(out, *c.pending) c.pending = nil c.lastFlush = h.clock() c.updated = c.lastFlush } h.mu.Unlock() for _, f := range out { h.deliver(f) } } // deliver fans one frame out to the matching subscriptions. It collects the // targets under the lock and hands them off outside it, so a slow write cannot // hold the hub and stall the producer. func (h *runEventHub) deliver(frame protocol.RunEventFrame) { h.mu.Lock() targets := make([]*runSubscription, 0, len(h.subs)) for sub := range h.subs { if sub.matches(frame.Workspace, frame.RunID) { targets = append(targets, sub) } } h.mu.Unlock() for _, sub := range targets { sub.send(frame) } } func (h *runEventHub) anySubscriberLocked(workspace string) bool { for sub := range h.subs { if sub.workspace == "" || sub.workspace == workspace { return true } } return false } // releaseIdleLocked drops every coalescer whose workspace no longer has a // subscriber, stopping its timer, so the registry is bounded and released on // unsubscribe/disconnect. func (h *runEventHub) releaseIdleLocked() { for id, c := range h.runs { if h.anySubscriberLocked(c.workspace) { continue } h.stopTimerLocked(c) delete(h.runs, id) } } // evictIdleLocked keeps the coalescer registry bounded by dropping the // least-recently-updated idle coalescer once the cap is reached. It never evicts // a coalescer with pending work or a running timer. func (h *runEventHub) evictIdleLocked() { for len(h.runs) >= runEventMaxCoalescers { var oldest *runCoalescer for _, c := range h.runs { if c.pending != nil || c.timer != nil { continue } if oldest == nil || c.updated.Before(oldest.updated) { oldest = c } } if oldest == nil { return } h.stopTimerLocked(oldest) delete(h.runs, oldest.runID) } } func (h *runEventHub) stopTimerLocked(c *runCoalescer) { if c.timer != nil { c.timer.Stop() c.timer = nil } } func (s *runSubscription) matches(workspace string, runID int64) bool { if s.workspace != "" && s.workspace != workspace { return false } if len(s.runIDs) > 0 { _, ok := s.runIDs[runID] return ok } return true } func (s *runSubscription) run() { for { select { case <-s.done: return case msg := <-s.ch: if err := s.write(msg); err != nil { s.close() return } } } } // writeSnapshot writes one frame of the subscription's snapshot directly on the // subscribing connection. It runs only during the snapshot phase, before the // writer goroutine exists, so it cannot reorder against a delta and is never // subject to the bounded channel's drop policy. It returns false when the frame // could not be written, having closed the subscription; the caller must then // stop writing the snapshot. func (s *runSubscription) writeSnapshot(msg protocol.Message) bool { select { case <-s.done: return false default: } if err := s.write(msg); err != nil { s.close() return false } return true } // ready ends the snapshot phase: the deltas held while the snapshot was written // are moved into the channel in order and the writer goroutine starts. The // channel is empty until then (sendMessage queues into pending) and pending is // bounded by the channel's capacity, so the move cannot block. A subscription // closed during its snapshot starts no goroutine at all. func (s *runSubscription) ready() { s.mu.Lock() select { case <-s.done: s.mu.Unlock() return default: } for _, msg := range s.pending { s.ch <- msg } s.pending = nil s.snapshotting = false s.mu.Unlock() go s.run() } func (s *runSubscription) send(frame protocol.RunEventFrame) { s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) } func (s *runSubscription) sendMessage(msg protocol.Message) { select { case <-s.done: return default: } s.mu.Lock() if s.snapshotting { // The snapshot is still being written on the subscribing connection and // no writer goroutine is draining yet, so closing here would truncate a // snapshot in flight and swallow its ack. Hold the delta instead, bounded // the same as the channel: the policy can only ever affect deltas, since // the snapshot and the ack are written directly by the subscriber. if len(s.pending) >= runEventSubscriberBuffer { s.mu.Unlock() s.close() return } s.pending = append(s.pending, msg) s.mu.Unlock() return } s.mu.Unlock() select { case s.ch <- msg: default: // The subscriber is not draining: drop it rather than block the // producer. Its next subscribe re-syncs from a fresh snapshot. s.close() } } // close unregisters the subscription and stops its writer goroutine exactly // once. It is safe to call from the writer goroutine, from unsubscribe and from // the disconnect path. func (s *runSubscription) close() { s.once.Do(func() { close(s.done) s.hub.remove(s) }) } func mergeRunEventFrame(into, in *protocol.RunEventFrame) { if in.NotifyMode != "" { into.NotifyMode = in.NotifyMode } if in.Scheduled { into.Scheduled = true into.ScheduleID = in.ScheduleID into.ScheduleName = in.ScheduleName } into.CancelRequested = into.CancelRequested || in.CancelRequested into.CleanupUnconfirmed = into.CleanupUnconfirmed || in.CleanupUnconfirmed if in.Workspace != "" { into.Workspace = in.Workspace } // Identity is fixed for a run's life, so the first frame that carries it wins // and a later delta never blanks it. if into.PipelineID == 0 { into.PipelineID = in.PipelineID } if into.Name == "" { into.Name = in.Name } if into.Title == "" { into.Title = in.Title } if into.Version == 0 { into.Version = in.Version } if in.Status != "" { into.Status = in.Status into.ExportPendingSequence = in.ExportPendingSequence } if in.Outcome != "" { into.Outcome = in.Outcome } if in.CurrentStage != "" { into.CurrentStage = in.CurrentStage } if in.CompletedSteps > into.CompletedSteps { into.CompletedSteps = in.CompletedSteps } if in.CreatedAt != "" { into.CreatedAt = in.CreatedAt } if in.FinishedAt != "" { into.FinishedAt = in.FinishedAt } if in.Artifacts > into.Artifacts { into.Artifacts = in.Artifacts } if in.Step != nil { into.Step = in.Step } into.GraphVisits = append(into.GraphVisits, in.GraphVisits...) } // handleRunEvents dispatches one subscribe/unsubscribe request and reports // whether it was a run-event type (so the caller never falls through to the // generic unknown-command response). Subscribe registers the watch, writes the // current state of the watched runs as its snapshot, acks, and only then lets // deltas flow: the snapshot and the ack go straight out on the subscribing // connection, so a snapshot larger than the subscription's buffer can no longer // drop the subscription it is seeding, and the ordering is always snapshot, // ack, deltas. func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool { switch request.Type { case protocol.Subscribe: if runs == nil || events == nil { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "run events unavailable: no store"}) return true } sub := events.subscribe(clientID, request.Workspace, request.RunIDs, write) if sub == nil { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "too many run subscriptions on this connection"}) return true } // One bounded read seeds the subscription. It reuses listRuns, the same // workspace-scoped read the polling clients use, instead of a new query. summaries, err := runs.listRuns() if err != nil { sub.close() _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) return true } for _, summary := range summaries { if !sub.matches(summary.Workspace, summary.ID) { continue } frame := protocol.RunEventFrame{ Scheduled: summary.Scheduled, ScheduleID: summary.ScheduleID, ScheduleName: summary.ScheduleName, GraphEvidence: summary.GraphEvidence, NotifyMode: summary.NotifyMode, RunID: summary.ID, Workspace: summary.Workspace, PipelineID: summary.PipelineID, Name: summary.Name, Title: summary.Title, Version: summary.Version, Status: summary.Status, Outcome: summary.Outcome, CurrentStage: summary.CurrentStage, CompletedSteps: summary.CompletedSteps, CreatedAt: summary.CreatedAt, FinishedAt: summary.FinishedAt, CancelRequested: summary.CancelRequested, CleanupUnconfirmed: summary.CleanupUnconfirmed, ExportPendingSequence: summary.ExportPendingSequence, } if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) { // The connection write failed; the subscription is closed and the // read loop will stop on its next read error, so there is nothing // left to reply to. return true } } if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil { sub.close() return true } sub.ready() return true case protocol.Unsubscribe: events.unsubscribeClient(clientID) _ = write(protocol.Message{Type: protocol.Unsubscribe, ID: request.ID}) return true } return false }