31e89e872b
Every run summary carries size_bytes (artifact tree, server-created Pi session directories and an estimate of the run's database rows), cached in pipeline_runs.size_bytes, invalidated by triggers, measured when a run finishes and backfilled in the background. delete_runs removes terminal runs with every scoped row and server-owned file, refuses active ones with a reason, and pushes runs_removed to subscribers. pipelines runs rm is the CLI form; retention apply now shares the complete row deletion.
596 lines
17 KiB
Go
596 lines
17 KiB
Go
package server
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// The run-event hub pushes compact run state deltas to subscribed connections so
|
|
// a client no longer has to poll. It mirrors the chat hub's shape: subscriptions
|
|
// live on the hub, keyed by connection, and every write goes through the
|
|
// connection-scoped write closure. Unlike chat, a run transition has no session
|
|
// to stream from, so deltas are emitted by the runner and coalesced here.
|
|
const (
|
|
// runEventFlushInterval throttles a busy run's pushes to at most a few frames
|
|
// per second. A terminal delta is flushed immediately regardless, so the final
|
|
// state is never delayed or dropped.
|
|
runEventFlushInterval = 220 * time.Millisecond
|
|
// runEventSubscriberBuffer bounds one subscription's pending frames. A
|
|
// subscriber that falls this far behind is dropped instead of blocking the
|
|
// producer or other subscribers; it re-syncs from a fresh subscribe snapshot.
|
|
runEventSubscriberBuffer = 32
|
|
// runEventMaxSubscriptionsPerClient bounds one connection's subscriptions so a
|
|
// client cannot grow the registry without limit.
|
|
runEventMaxSubscriptionsPerClient = 16
|
|
// runEventMaxCoalescers bounds the per-run coalescer registry; idle
|
|
// coalescers are evicted once it is exceeded.
|
|
runEventMaxCoalescers = 512
|
|
)
|
|
|
|
// runEventHub tracks the connections subscribed to run deltas and one coalescer
|
|
// per active run. All shared maps are guarded by mu. Nothing here performs a
|
|
// database read: the snapshot is supplied by the caller and every delta carries
|
|
// values already computed at the emit site.
|
|
type runEventHub struct {
|
|
mu sync.Mutex
|
|
subs map[*runSubscription]struct{}
|
|
byClient map[string]map[*runSubscription]struct{}
|
|
runs map[int64]*runCoalescer
|
|
now func() time.Time
|
|
interval time.Duration
|
|
}
|
|
|
|
// runCoalescer merges a run's rapid field updates into the latest pending delta
|
|
// and flushes at most once per interval. A terminal delta bypasses the timer.
|
|
type runCoalescer struct {
|
|
runID int64
|
|
workspace string
|
|
pending *protocol.RunEventFrame
|
|
timer *time.Timer
|
|
lastFlush time.Time
|
|
updated time.Time
|
|
}
|
|
|
|
// runSubscription is one connection's watch over a workspace (and optionally
|
|
// explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow
|
|
// or broken subscriber never stalls the producer, and its buffered channel
|
|
// bounds how far behind it may fall.
|
|
//
|
|
// A subscription starts in its snapshot phase: the subscribing connection
|
|
// writes the snapshot and the ack itself (writeSnapshot) and only then starts
|
|
// the writer goroutine (ready). mu guards that phase, so the drop-a-slow-
|
|
// subscriber policy can never fire on the snapshot or lose its ack: while
|
|
// snapshotting a delta is held in pending instead of the bounded channel, and
|
|
// nothing writes to the channel until ready has released the held frames.
|
|
type runSubscription struct {
|
|
hub *runEventHub
|
|
clientID string
|
|
workspace string
|
|
runIDs map[int64]struct{}
|
|
ch chan protocol.Message
|
|
done chan struct{}
|
|
once sync.Once
|
|
write func(protocol.Message) error
|
|
|
|
mu sync.Mutex
|
|
snapshotting bool
|
|
pending []protocol.Message
|
|
}
|
|
|
|
func newRunEventHub() *runEventHub {
|
|
return &runEventHub{
|
|
subs: map[*runSubscription]struct{}{},
|
|
byClient: map[string]map[*runSubscription]struct{}{},
|
|
runs: map[int64]*runCoalescer{},
|
|
now: time.Now,
|
|
interval: runEventFlushInterval,
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) clock() time.Time {
|
|
if h.now != nil {
|
|
return h.now()
|
|
}
|
|
return time.Now()
|
|
}
|
|
|
|
// subscribe registers one connection's watch and returns it in its snapshot
|
|
// phase: the caller delivers the snapshot and the ack with writeSnapshot and
|
|
// then calls ready, which releases the deltas emitted meanwhile and starts the
|
|
// writer goroutine. It returns nil when the connection already holds the
|
|
// per-client subscription cap.
|
|
func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription {
|
|
if h == nil || write == nil {
|
|
return nil
|
|
}
|
|
h.mu.Lock()
|
|
if len(h.byClient[clientID]) >= runEventMaxSubscriptionsPerClient {
|
|
h.mu.Unlock()
|
|
return nil
|
|
}
|
|
sub := &runSubscription{
|
|
hub: h,
|
|
clientID: clientID,
|
|
workspace: workspace,
|
|
ch: make(chan protocol.Message, runEventSubscriberBuffer),
|
|
done: make(chan struct{}),
|
|
write: write,
|
|
snapshotting: true,
|
|
}
|
|
for _, id := range runIDs {
|
|
if id <= 0 {
|
|
continue
|
|
}
|
|
if sub.runIDs == nil {
|
|
sub.runIDs = map[int64]struct{}{}
|
|
}
|
|
sub.runIDs[id] = struct{}{}
|
|
}
|
|
h.subs[sub] = struct{}{}
|
|
if h.byClient[clientID] == nil {
|
|
h.byClient[clientID] = map[*runSubscription]struct{}{}
|
|
}
|
|
h.byClient[clientID][sub] = struct{}{}
|
|
h.mu.Unlock()
|
|
// The writer goroutine starts only in ready, after the snapshot and the ack
|
|
// are on the wire, so a delta can never overtake them.
|
|
return sub
|
|
}
|
|
|
|
// unsubscribeClient drops every subscription held by one connection; the read
|
|
// loop calls it when the connection ends so nothing leaks.
|
|
func (h *runEventHub) unsubscribeClient(clientID string) {
|
|
if h == nil {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
var subs []*runSubscription
|
|
for sub := range h.byClient[clientID] {
|
|
subs = append(subs, sub)
|
|
}
|
|
h.mu.Unlock()
|
|
for _, sub := range subs {
|
|
sub.close()
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) remove(sub *runSubscription) {
|
|
h.mu.Lock()
|
|
delete(h.subs, sub)
|
|
if set := h.byClient[sub.clientID]; set != nil {
|
|
delete(set, sub)
|
|
if len(set) == 0 {
|
|
delete(h.byClient, sub.clientID)
|
|
}
|
|
}
|
|
h.releaseIdleLocked()
|
|
h.mu.Unlock()
|
|
}
|
|
|
|
// emit folds one transition into its run's coalescer and schedules the flush.
|
|
// A terminal transition flushes any pending delta first, then the terminal frame
|
|
// immediately, so a client always observes the final state as its own frame.
|
|
func (h *runEventHub) emit(frame protocol.RunEventFrame, terminal bool) {
|
|
if step := frame.Step; step != nil {
|
|
frame.GraphVisits = append(frame.GraphVisits, protocol.RunGraphVisit{Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target})
|
|
}
|
|
if h == nil || frame.RunID <= 0 {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
if !h.anySubscriberLocked(frame.Workspace) {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
c := h.runs[frame.RunID]
|
|
if c == nil {
|
|
h.evictIdleLocked()
|
|
c = &runCoalescer{runID: frame.RunID}
|
|
h.runs[frame.RunID] = c
|
|
}
|
|
c.workspace = frame.Workspace
|
|
now := h.clock()
|
|
c.updated = now
|
|
var out []protocol.RunEventFrame
|
|
if terminal {
|
|
if c.pending != nil {
|
|
out = append(out, *c.pending)
|
|
c.pending = nil
|
|
}
|
|
out = append(out, frame)
|
|
h.stopTimerLocked(c)
|
|
c.lastFlush = now
|
|
} else {
|
|
if c.pending == nil {
|
|
pending := frame
|
|
c.pending = &pending
|
|
} else {
|
|
mergeRunEventFrame(c.pending, &frame)
|
|
}
|
|
if c.timer == nil {
|
|
if wait := h.interval - now.Sub(c.lastFlush); wait <= 0 {
|
|
out = append(out, *c.pending)
|
|
c.pending = nil
|
|
c.lastFlush = now
|
|
} else {
|
|
c.timer = time.AfterFunc(wait, func() { h.flushRun(frame.RunID) })
|
|
}
|
|
}
|
|
}
|
|
h.mu.Unlock()
|
|
for _, f := range out {
|
|
h.deliver(f)
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) flushRun(runID int64) {
|
|
h.mu.Lock()
|
|
c := h.runs[runID]
|
|
if c == nil {
|
|
h.mu.Unlock()
|
|
return
|
|
}
|
|
c.timer = nil
|
|
var out []protocol.RunEventFrame
|
|
if c.pending != nil {
|
|
out = append(out, *c.pending)
|
|
c.pending = nil
|
|
c.lastFlush = h.clock()
|
|
c.updated = c.lastFlush
|
|
}
|
|
h.mu.Unlock()
|
|
for _, f := range out {
|
|
h.deliver(f)
|
|
}
|
|
}
|
|
|
|
// deliver fans one frame out to the matching subscriptions. It collects the
|
|
// targets under the lock and hands them off outside it, so a slow write cannot
|
|
// hold the hub and stall the producer.
|
|
func (h *runEventHub) deliver(frame protocol.RunEventFrame) {
|
|
h.mu.Lock()
|
|
targets := make([]*runSubscription, 0, len(h.subs))
|
|
for sub := range h.subs {
|
|
if sub.matches(frame.Workspace, frame.RunID) {
|
|
targets = append(targets, sub)
|
|
}
|
|
}
|
|
h.mu.Unlock()
|
|
for _, sub := range targets {
|
|
sub.send(frame)
|
|
}
|
|
}
|
|
|
|
// removeRuns tells every subscription watching any of the deleted runs that
|
|
// they are gone (a runs_removed push carrying the matching run_ids) and drops
|
|
// the runs' coalescers so no pending delta resurrects a deleted row.
|
|
// workspaces maps each run to its workspace for subscription scoping.
|
|
func (h *runEventHub) removeRuns(ids []int64, workspaces map[int64]string) {
|
|
if h == nil || len(ids) == 0 {
|
|
return
|
|
}
|
|
h.mu.Lock()
|
|
for _, id := range ids {
|
|
if c := h.runs[id]; c != nil {
|
|
h.stopTimerLocked(c)
|
|
delete(h.runs, id)
|
|
}
|
|
}
|
|
targets := make([]*runSubscription, 0, len(h.subs))
|
|
for sub := range h.subs {
|
|
targets = append(targets, sub)
|
|
}
|
|
h.mu.Unlock()
|
|
for _, sub := range targets {
|
|
var matched []int64
|
|
for _, id := range ids {
|
|
if sub.matches(workspaces[id], id) {
|
|
matched = append(matched, id)
|
|
}
|
|
}
|
|
if len(matched) > 0 {
|
|
sub.sendMessage(protocol.Message{Type: protocol.RunsRemoved, RunIDs: matched})
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) anySubscriberLocked(workspace string) bool {
|
|
for sub := range h.subs {
|
|
if sub.workspace == "" || sub.workspace == workspace {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// releaseIdleLocked drops every coalescer whose workspace no longer has a
|
|
// subscriber, stopping its timer, so the registry is bounded and released on
|
|
// unsubscribe/disconnect.
|
|
func (h *runEventHub) releaseIdleLocked() {
|
|
for id, c := range h.runs {
|
|
if h.anySubscriberLocked(c.workspace) {
|
|
continue
|
|
}
|
|
h.stopTimerLocked(c)
|
|
delete(h.runs, id)
|
|
}
|
|
}
|
|
|
|
// evictIdleLocked keeps the coalescer registry bounded by dropping the
|
|
// least-recently-updated idle coalescer once the cap is reached. It never evicts
|
|
// a coalescer with pending work or a running timer.
|
|
func (h *runEventHub) evictIdleLocked() {
|
|
for len(h.runs) >= runEventMaxCoalescers {
|
|
var oldest *runCoalescer
|
|
for _, c := range h.runs {
|
|
if c.pending != nil || c.timer != nil {
|
|
continue
|
|
}
|
|
if oldest == nil || c.updated.Before(oldest.updated) {
|
|
oldest = c
|
|
}
|
|
}
|
|
if oldest == nil {
|
|
return
|
|
}
|
|
h.stopTimerLocked(oldest)
|
|
delete(h.runs, oldest.runID)
|
|
}
|
|
}
|
|
|
|
func (h *runEventHub) stopTimerLocked(c *runCoalescer) {
|
|
if c.timer != nil {
|
|
c.timer.Stop()
|
|
c.timer = nil
|
|
}
|
|
}
|
|
|
|
func (s *runSubscription) matches(workspace string, runID int64) bool {
|
|
if s.workspace != "" && s.workspace != workspace {
|
|
return false
|
|
}
|
|
if len(s.runIDs) > 0 {
|
|
_, ok := s.runIDs[runID]
|
|
return ok
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (s *runSubscription) run() {
|
|
for {
|
|
select {
|
|
case <-s.done:
|
|
return
|
|
case msg := <-s.ch:
|
|
if err := s.write(msg); err != nil {
|
|
s.close()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// writeSnapshot writes one frame of the subscription's snapshot directly on the
|
|
// subscribing connection. It runs only during the snapshot phase, before the
|
|
// writer goroutine exists, so it cannot reorder against a delta and is never
|
|
// subject to the bounded channel's drop policy. It returns false when the frame
|
|
// could not be written, having closed the subscription; the caller must then
|
|
// stop writing the snapshot.
|
|
func (s *runSubscription) writeSnapshot(msg protocol.Message) bool {
|
|
select {
|
|
case <-s.done:
|
|
return false
|
|
default:
|
|
}
|
|
if err := s.write(msg); err != nil {
|
|
s.close()
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// ready ends the snapshot phase: the deltas held while the snapshot was written
|
|
// are moved into the channel in order and the writer goroutine starts. The
|
|
// channel is empty until then (sendMessage queues into pending) and pending is
|
|
// bounded by the channel's capacity, so the move cannot block. A subscription
|
|
// closed during its snapshot starts no goroutine at all.
|
|
func (s *runSubscription) ready() {
|
|
s.mu.Lock()
|
|
select {
|
|
case <-s.done:
|
|
s.mu.Unlock()
|
|
return
|
|
default:
|
|
}
|
|
for _, msg := range s.pending {
|
|
s.ch <- msg
|
|
}
|
|
s.pending = nil
|
|
s.snapshotting = false
|
|
s.mu.Unlock()
|
|
go s.run()
|
|
}
|
|
|
|
func (s *runSubscription) send(frame protocol.RunEventFrame) {
|
|
s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame})
|
|
}
|
|
|
|
func (s *runSubscription) sendMessage(msg protocol.Message) {
|
|
select {
|
|
case <-s.done:
|
|
return
|
|
default:
|
|
}
|
|
s.mu.Lock()
|
|
if s.snapshotting {
|
|
// The snapshot is still being written on the subscribing connection and
|
|
// no writer goroutine is draining yet, so closing here would truncate a
|
|
// snapshot in flight and swallow its ack. Hold the delta instead, bounded
|
|
// the same as the channel: the policy can only ever affect deltas, since
|
|
// the snapshot and the ack are written directly by the subscriber.
|
|
if len(s.pending) >= runEventSubscriberBuffer {
|
|
s.mu.Unlock()
|
|
s.close()
|
|
return
|
|
}
|
|
s.pending = append(s.pending, msg)
|
|
s.mu.Unlock()
|
|
return
|
|
}
|
|
s.mu.Unlock()
|
|
select {
|
|
case s.ch <- msg:
|
|
default:
|
|
// The subscriber is not draining: drop it rather than block the
|
|
// producer. Its next subscribe re-syncs from a fresh snapshot.
|
|
s.close()
|
|
}
|
|
}
|
|
|
|
// close unregisters the subscription and stops its writer goroutine exactly
|
|
// once. It is safe to call from the writer goroutine, from unsubscribe and from
|
|
// the disconnect path.
|
|
func (s *runSubscription) close() {
|
|
s.once.Do(func() {
|
|
close(s.done)
|
|
s.hub.remove(s)
|
|
})
|
|
}
|
|
|
|
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
|
if in.NotifyMode != "" {
|
|
into.NotifyMode = in.NotifyMode
|
|
}
|
|
if in.Scheduled {
|
|
into.Scheduled = true
|
|
into.ScheduleID = in.ScheduleID
|
|
into.ScheduleName = in.ScheduleName
|
|
}
|
|
into.CancelRequested = into.CancelRequested || in.CancelRequested
|
|
into.CleanupUnconfirmed = into.CleanupUnconfirmed || in.CleanupUnconfirmed
|
|
if in.Workspace != "" {
|
|
into.Workspace = in.Workspace
|
|
}
|
|
// Identity is fixed for a run's life, so the first frame that carries it wins
|
|
// and a later delta never blanks it.
|
|
if into.PipelineID == 0 {
|
|
into.PipelineID = in.PipelineID
|
|
}
|
|
if into.Name == "" {
|
|
into.Name = in.Name
|
|
}
|
|
if into.Title == "" {
|
|
into.Title = in.Title
|
|
}
|
|
if into.Version == 0 {
|
|
into.Version = in.Version
|
|
}
|
|
if in.Status != "" {
|
|
into.Status = in.Status
|
|
into.ExportPendingSequence = in.ExportPendingSequence
|
|
}
|
|
if in.Outcome != "" {
|
|
into.Outcome = in.Outcome
|
|
}
|
|
if in.CurrentStage != "" {
|
|
into.CurrentStage = in.CurrentStage
|
|
}
|
|
if in.CompletedSteps > into.CompletedSteps {
|
|
into.CompletedSteps = in.CompletedSteps
|
|
}
|
|
if in.CreatedAt != "" {
|
|
into.CreatedAt = in.CreatedAt
|
|
}
|
|
if in.FinishedAt != "" {
|
|
into.FinishedAt = in.FinishedAt
|
|
}
|
|
if in.Artifacts > into.Artifacts {
|
|
into.Artifacts = in.Artifacts
|
|
}
|
|
if in.SizeBytes > 0 {
|
|
into.SizeBytes = in.SizeBytes
|
|
}
|
|
if in.Step != nil {
|
|
into.Step = in.Step
|
|
}
|
|
into.GraphVisits = append(into.GraphVisits, in.GraphVisits...)
|
|
}
|
|
|
|
// handleRunEvents dispatches one subscribe/unsubscribe request and reports
|
|
// whether it was a run-event type (so the caller never falls through to the
|
|
// generic unknown-command response). Subscribe registers the watch, writes the
|
|
// current state of the watched runs as its snapshot, acks, and only then lets
|
|
// deltas flow: the snapshot and the ack go straight out on the subscribing
|
|
// connection, so a snapshot larger than the subscription's buffer can no longer
|
|
// drop the subscription it is seeding, and the ordering is always snapshot,
|
|
// ack, deltas.
|
|
func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool {
|
|
switch request.Type {
|
|
case protocol.Subscribe:
|
|
if runs == nil || events == nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "run events unavailable: no store"})
|
|
return true
|
|
}
|
|
sub := events.subscribe(clientID, request.Workspace, request.RunIDs, write)
|
|
if sub == nil {
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "too many run subscriptions on this connection"})
|
|
return true
|
|
}
|
|
// One bounded read seeds the subscription. It reuses listRuns, the same
|
|
// workspace-scoped read the polling clients use, instead of a new query.
|
|
summaries, err := runs.listRuns()
|
|
if err != nil {
|
|
sub.close()
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
|
return true
|
|
}
|
|
for _, summary := range summaries {
|
|
if !sub.matches(summary.Workspace, summary.ID) {
|
|
continue
|
|
}
|
|
frame := protocol.RunEventFrame{
|
|
Scheduled: summary.Scheduled,
|
|
ScheduleID: summary.ScheduleID,
|
|
ScheduleName: summary.ScheduleName,
|
|
GraphEvidence: summary.GraphEvidence,
|
|
NotifyMode: summary.NotifyMode,
|
|
RunID: summary.ID,
|
|
Workspace: summary.Workspace,
|
|
PipelineID: summary.PipelineID,
|
|
Name: summary.Name,
|
|
Title: summary.Title,
|
|
Version: summary.Version,
|
|
Status: summary.Status,
|
|
Outcome: summary.Outcome,
|
|
CurrentStage: summary.CurrentStage,
|
|
CompletedSteps: summary.CompletedSteps,
|
|
CreatedAt: summary.CreatedAt,
|
|
FinishedAt: summary.FinishedAt,
|
|
CancelRequested: summary.CancelRequested,
|
|
CleanupUnconfirmed: summary.CleanupUnconfirmed,
|
|
ExportPendingSequence: summary.ExportPendingSequence,
|
|
SizeBytes: summary.SizeBytes,
|
|
}
|
|
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
|
|
// The connection write failed; the subscription is closed and the
|
|
// read loop will stop on its next read error, so there is nothing
|
|
// left to reply to.
|
|
return true
|
|
}
|
|
}
|
|
if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil {
|
|
sub.close()
|
|
return true
|
|
}
|
|
sub.ready()
|
|
return true
|
|
case protocol.Unsubscribe:
|
|
events.unsubscribeClient(clientID)
|
|
_ = write(protocol.Message{Type: protocol.Unsubscribe, ID: request.ID})
|
|
return true
|
|
}
|
|
return false
|
|
}
|