Files
alan 31e89e872b Add run size_bytes and delete_runs to free disk space
Every run summary carries size_bytes (artifact tree, server-created Pi
session directories and an estimate of the run's database rows), cached in
pipeline_runs.size_bytes, invalidated by triggers, measured when a run
finishes and backfilled in the background. delete_runs removes terminal
runs with every scoped row and server-owned file, refuses active ones with
a reason, and pushes runs_removed to subscribers. pipelines runs rm is the
CLI form; retention apply now shares the complete row deletion.
2026-10-31 22:11:14 +00:00

596 lines
17 KiB
Go

package server
import (
"sync"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// The run-event hub pushes compact run state deltas to subscribed connections so
// a client no longer has to poll. It mirrors the chat hub's shape: subscriptions
// live on the hub, keyed by connection, and every write goes through the
// connection-scoped write closure. Unlike chat, a run transition has no session
// to stream from, so deltas are emitted by the runner and coalesced here.
const (
// runEventFlushInterval throttles a busy run's pushes to at most a few frames
// per second. A terminal delta is flushed immediately regardless, so the final
// state is never delayed or dropped.
runEventFlushInterval = 220 * time.Millisecond
// runEventSubscriberBuffer bounds one subscription's pending frames. A
// subscriber that falls this far behind is dropped instead of blocking the
// producer or other subscribers; it re-syncs from a fresh subscribe snapshot.
runEventSubscriberBuffer = 32
// runEventMaxSubscriptionsPerClient bounds one connection's subscriptions so a
// client cannot grow the registry without limit.
runEventMaxSubscriptionsPerClient = 16
// runEventMaxCoalescers bounds the per-run coalescer registry; idle
// coalescers are evicted once it is exceeded.
runEventMaxCoalescers = 512
)
// runEventHub tracks the connections subscribed to run deltas and one coalescer
// per active run. All shared maps are guarded by mu. Nothing here performs a
// database read: the snapshot is supplied by the caller and every delta carries
// values already computed at the emit site.
type runEventHub struct {
mu sync.Mutex
subs map[*runSubscription]struct{}
byClient map[string]map[*runSubscription]struct{}
runs map[int64]*runCoalescer
now func() time.Time
interval time.Duration
}
// runCoalescer merges a run's rapid field updates into the latest pending delta
// and flushes at most once per interval. A terminal delta bypasses the timer.
type runCoalescer struct {
runID int64
workspace string
pending *protocol.RunEventFrame
timer *time.Timer
lastFlush time.Time
updated time.Time
}
// runSubscription is one connection's watch over a workspace (and optionally
// explicit run ids). Deltas are handed to a dedicated writer goroutine so a slow
// or broken subscriber never stalls the producer, and its buffered channel
// bounds how far behind it may fall.
//
// A subscription starts in its snapshot phase: the subscribing connection
// writes the snapshot and the ack itself (writeSnapshot) and only then starts
// the writer goroutine (ready). mu guards that phase, so the drop-a-slow-
// subscriber policy can never fire on the snapshot or lose its ack: while
// snapshotting a delta is held in pending instead of the bounded channel, and
// nothing writes to the channel until ready has released the held frames.
type runSubscription struct {
hub *runEventHub
clientID string
workspace string
runIDs map[int64]struct{}
ch chan protocol.Message
done chan struct{}
once sync.Once
write func(protocol.Message) error
mu sync.Mutex
snapshotting bool
pending []protocol.Message
}
func newRunEventHub() *runEventHub {
return &runEventHub{
subs: map[*runSubscription]struct{}{},
byClient: map[string]map[*runSubscription]struct{}{},
runs: map[int64]*runCoalescer{},
now: time.Now,
interval: runEventFlushInterval,
}
}
func (h *runEventHub) clock() time.Time {
if h.now != nil {
return h.now()
}
return time.Now()
}
// subscribe registers one connection's watch and returns it in its snapshot
// phase: the caller delivers the snapshot and the ack with writeSnapshot and
// then calls ready, which releases the deltas emitted meanwhile and starts the
// writer goroutine. It returns nil when the connection already holds the
// per-client subscription cap.
func (h *runEventHub) subscribe(clientID, workspace string, runIDs []int64, write func(protocol.Message) error) *runSubscription {
if h == nil || write == nil {
return nil
}
h.mu.Lock()
if len(h.byClient[clientID]) >= runEventMaxSubscriptionsPerClient {
h.mu.Unlock()
return nil
}
sub := &runSubscription{
hub: h,
clientID: clientID,
workspace: workspace,
ch: make(chan protocol.Message, runEventSubscriberBuffer),
done: make(chan struct{}),
write: write,
snapshotting: true,
}
for _, id := range runIDs {
if id <= 0 {
continue
}
if sub.runIDs == nil {
sub.runIDs = map[int64]struct{}{}
}
sub.runIDs[id] = struct{}{}
}
h.subs[sub] = struct{}{}
if h.byClient[clientID] == nil {
h.byClient[clientID] = map[*runSubscription]struct{}{}
}
h.byClient[clientID][sub] = struct{}{}
h.mu.Unlock()
// The writer goroutine starts only in ready, after the snapshot and the ack
// are on the wire, so a delta can never overtake them.
return sub
}
// unsubscribeClient drops every subscription held by one connection; the read
// loop calls it when the connection ends so nothing leaks.
func (h *runEventHub) unsubscribeClient(clientID string) {
if h == nil {
return
}
h.mu.Lock()
var subs []*runSubscription
for sub := range h.byClient[clientID] {
subs = append(subs, sub)
}
h.mu.Unlock()
for _, sub := range subs {
sub.close()
}
}
func (h *runEventHub) remove(sub *runSubscription) {
h.mu.Lock()
delete(h.subs, sub)
if set := h.byClient[sub.clientID]; set != nil {
delete(set, sub)
if len(set) == 0 {
delete(h.byClient, sub.clientID)
}
}
h.releaseIdleLocked()
h.mu.Unlock()
}
// emit folds one transition into its run's coalescer and schedules the flush.
// A terminal transition flushes any pending delta first, then the terminal frame
// immediately, so a client always observes the final state as its own frame.
func (h *runEventHub) emit(frame protocol.RunEventFrame, terminal bool) {
if step := frame.Step; step != nil {
frame.GraphVisits = append(frame.GraphVisits, protocol.RunGraphVisit{Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target})
}
if h == nil || frame.RunID <= 0 {
return
}
h.mu.Lock()
if !h.anySubscriberLocked(frame.Workspace) {
h.mu.Unlock()
return
}
c := h.runs[frame.RunID]
if c == nil {
h.evictIdleLocked()
c = &runCoalescer{runID: frame.RunID}
h.runs[frame.RunID] = c
}
c.workspace = frame.Workspace
now := h.clock()
c.updated = now
var out []protocol.RunEventFrame
if terminal {
if c.pending != nil {
out = append(out, *c.pending)
c.pending = nil
}
out = append(out, frame)
h.stopTimerLocked(c)
c.lastFlush = now
} else {
if c.pending == nil {
pending := frame
c.pending = &pending
} else {
mergeRunEventFrame(c.pending, &frame)
}
if c.timer == nil {
if wait := h.interval - now.Sub(c.lastFlush); wait <= 0 {
out = append(out, *c.pending)
c.pending = nil
c.lastFlush = now
} else {
c.timer = time.AfterFunc(wait, func() { h.flushRun(frame.RunID) })
}
}
}
h.mu.Unlock()
for _, f := range out {
h.deliver(f)
}
}
func (h *runEventHub) flushRun(runID int64) {
h.mu.Lock()
c := h.runs[runID]
if c == nil {
h.mu.Unlock()
return
}
c.timer = nil
var out []protocol.RunEventFrame
if c.pending != nil {
out = append(out, *c.pending)
c.pending = nil
c.lastFlush = h.clock()
c.updated = c.lastFlush
}
h.mu.Unlock()
for _, f := range out {
h.deliver(f)
}
}
// deliver fans one frame out to the matching subscriptions. It collects the
// targets under the lock and hands them off outside it, so a slow write cannot
// hold the hub and stall the producer.
func (h *runEventHub) deliver(frame protocol.RunEventFrame) {
h.mu.Lock()
targets := make([]*runSubscription, 0, len(h.subs))
for sub := range h.subs {
if sub.matches(frame.Workspace, frame.RunID) {
targets = append(targets, sub)
}
}
h.mu.Unlock()
for _, sub := range targets {
sub.send(frame)
}
}
// removeRuns tells every subscription watching any of the deleted runs that
// they are gone (a runs_removed push carrying the matching run_ids) and drops
// the runs' coalescers so no pending delta resurrects a deleted row.
// workspaces maps each run to its workspace for subscription scoping.
func (h *runEventHub) removeRuns(ids []int64, workspaces map[int64]string) {
if h == nil || len(ids) == 0 {
return
}
h.mu.Lock()
for _, id := range ids {
if c := h.runs[id]; c != nil {
h.stopTimerLocked(c)
delete(h.runs, id)
}
}
targets := make([]*runSubscription, 0, len(h.subs))
for sub := range h.subs {
targets = append(targets, sub)
}
h.mu.Unlock()
for _, sub := range targets {
var matched []int64
for _, id := range ids {
if sub.matches(workspaces[id], id) {
matched = append(matched, id)
}
}
if len(matched) > 0 {
sub.sendMessage(protocol.Message{Type: protocol.RunsRemoved, RunIDs: matched})
}
}
}
func (h *runEventHub) anySubscriberLocked(workspace string) bool {
for sub := range h.subs {
if sub.workspace == "" || sub.workspace == workspace {
return true
}
}
return false
}
// releaseIdleLocked drops every coalescer whose workspace no longer has a
// subscriber, stopping its timer, so the registry is bounded and released on
// unsubscribe/disconnect.
func (h *runEventHub) releaseIdleLocked() {
for id, c := range h.runs {
if h.anySubscriberLocked(c.workspace) {
continue
}
h.stopTimerLocked(c)
delete(h.runs, id)
}
}
// evictIdleLocked keeps the coalescer registry bounded by dropping the
// least-recently-updated idle coalescer once the cap is reached. It never evicts
// a coalescer with pending work or a running timer.
func (h *runEventHub) evictIdleLocked() {
for len(h.runs) >= runEventMaxCoalescers {
var oldest *runCoalescer
for _, c := range h.runs {
if c.pending != nil || c.timer != nil {
continue
}
if oldest == nil || c.updated.Before(oldest.updated) {
oldest = c
}
}
if oldest == nil {
return
}
h.stopTimerLocked(oldest)
delete(h.runs, oldest.runID)
}
}
func (h *runEventHub) stopTimerLocked(c *runCoalescer) {
if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
}
func (s *runSubscription) matches(workspace string, runID int64) bool {
if s.workspace != "" && s.workspace != workspace {
return false
}
if len(s.runIDs) > 0 {
_, ok := s.runIDs[runID]
return ok
}
return true
}
func (s *runSubscription) run() {
for {
select {
case <-s.done:
return
case msg := <-s.ch:
if err := s.write(msg); err != nil {
s.close()
return
}
}
}
}
// writeSnapshot writes one frame of the subscription's snapshot directly on the
// subscribing connection. It runs only during the snapshot phase, before the
// writer goroutine exists, so it cannot reorder against a delta and is never
// subject to the bounded channel's drop policy. It returns false when the frame
// could not be written, having closed the subscription; the caller must then
// stop writing the snapshot.
func (s *runSubscription) writeSnapshot(msg protocol.Message) bool {
select {
case <-s.done:
return false
default:
}
if err := s.write(msg); err != nil {
s.close()
return false
}
return true
}
// ready ends the snapshot phase: the deltas held while the snapshot was written
// are moved into the channel in order and the writer goroutine starts. The
// channel is empty until then (sendMessage queues into pending) and pending is
// bounded by the channel's capacity, so the move cannot block. A subscription
// closed during its snapshot starts no goroutine at all.
func (s *runSubscription) ready() {
s.mu.Lock()
select {
case <-s.done:
s.mu.Unlock()
return
default:
}
for _, msg := range s.pending {
s.ch <- msg
}
s.pending = nil
s.snapshotting = false
s.mu.Unlock()
go s.run()
}
func (s *runSubscription) send(frame protocol.RunEventFrame) {
s.sendMessage(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame})
}
func (s *runSubscription) sendMessage(msg protocol.Message) {
select {
case <-s.done:
return
default:
}
s.mu.Lock()
if s.snapshotting {
// The snapshot is still being written on the subscribing connection and
// no writer goroutine is draining yet, so closing here would truncate a
// snapshot in flight and swallow its ack. Hold the delta instead, bounded
// the same as the channel: the policy can only ever affect deltas, since
// the snapshot and the ack are written directly by the subscriber.
if len(s.pending) >= runEventSubscriberBuffer {
s.mu.Unlock()
s.close()
return
}
s.pending = append(s.pending, msg)
s.mu.Unlock()
return
}
s.mu.Unlock()
select {
case s.ch <- msg:
default:
// The subscriber is not draining: drop it rather than block the
// producer. Its next subscribe re-syncs from a fresh snapshot.
s.close()
}
}
// close unregisters the subscription and stops its writer goroutine exactly
// once. It is safe to call from the writer goroutine, from unsubscribe and from
// the disconnect path.
func (s *runSubscription) close() {
s.once.Do(func() {
close(s.done)
s.hub.remove(s)
})
}
func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.NotifyMode != "" {
into.NotifyMode = in.NotifyMode
}
if in.Scheduled {
into.Scheduled = true
into.ScheduleID = in.ScheduleID
into.ScheduleName = in.ScheduleName
}
into.CancelRequested = into.CancelRequested || in.CancelRequested
into.CleanupUnconfirmed = into.CleanupUnconfirmed || in.CleanupUnconfirmed
if in.Workspace != "" {
into.Workspace = in.Workspace
}
// Identity is fixed for a run's life, so the first frame that carries it wins
// and a later delta never blanks it.
if into.PipelineID == 0 {
into.PipelineID = in.PipelineID
}
if into.Name == "" {
into.Name = in.Name
}
if into.Title == "" {
into.Title = in.Title
}
if into.Version == 0 {
into.Version = in.Version
}
if in.Status != "" {
into.Status = in.Status
into.ExportPendingSequence = in.ExportPendingSequence
}
if in.Outcome != "" {
into.Outcome = in.Outcome
}
if in.CurrentStage != "" {
into.CurrentStage = in.CurrentStage
}
if in.CompletedSteps > into.CompletedSteps {
into.CompletedSteps = in.CompletedSteps
}
if in.CreatedAt != "" {
into.CreatedAt = in.CreatedAt
}
if in.FinishedAt != "" {
into.FinishedAt = in.FinishedAt
}
if in.Artifacts > into.Artifacts {
into.Artifacts = in.Artifacts
}
if in.SizeBytes > 0 {
into.SizeBytes = in.SizeBytes
}
if in.Step != nil {
into.Step = in.Step
}
into.GraphVisits = append(into.GraphVisits, in.GraphVisits...)
}
// handleRunEvents dispatches one subscribe/unsubscribe request and reports
// whether it was a run-event type (so the caller never falls through to the
// generic unknown-command response). Subscribe registers the watch, writes the
// current state of the watched runs as its snapshot, acks, and only then lets
// deltas flow: the snapshot and the ack go straight out on the subscribing
// connection, so a snapshot larger than the subscription's buffer can no longer
// drop the subscription it is seeding, and the ordering is always snapshot,
// ack, deltas.
func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner, clientID string, write func(protocol.Message) error) bool {
switch request.Type {
case protocol.Subscribe:
if runs == nil || events == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "run events unavailable: no store"})
return true
}
sub := events.subscribe(clientID, request.Workspace, request.RunIDs, write)
if sub == nil {
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "too many run subscriptions on this connection"})
return true
}
// One bounded read seeds the subscription. It reuses listRuns, the same
// workspace-scoped read the polling clients use, instead of a new query.
summaries, err := runs.listRuns()
if err != nil {
sub.close()
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
return true
}
for _, summary := range summaries {
if !sub.matches(summary.Workspace, summary.ID) {
continue
}
frame := protocol.RunEventFrame{
Scheduled: summary.Scheduled,
ScheduleID: summary.ScheduleID,
ScheduleName: summary.ScheduleName,
GraphEvidence: summary.GraphEvidence,
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
ExportPendingSequence: summary.ExportPendingSequence,
SizeBytes: summary.SizeBytes,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
// read loop will stop on its next read error, so there is nothing
// left to reply to.
return true
}
}
if err := write(protocol.Message{Type: protocol.Subscribe, ID: request.ID}); err != nil {
sub.close()
return true
}
sub.ready()
return true
case protocol.Unsubscribe:
events.unsubscribeClient(clientID)
_ = write(protocol.Message{Type: protocol.Unsubscribe, ID: request.ID})
return true
}
return false
}