Files
pipelines/internal/pipeline/run_size.go
T
alan 31e89e872b Add run size_bytes and delete_runs to free disk space
Every run summary carries size_bytes (artifact tree, server-created Pi
session directories and an estimate of the run's database rows), cached in
pipeline_runs.size_bytes, invalidated by triggers, measured when a run
finishes and backfilled in the background. delete_runs removes terminal
runs with every scoped row and server-owned file, refuses active ones with
a reason, and pushes runs_removed to subscribers. pipelines runs rm is the
CLI form; retention apply now shares the complete row deletion.
2026-10-31 22:11:14 +00:00

265 lines
10 KiB
Go

package pipeline
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
)
// Run size accounting. pipeline_runs.size_bytes caches the bytes on disk a run
// accounts for: its artifact tree (<artifacts>/<run>, which holds turn
// documents, inputs, outputs and session exports), any Pi session directory the
// server created for its stages, and an estimate of its database rows (the byte
// length of every large text column plus a small per-row overhead).
//
// -1 means "not computed or stale". Triggers mark a run stale whenever its row,
// a visit or visit evidence changes, and bump size_gen so a size computed
// concurrently with a change is never stored as current (see storeRunSize).
// Existing databases start with every run at -1 and are backfilled lazily.
// runSizeTriggers are the tables whose writes change a run's footprint. Each
// gets an insert and an update trigger that invalidates the owning run.
var runSizeTriggers = []string{
"run_steps", "step_notifications", "step_file_changes", "step_control", "step_artifacts",
"step_verdicts", "step_session_exports", "step_usage", "run_session_owners",
"session_export_repairs", "run_notifications", "retry_checkpoints",
}
// initRunSizes adds the cache columns and invalidation triggers. Every step is
// idempotent so a repeated Open and an upgraded database both open cleanly.
func (s *Store) initRunSizes() error {
if err := s.addColumnIfMissing("pipeline_runs", "size_bytes", "INTEGER NOT NULL DEFAULT -1"); err != nil {
return err
}
if err := s.addColumnIfMissing("pipeline_runs", "size_gen", "INTEGER NOT NULL DEFAULT 0"); err != nil {
return err
}
statements := []string{
// The nested UPDATE touches only size columns, which are not in the OF
// list, so this trigger cannot re-fire itself.
`CREATE TRIGGER IF NOT EXISTS run_size_stale_run AFTER UPDATE OF status,outcome,output_json,input_json,diagnostic,finished_at,title ON pipeline_runs
BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.id; END`,
}
for _, table := range runSizeTriggers {
for _, event := range []string{"INSERT", "UPDATE"} {
statements = append(statements, fmt.Sprintf(`CREATE TRIGGER IF NOT EXISTS run_size_stale_%s_%s AFTER %s ON %s
BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.run_id; END`, table, strings.ToLower(event), event, table))
}
}
for _, statement := range statements {
if _, err := s.db.Exec(statement); err != nil {
return err
}
}
return nil
}
// runSizeRowOverhead approximates SQLite's per-row cost (record header, rowid,
// index entries) beyond the measured column bytes.
const runSizeRowOverhead = 64
// runDBBytesQuery estimates one run's database footprint: the byte length of
// every variable-size column in every table scoped to the run, plus a fixed
// per-row overhead. Each subquery uses the table's run_id primary-key prefix or
// index, so the cost is bounded by the run's own rows.
var runDBBytesQuery = func() string {
type part struct {
table, where string
columns []string
}
parts := []part{
{"pipeline_runs", "id=?1", []string{"request_key", "workspace", "title", "title_source", "title_model", "model", "stage_models", "write_paths", "input_json", "output_json", "outcome", "current_stage", "diagnostic", "output_format"}},
{"run_steps", "run_id=?1", []string{"stage_key", "outcome", "target", "input_json", "output_json", "diagnostic", "provider", "model", "session_id", "transcript"}},
{"step_notifications", "run_id=?1", []string{"receipts_json"}},
{"step_file_changes", "run_id=?1", []string{"changes_json"}},
{"step_control", "run_id=?1", []string{"evidence_json"}},
{"step_artifacts", "run_id=?1", []string{"artifacts_json"}},
{"step_verdicts", "run_id=?1", []string{"verdict_json"}},
{"step_session_exports", "run_id=?1", []string{"export_json"}},
{"step_usage", "run_id=?1", []string{"usage_json"}},
{"run_session_owners", "run_id=?1", []string{"backend", "reference"}},
{"session_export_repairs", "run_id=?1", []string{"export_json"}},
{"session_export_jobs", "run_id=?1", []string{"session_id", "backend", "directory", "reference", "token", "diagnostic"}},
{"run_notifications", "run_id=?1", []string{"event", "channel", "title", "body", "click", "diagnostic", "message_id"}},
{"retry_checkpoints", "run_id=?1", []string{"mode", "handoff_json", "source_hash"}},
{"run_schedule_origin", "run_id=?1", []string{"name"}},
}
terms := make([]string, 0, len(parts))
for _, p := range parts {
lengths := make([]string, 0, len(p.columns)+1)
for _, column := range p.columns {
lengths = append(lengths, "COALESCE(length(CAST("+column+" AS BLOB)),0)")
}
lengths = append(lengths, fmt.Sprint(runSizeRowOverhead))
terms = append(terms, "COALESCE((SELECT SUM("+strings.Join(lengths, "+")+") FROM "+p.table+" WHERE "+p.where+"),0)")
}
return "SELECT " + strings.Join(terms, "+")
}()
// sizeQuerier is the read surface shared by *sql.DB and *sql.Tx, so a size is
// computed identically by the cache refresh and inside the delete transaction.
type sizeQuerier interface {
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}
// piSessionsBase is the directory the Pi stage client creates its per-visit
// session directories under (see internal/stage/pi.go). It is a variable so
// tests can point it at a temporary directory.
var piSessionsBase = func() (string, error) {
cache, err := os.UserCacheDir()
if err != nil {
return "", err
}
return filepath.Join(cache, "pipelines", "pi-sessions"), nil
}
// ownedPiSessionDir reports whether reference is a server-created Pi session
// directory: an absolute, clean path directly under piSessionsBase named
// stage-*, that is a real directory rather than a symlink. Anything else - a
// user path, a workspace, a symlink - is never sized or removed.
func ownedPiSessionDir(reference string) bool {
base, err := piSessionsBase()
if err != nil || base == "" || reference == "" || !filepath.IsAbs(reference) || filepath.Clean(reference) != reference {
return false
}
if filepath.Dir(reference) != base || !strings.HasPrefix(filepath.Base(reference), "stage-") {
return false
}
info, err := os.Lstat(reference)
return err == nil && info.IsDir()
}
// runPiSessionDirs lists the owned Pi session directories recorded for a run.
func runPiSessionDirs(ctx context.Context, q sizeQuerier, id int64) ([]string, error) {
rows, err := q.QueryContext(ctx, `SELECT DISTINCT reference FROM run_session_owners WHERE run_id=? AND backend='pi' AND reference<>''`, id)
if err != nil {
return nil, err
}
defer rows.Close()
var dirs []string
for rows.Next() {
var reference string
if err := rows.Scan(&reference); err != nil {
return nil, err
}
if ownedPiSessionDir(reference) {
dirs = append(dirs, reference)
}
}
return dirs, rows.Err()
}
// errRunNotFound marks a size or delete request for a run id with no row.
var errRunNotFound = fmt.Errorf("run not found")
// computeRunSize measures one run now: artifact tree, owned Pi session
// directories and the database estimate. Unreadable files are skipped rather
// than failing the measurement; the size is a best-effort lower bound then.
func computeRunSize(ctx context.Context, q sizeQuerier, artifactsRoot string, id int64) (int64, error) {
var exists bool
if err := q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pipeline_runs WHERE id=?)`, id).Scan(&exists); err != nil {
return 0, err
}
if !exists {
return 0, errRunNotFound
}
var dbBytes int64
if err := q.QueryRowContext(ctx, runDBBytesQuery, id).Scan(&dbBytes); err != nil {
return 0, err
}
_, artifactBytes, _ := retentionArtifactsSize(artifactsRoot, id)
total := dbBytes + artifactBytes
dirs, err := runPiSessionDirs(ctx, q, id)
if err != nil {
return 0, err
}
for _, dir := range dirs {
_, bytes, _ := retentionWalkSize(dir)
total += bytes
}
return total, nil
}
// RunSize is one run whose cached size was (re)computed.
type RunSize struct {
ID int64
Workspace string
SizeBytes int64
}
// RefreshRunSize recomputes and caches one run's size and returns it. The value
// is stored only if the run did not change while it was measured; it is
// returned either way, since it is the freshest measurement available.
func (s *Store) RefreshRunSize(ctx context.Context, artifactsRoot string, id int64) (int64, error) {
var gen int64
if err := s.db.QueryRowContext(ctx, `SELECT size_gen FROM pipeline_runs WHERE id=?`, id).Scan(&gen); err != nil {
if err == sql.ErrNoRows {
return 0, errRunNotFound
}
return 0, err
}
size, err := computeRunSize(ctx, s.db, artifactsRoot, id)
if err != nil {
return 0, err
}
if _, err := s.db.ExecContext(ctx, `UPDATE pipeline_runs SET size_bytes=? WHERE id=? AND size_gen=?`, size, id, gen); err != nil {
return 0, err
}
return size, nil
}
// RefreshRunSizes computes and caches the size of up to limit runs whose cached
// size is missing or stale, newest first, and returns the runs it measured. A
// limit of zero or less means no limit. It is the lazy backfill: callers run it
// in the background or with a small limit on a request path.
func (s *Store) RefreshRunSizes(ctx context.Context, artifactsRoot string, limit int) ([]RunSize, error) {
if limit <= 0 {
limit = -1
}
rows, err := s.db.QueryContext(ctx, `SELECT id,workspace FROM pipeline_runs WHERE size_bytes<0 ORDER BY id DESC LIMIT ?`, limit)
if err != nil {
return nil, err
}
var stale []RunSize
for rows.Next() {
var item RunSize
if err := rows.Scan(&item.ID, &item.Workspace); err != nil {
rows.Close()
return nil, err
}
stale = append(stale, item)
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
measured := make([]RunSize, 0, len(stale))
for _, item := range stale {
if err := ctx.Err(); err != nil {
return measured, err
}
size, err := s.RefreshRunSize(ctx, artifactsRoot, item.ID)
if err == errRunNotFound {
continue
}
if err != nil {
return measured, err
}
item.SizeBytes = size
measured = append(measured, item)
}
return measured, nil
}
// StaleRunSizes counts runs whose cached size is missing or stale.
func (s *Store) StaleRunSizes(ctx context.Context) (int, error) {
var count int
err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pipeline_runs WHERE size_bytes<0`).Scan(&count)
return count, err
}