package pipeline import ( "context" "database/sql" "fmt" "os" "path/filepath" "strings" ) // Run size accounting. pipeline_runs.size_bytes caches the bytes on disk a run // accounts for: its artifact tree (/, which holds turn // documents, inputs, outputs and session exports), any Pi session directory the // server created for its stages, and an estimate of its database rows (the byte // length of every large text column plus a small per-row overhead). // // -1 means "not computed or stale". Triggers mark a run stale whenever its row, // a visit or visit evidence changes, and bump size_gen so a size computed // concurrently with a change is never stored as current (see storeRunSize). // Existing databases start with every run at -1 and are backfilled lazily. // runSizeTriggers are the tables whose writes change a run's footprint. Each // gets an insert and an update trigger that invalidates the owning run. var runSizeTriggers = []string{ "run_steps", "step_notifications", "step_file_changes", "step_control", "step_artifacts", "step_verdicts", "step_session_exports", "step_usage", "run_session_owners", "session_export_repairs", "run_notifications", "retry_checkpoints", } // initRunSizes adds the cache columns and invalidation triggers. Every step is // idempotent so a repeated Open and an upgraded database both open cleanly. func (s *Store) initRunSizes() error { if err := s.addColumnIfMissing("pipeline_runs", "size_bytes", "INTEGER NOT NULL DEFAULT -1"); err != nil { return err } if err := s.addColumnIfMissing("pipeline_runs", "size_gen", "INTEGER NOT NULL DEFAULT 0"); err != nil { return err } statements := []string{ // The nested UPDATE touches only size columns, which are not in the OF // list, so this trigger cannot re-fire itself. `CREATE TRIGGER IF NOT EXISTS run_size_stale_run AFTER UPDATE OF status,outcome,output_json,input_json,diagnostic,finished_at,title ON pipeline_runs BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.id; END`, } for _, table := range runSizeTriggers { for _, event := range []string{"INSERT", "UPDATE"} { statements = append(statements, fmt.Sprintf(`CREATE TRIGGER IF NOT EXISTS run_size_stale_%s_%s AFTER %s ON %s BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.run_id; END`, table, strings.ToLower(event), event, table)) } } for _, statement := range statements { if _, err := s.db.Exec(statement); err != nil { return err } } return nil } // runSizeRowOverhead approximates SQLite's per-row cost (record header, rowid, // index entries) beyond the measured column bytes. const runSizeRowOverhead = 64 // runDBBytesQuery estimates one run's database footprint: the byte length of // every variable-size column in every table scoped to the run, plus a fixed // per-row overhead. Each subquery uses the table's run_id primary-key prefix or // index, so the cost is bounded by the run's own rows. var runDBBytesQuery = func() string { type part struct { table, where string columns []string } parts := []part{ {"pipeline_runs", "id=?1", []string{"request_key", "workspace", "title", "title_source", "title_model", "model", "stage_models", "write_paths", "input_json", "output_json", "outcome", "current_stage", "diagnostic", "output_format"}}, {"run_steps", "run_id=?1", []string{"stage_key", "outcome", "target", "input_json", "output_json", "diagnostic", "provider", "model", "session_id", "transcript"}}, {"step_notifications", "run_id=?1", []string{"receipts_json"}}, {"step_file_changes", "run_id=?1", []string{"changes_json"}}, {"step_control", "run_id=?1", []string{"evidence_json"}}, {"step_artifacts", "run_id=?1", []string{"artifacts_json"}}, {"step_verdicts", "run_id=?1", []string{"verdict_json"}}, {"step_session_exports", "run_id=?1", []string{"export_json"}}, {"step_usage", "run_id=?1", []string{"usage_json"}}, {"run_session_owners", "run_id=?1", []string{"backend", "reference"}}, {"session_export_repairs", "run_id=?1", []string{"export_json"}}, {"session_export_jobs", "run_id=?1", []string{"session_id", "backend", "directory", "reference", "token", "diagnostic"}}, {"run_notifications", "run_id=?1", []string{"event", "channel", "title", "body", "click", "diagnostic", "message_id"}}, {"retry_checkpoints", "run_id=?1", []string{"mode", "handoff_json", "source_hash"}}, {"run_schedule_origin", "run_id=?1", []string{"name"}}, } terms := make([]string, 0, len(parts)) for _, p := range parts { lengths := make([]string, 0, len(p.columns)+1) for _, column := range p.columns { lengths = append(lengths, "COALESCE(length(CAST("+column+" AS BLOB)),0)") } lengths = append(lengths, fmt.Sprint(runSizeRowOverhead)) terms = append(terms, "COALESCE((SELECT SUM("+strings.Join(lengths, "+")+") FROM "+p.table+" WHERE "+p.where+"),0)") } return "SELECT " + strings.Join(terms, "+") }() // sizeQuerier is the read surface shared by *sql.DB and *sql.Tx, so a size is // computed identically by the cache refresh and inside the delete transaction. type sizeQuerier interface { QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row } // piSessionsBase is the directory the Pi stage client creates its per-visit // session directories under (see internal/stage/pi.go). It is a variable so // tests can point it at a temporary directory. var piSessionsBase = func() (string, error) { cache, err := os.UserCacheDir() if err != nil { return "", err } return filepath.Join(cache, "pipelines", "pi-sessions"), nil } // ownedPiSessionDir reports whether reference is a server-created Pi session // directory: an absolute, clean path directly under piSessionsBase named // stage-*, that is a real directory rather than a symlink. Anything else - a // user path, a workspace, a symlink - is never sized or removed. func ownedPiSessionDir(reference string) bool { base, err := piSessionsBase() if err != nil || base == "" || reference == "" || !filepath.IsAbs(reference) || filepath.Clean(reference) != reference { return false } if filepath.Dir(reference) != base || !strings.HasPrefix(filepath.Base(reference), "stage-") { return false } info, err := os.Lstat(reference) return err == nil && info.IsDir() } // runPiSessionDirs lists the owned Pi session directories recorded for a run. func runPiSessionDirs(ctx context.Context, q sizeQuerier, id int64) ([]string, error) { rows, err := q.QueryContext(ctx, `SELECT DISTINCT reference FROM run_session_owners WHERE run_id=? AND backend='pi' AND reference<>''`, id) if err != nil { return nil, err } defer rows.Close() var dirs []string for rows.Next() { var reference string if err := rows.Scan(&reference); err != nil { return nil, err } if ownedPiSessionDir(reference) { dirs = append(dirs, reference) } } return dirs, rows.Err() } // errRunNotFound marks a size or delete request for a run id with no row. var errRunNotFound = fmt.Errorf("run not found") // computeRunSize measures one run now: artifact tree, owned Pi session // directories and the database estimate. Unreadable files are skipped rather // than failing the measurement; the size is a best-effort lower bound then. func computeRunSize(ctx context.Context, q sizeQuerier, artifactsRoot string, id int64) (int64, error) { var exists bool if err := q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pipeline_runs WHERE id=?)`, id).Scan(&exists); err != nil { return 0, err } if !exists { return 0, errRunNotFound } var dbBytes int64 if err := q.QueryRowContext(ctx, runDBBytesQuery, id).Scan(&dbBytes); err != nil { return 0, err } _, artifactBytes, _ := retentionArtifactsSize(artifactsRoot, id) total := dbBytes + artifactBytes dirs, err := runPiSessionDirs(ctx, q, id) if err != nil { return 0, err } for _, dir := range dirs { _, bytes, _ := retentionWalkSize(dir) total += bytes } return total, nil } // RunSize is one run whose cached size was (re)computed. type RunSize struct { ID int64 Workspace string SizeBytes int64 } // RefreshRunSize recomputes and caches one run's size and returns it. The value // is stored only if the run did not change while it was measured; it is // returned either way, since it is the freshest measurement available. func (s *Store) RefreshRunSize(ctx context.Context, artifactsRoot string, id int64) (int64, error) { var gen int64 if err := s.db.QueryRowContext(ctx, `SELECT size_gen FROM pipeline_runs WHERE id=?`, id).Scan(&gen); err != nil { if err == sql.ErrNoRows { return 0, errRunNotFound } return 0, err } size, err := computeRunSize(ctx, s.db, artifactsRoot, id) if err != nil { return 0, err } if _, err := s.db.ExecContext(ctx, `UPDATE pipeline_runs SET size_bytes=? WHERE id=? AND size_gen=?`, size, id, gen); err != nil { return 0, err } return size, nil } // RefreshRunSizes computes and caches the size of up to limit runs whose cached // size is missing or stale, newest first, and returns the runs it measured. A // limit of zero or less means no limit. It is the lazy backfill: callers run it // in the background or with a small limit on a request path. func (s *Store) RefreshRunSizes(ctx context.Context, artifactsRoot string, limit int) ([]RunSize, error) { if limit <= 0 { limit = -1 } rows, err := s.db.QueryContext(ctx, `SELECT id,workspace FROM pipeline_runs WHERE size_bytes<0 ORDER BY id DESC LIMIT ?`, limit) if err != nil { return nil, err } var stale []RunSize for rows.Next() { var item RunSize if err := rows.Scan(&item.ID, &item.Workspace); err != nil { rows.Close() return nil, err } stale = append(stale, item) } err = rows.Err() rows.Close() if err != nil { return nil, err } measured := make([]RunSize, 0, len(stale)) for _, item := range stale { if err := ctx.Err(); err != nil { return measured, err } size, err := s.RefreshRunSize(ctx, artifactsRoot, item.ID) if err == errRunNotFound { continue } if err != nil { return measured, err } item.SizeBytes = size measured = append(measured, item) } return measured, nil } // StaleRunSizes counts runs whose cached size is missing or stale. func (s *Store) StaleRunSizes(ctx context.Context) (int, error) { var count int err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pipeline_runs WHERE size_bytes<0`).Scan(&count) return count, err }