31e89e872b
Every run summary carries size_bytes (artifact tree, server-created Pi session directories and an estimate of the run's database rows), cached in pipeline_runs.size_bytes, invalidated by triggers, measured when a run finishes and backfilled in the background. delete_runs removes terminal runs with every scoped row and server-owned file, refuses active ones with a reason, and pushes runs_removed to subscribers. pipelines runs rm is the CLI form; retention apply now shares the complete row deletion.
265 lines
10 KiB
Go
265 lines
10 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
)
|
|
|
|
// Run size accounting. pipeline_runs.size_bytes caches the bytes on disk a run
|
|
// accounts for: its artifact tree (<artifacts>/<run>, which holds turn
|
|
// documents, inputs, outputs and session exports), any Pi session directory the
|
|
// server created for its stages, and an estimate of its database rows (the byte
|
|
// length of every large text column plus a small per-row overhead).
|
|
//
|
|
// -1 means "not computed or stale". Triggers mark a run stale whenever its row,
|
|
// a visit or visit evidence changes, and bump size_gen so a size computed
|
|
// concurrently with a change is never stored as current (see storeRunSize).
|
|
// Existing databases start with every run at -1 and are backfilled lazily.
|
|
|
|
// runSizeTriggers are the tables whose writes change a run's footprint. Each
|
|
// gets an insert and an update trigger that invalidates the owning run.
|
|
var runSizeTriggers = []string{
|
|
"run_steps", "step_notifications", "step_file_changes", "step_control", "step_artifacts",
|
|
"step_verdicts", "step_session_exports", "step_usage", "run_session_owners",
|
|
"session_export_repairs", "run_notifications", "retry_checkpoints",
|
|
}
|
|
|
|
// initRunSizes adds the cache columns and invalidation triggers. Every step is
|
|
// idempotent so a repeated Open and an upgraded database both open cleanly.
|
|
func (s *Store) initRunSizes() error {
|
|
if err := s.addColumnIfMissing("pipeline_runs", "size_bytes", "INTEGER NOT NULL DEFAULT -1"); err != nil {
|
|
return err
|
|
}
|
|
if err := s.addColumnIfMissing("pipeline_runs", "size_gen", "INTEGER NOT NULL DEFAULT 0"); err != nil {
|
|
return err
|
|
}
|
|
statements := []string{
|
|
// The nested UPDATE touches only size columns, which are not in the OF
|
|
// list, so this trigger cannot re-fire itself.
|
|
`CREATE TRIGGER IF NOT EXISTS run_size_stale_run AFTER UPDATE OF status,outcome,output_json,input_json,diagnostic,finished_at,title ON pipeline_runs
|
|
BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.id; END`,
|
|
}
|
|
for _, table := range runSizeTriggers {
|
|
for _, event := range []string{"INSERT", "UPDATE"} {
|
|
statements = append(statements, fmt.Sprintf(`CREATE TRIGGER IF NOT EXISTS run_size_stale_%s_%s AFTER %s ON %s
|
|
BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.run_id; END`, table, strings.ToLower(event), event, table))
|
|
}
|
|
}
|
|
for _, statement := range statements {
|
|
if _, err := s.db.Exec(statement); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// runSizeRowOverhead approximates SQLite's per-row cost (record header, rowid,
|
|
// index entries) beyond the measured column bytes.
|
|
const runSizeRowOverhead = 64
|
|
|
|
// runDBBytesQuery estimates one run's database footprint: the byte length of
|
|
// every variable-size column in every table scoped to the run, plus a fixed
|
|
// per-row overhead. Each subquery uses the table's run_id primary-key prefix or
|
|
// index, so the cost is bounded by the run's own rows.
|
|
var runDBBytesQuery = func() string {
|
|
type part struct {
|
|
table, where string
|
|
columns []string
|
|
}
|
|
parts := []part{
|
|
{"pipeline_runs", "id=?1", []string{"request_key", "workspace", "title", "title_source", "title_model", "model", "stage_models", "write_paths", "input_json", "output_json", "outcome", "current_stage", "diagnostic", "output_format"}},
|
|
{"run_steps", "run_id=?1", []string{"stage_key", "outcome", "target", "input_json", "output_json", "diagnostic", "provider", "model", "session_id", "transcript"}},
|
|
{"step_notifications", "run_id=?1", []string{"receipts_json"}},
|
|
{"step_file_changes", "run_id=?1", []string{"changes_json"}},
|
|
{"step_control", "run_id=?1", []string{"evidence_json"}},
|
|
{"step_artifacts", "run_id=?1", []string{"artifacts_json"}},
|
|
{"step_verdicts", "run_id=?1", []string{"verdict_json"}},
|
|
{"step_session_exports", "run_id=?1", []string{"export_json"}},
|
|
{"step_usage", "run_id=?1", []string{"usage_json"}},
|
|
{"run_session_owners", "run_id=?1", []string{"backend", "reference"}},
|
|
{"session_export_repairs", "run_id=?1", []string{"export_json"}},
|
|
{"session_export_jobs", "run_id=?1", []string{"session_id", "backend", "directory", "reference", "token", "diagnostic"}},
|
|
{"run_notifications", "run_id=?1", []string{"event", "channel", "title", "body", "click", "diagnostic", "message_id"}},
|
|
{"retry_checkpoints", "run_id=?1", []string{"mode", "handoff_json", "source_hash"}},
|
|
{"run_schedule_origin", "run_id=?1", []string{"name"}},
|
|
}
|
|
terms := make([]string, 0, len(parts))
|
|
for _, p := range parts {
|
|
lengths := make([]string, 0, len(p.columns)+1)
|
|
for _, column := range p.columns {
|
|
lengths = append(lengths, "COALESCE(length(CAST("+column+" AS BLOB)),0)")
|
|
}
|
|
lengths = append(lengths, fmt.Sprint(runSizeRowOverhead))
|
|
terms = append(terms, "COALESCE((SELECT SUM("+strings.Join(lengths, "+")+") FROM "+p.table+" WHERE "+p.where+"),0)")
|
|
}
|
|
return "SELECT " + strings.Join(terms, "+")
|
|
}()
|
|
|
|
// sizeQuerier is the read surface shared by *sql.DB and *sql.Tx, so a size is
|
|
// computed identically by the cache refresh and inside the delete transaction.
|
|
type sizeQuerier interface {
|
|
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
|
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
|
}
|
|
|
|
// piSessionsBase is the directory the Pi stage client creates its per-visit
|
|
// session directories under (see internal/stage/pi.go). It is a variable so
|
|
// tests can point it at a temporary directory.
|
|
var piSessionsBase = func() (string, error) {
|
|
cache, err := os.UserCacheDir()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return filepath.Join(cache, "pipelines", "pi-sessions"), nil
|
|
}
|
|
|
|
// ownedPiSessionDir reports whether reference is a server-created Pi session
|
|
// directory: an absolute, clean path directly under piSessionsBase named
|
|
// stage-*, that is a real directory rather than a symlink. Anything else - a
|
|
// user path, a workspace, a symlink - is never sized or removed.
|
|
func ownedPiSessionDir(reference string) bool {
|
|
base, err := piSessionsBase()
|
|
if err != nil || base == "" || reference == "" || !filepath.IsAbs(reference) || filepath.Clean(reference) != reference {
|
|
return false
|
|
}
|
|
if filepath.Dir(reference) != base || !strings.HasPrefix(filepath.Base(reference), "stage-") {
|
|
return false
|
|
}
|
|
info, err := os.Lstat(reference)
|
|
return err == nil && info.IsDir()
|
|
}
|
|
|
|
// runPiSessionDirs lists the owned Pi session directories recorded for a run.
|
|
func runPiSessionDirs(ctx context.Context, q sizeQuerier, id int64) ([]string, error) {
|
|
rows, err := q.QueryContext(ctx, `SELECT DISTINCT reference FROM run_session_owners WHERE run_id=? AND backend='pi' AND reference<>''`, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var dirs []string
|
|
for rows.Next() {
|
|
var reference string
|
|
if err := rows.Scan(&reference); err != nil {
|
|
return nil, err
|
|
}
|
|
if ownedPiSessionDir(reference) {
|
|
dirs = append(dirs, reference)
|
|
}
|
|
}
|
|
return dirs, rows.Err()
|
|
}
|
|
|
|
// errRunNotFound marks a size or delete request for a run id with no row.
|
|
var errRunNotFound = fmt.Errorf("run not found")
|
|
|
|
// computeRunSize measures one run now: artifact tree, owned Pi session
|
|
// directories and the database estimate. Unreadable files are skipped rather
|
|
// than failing the measurement; the size is a best-effort lower bound then.
|
|
func computeRunSize(ctx context.Context, q sizeQuerier, artifactsRoot string, id int64) (int64, error) {
|
|
var exists bool
|
|
if err := q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pipeline_runs WHERE id=?)`, id).Scan(&exists); err != nil {
|
|
return 0, err
|
|
}
|
|
if !exists {
|
|
return 0, errRunNotFound
|
|
}
|
|
var dbBytes int64
|
|
if err := q.QueryRowContext(ctx, runDBBytesQuery, id).Scan(&dbBytes); err != nil {
|
|
return 0, err
|
|
}
|
|
_, artifactBytes, _ := retentionArtifactsSize(artifactsRoot, id)
|
|
total := dbBytes + artifactBytes
|
|
dirs, err := runPiSessionDirs(ctx, q, id)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
for _, dir := range dirs {
|
|
_, bytes, _ := retentionWalkSize(dir)
|
|
total += bytes
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
// RunSize is one run whose cached size was (re)computed.
|
|
type RunSize struct {
|
|
ID int64
|
|
Workspace string
|
|
SizeBytes int64
|
|
}
|
|
|
|
// RefreshRunSize recomputes and caches one run's size and returns it. The value
|
|
// is stored only if the run did not change while it was measured; it is
|
|
// returned either way, since it is the freshest measurement available.
|
|
func (s *Store) RefreshRunSize(ctx context.Context, artifactsRoot string, id int64) (int64, error) {
|
|
var gen int64
|
|
if err := s.db.QueryRowContext(ctx, `SELECT size_gen FROM pipeline_runs WHERE id=?`, id).Scan(&gen); err != nil {
|
|
if err == sql.ErrNoRows {
|
|
return 0, errRunNotFound
|
|
}
|
|
return 0, err
|
|
}
|
|
size, err := computeRunSize(ctx, s.db, artifactsRoot, id)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if _, err := s.db.ExecContext(ctx, `UPDATE pipeline_runs SET size_bytes=? WHERE id=? AND size_gen=?`, size, id, gen); err != nil {
|
|
return 0, err
|
|
}
|
|
return size, nil
|
|
}
|
|
|
|
// RefreshRunSizes computes and caches the size of up to limit runs whose cached
|
|
// size is missing or stale, newest first, and returns the runs it measured. A
|
|
// limit of zero or less means no limit. It is the lazy backfill: callers run it
|
|
// in the background or with a small limit on a request path.
|
|
func (s *Store) RefreshRunSizes(ctx context.Context, artifactsRoot string, limit int) ([]RunSize, error) {
|
|
if limit <= 0 {
|
|
limit = -1
|
|
}
|
|
rows, err := s.db.QueryContext(ctx, `SELECT id,workspace FROM pipeline_runs WHERE size_bytes<0 ORDER BY id DESC LIMIT ?`, limit)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var stale []RunSize
|
|
for rows.Next() {
|
|
var item RunSize
|
|
if err := rows.Scan(&item.ID, &item.Workspace); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
stale = append(stale, item)
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
measured := make([]RunSize, 0, len(stale))
|
|
for _, item := range stale {
|
|
if err := ctx.Err(); err != nil {
|
|
return measured, err
|
|
}
|
|
size, err := s.RefreshRunSize(ctx, artifactsRoot, item.ID)
|
|
if err == errRunNotFound {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return measured, err
|
|
}
|
|
item.SizeBytes = size
|
|
measured = append(measured, item)
|
|
}
|
|
return measured, nil
|
|
}
|
|
|
|
// StaleRunSizes counts runs whose cached size is missing or stale.
|
|
func (s *Store) StaleRunSizes(ctx context.Context) (int, error) {
|
|
var count int
|
|
err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pipeline_runs WHERE size_bytes<0`).Scan(&count)
|
|
return count, err
|
|
}
|