31e89e872b
Every run summary carries size_bytes (artifact tree, server-created Pi session directories and an estimate of the run's database rows), cached in pipeline_runs.size_bytes, invalidated by triggers, measured when a run finishes and backfilled in the background. delete_runs removes terminal runs with every scoped row and server-owned file, refuses active ones with a reason, and pushes runs_removed to subscribers. pipelines runs rm is the CLI form; retention apply now shares the complete row deletion.
309 lines
11 KiB
Go
309 lines
11 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
func (f *retentionFixture) sizeColumn(t *testing.T, id int64) int64 {
|
|
t.Helper()
|
|
var size int64
|
|
if err := f.store.db.QueryRow("SELECT size_bytes FROM pipeline_runs WHERE id=?", id).Scan(&size); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return size
|
|
}
|
|
|
|
func (f *retentionFixture) listedSize(t *testing.T, id int64) int64 {
|
|
t.Helper()
|
|
runs, err := f.store.ListRuns(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, run := range runs {
|
|
if run.ID == id {
|
|
return run.SizeBytes
|
|
}
|
|
}
|
|
t.Fatalf("run %d not listed", id)
|
|
return 0
|
|
}
|
|
|
|
// usePiSessionsBase points Pi session ownership at a temporary directory so a
|
|
// test never reads or removes the real user cache.
|
|
func usePiSessionsBase(t *testing.T) string {
|
|
t.Helper()
|
|
base := filepath.Join(t.TempDir(), "pi-sessions")
|
|
if err := os.MkdirAll(base, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
previous := piSessionsBase
|
|
piSessionsBase = func() (string, error) { return base, nil }
|
|
t.Cleanup(func() { piSessionsBase = previous })
|
|
return base
|
|
}
|
|
|
|
func TestRunSizeComputedCachedAndInvalidated(t *testing.T) {
|
|
ctx := context.Background()
|
|
base := usePiSessionsBase(t)
|
|
f := newRetentionFixture(t)
|
|
id := f.run(t, "sized", 0)
|
|
f.step(t, id, 1, "completed", false)
|
|
f.artifact(t, id, 1, "report.md", 10_000)
|
|
piDir := filepath.Join(base, "stage-1")
|
|
if err := os.MkdirAll(piDir, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(piDir, "rpc.jsonl"), make([]byte, 3_000), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.db.Exec("INSERT INTO run_session_owners(run_id,sequence,backend,reference) VALUES(?,1,'pi',?)", id, piDir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := f.sizeColumn(t, id); got != -1 {
|
|
t.Fatalf("new run size = %d, want -1 (unmeasured)", got)
|
|
}
|
|
if got := f.listedSize(t, id); got != 0 {
|
|
t.Fatalf("unmeasured listed size = %d, want 0", got)
|
|
}
|
|
measured, err := f.store.RefreshRunSizes(ctx, f.root, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(measured) != 1 || measured[0].ID != id {
|
|
t.Fatalf("measured = %+v", measured)
|
|
}
|
|
size := measured[0].SizeBytes
|
|
if size < 13_000 || size > 20_000 {
|
|
t.Fatalf("size = %d, want artifacts + pi dir + modest DB estimate", size)
|
|
}
|
|
if got := f.listedSize(t, id); got != size {
|
|
t.Fatalf("listed size = %d, want cached %d", got, size)
|
|
}
|
|
again, err := f.store.RefreshRunSizes(ctx, f.root, 0)
|
|
if err != nil || len(again) != 0 {
|
|
t.Fatalf("cached run re-measured: %+v %v", again, err)
|
|
}
|
|
// A visit change invalidates the cache; the refreshed size includes it.
|
|
if _, err := f.store.db.Exec("UPDATE run_steps SET transcript=? WHERE run_id=?", string(make([]byte, 5_000)), id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := f.sizeColumn(t, id); got != -1 {
|
|
t.Fatalf("size after step update = %d, want -1", got)
|
|
}
|
|
grown, err := f.store.RefreshRunSize(ctx, f.root, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if grown < size+5_000 {
|
|
t.Fatalf("size after transcript = %d, want >= %d", grown, size+5_000)
|
|
}
|
|
// A run status change invalidates it too.
|
|
f.setStatus(t, id, "failed")
|
|
if got := f.sizeColumn(t, id); got != -1 {
|
|
t.Fatalf("size after status change = %d, want -1", got)
|
|
}
|
|
}
|
|
|
|
func TestRunSizeNotStoredWhenRunChangedDuringMeasurement(t *testing.T) {
|
|
ctx := context.Background()
|
|
f := newRetentionFixture(t)
|
|
id := f.run(t, "racy", 0)
|
|
var gen int64
|
|
if err := f.store.db.QueryRow("SELECT size_gen FROM pipeline_runs WHERE id=?", id).Scan(&gen); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.step(t, id, 1, "completed", false) // a change after the generation was read
|
|
if _, err := f.store.db.ExecContext(ctx, "UPDATE pipeline_runs SET size_bytes=? WHERE id=? AND size_gen=?", 1, id, gen); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := f.sizeColumn(t, id); got != -1 {
|
|
t.Fatalf("stale measurement stored: %d", got)
|
|
}
|
|
}
|
|
|
|
// runTables lists every table delete_runs must clear, with its run column.
|
|
var runTables = map[string]string{
|
|
"pipeline_runs": "id", "run_steps": "run_id", "step_notifications": "run_id", "step_file_changes": "run_id",
|
|
"step_control": "run_id", "step_artifacts": "run_id", "step_verdicts": "run_id", "step_session_exports": "run_id",
|
|
"step_usage": "run_id", "run_session_owners": "run_id", "session_export_repairs": "run_id",
|
|
"session_export_jobs": "run_id", "session_export_gates": "run_id", "run_notifications": "run_id",
|
|
"attention_dismissals": "run_id", "run_schedule_origin": "run_id", "run_retries": "run_id",
|
|
"retry_checkpoints": "run_id",
|
|
}
|
|
|
|
func (f *retentionFixture) seedEverything(t *testing.T, id int64, piDir string) {
|
|
t.Helper()
|
|
f.step(t, id, 1, "completed", false)
|
|
statements := []string{
|
|
"INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,1,'[]')",
|
|
"INSERT INTO step_file_changes(run_id,sequence,changes_json) VALUES(?,1,'[]')",
|
|
"INSERT INTO step_control(run_id,sequence,evidence_json) VALUES(?,1,'{}')",
|
|
"INSERT INTO step_artifacts(run_id,sequence,artifacts_json) VALUES(?,1,'{}')",
|
|
"INSERT INTO step_verdicts(run_id,sequence,verdict_json) VALUES(?,1,'{}')",
|
|
"INSERT INTO step_session_exports(run_id,sequence,export_json) VALUES(?,1,'{}')",
|
|
"INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,1,'{}')",
|
|
"INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,1,'{}')",
|
|
"INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at,status) VALUES(?,1,'s','pi','/tmp','r',0,'complete')",
|
|
"INSERT INTO session_export_gates(run_id,sequence) VALUES(?,1)",
|
|
"INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click) VALUES(?,1,'finished','system','t','b','c')",
|
|
"INSERT INTO attention_dismissals(key,run_id,dismissed_at) VALUES('run:'||?1,?1,0)",
|
|
"INSERT INTO run_schedule_origin(run_id,schedule_id,name) VALUES(?,1,'nightly')",
|
|
}
|
|
for _, statement := range statements {
|
|
if _, err := f.store.db.Exec(statement, id); err != nil {
|
|
t.Fatalf("%s: %v", statement, err)
|
|
}
|
|
}
|
|
if _, err := f.store.db.Exec("INSERT INTO run_session_owners(run_id,sequence,backend,reference) VALUES(?,1,'pi',?)", id, piDir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.artifact(t, id, 1, "out.md", 4_096)
|
|
if err := os.MkdirAll(piDir, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(piDir, "rpc.jsonl"), []byte("x"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestDeleteRunsRemovesRowsAndFiles(t *testing.T) {
|
|
ctx := context.Background()
|
|
base := usePiSessionsBase(t)
|
|
f := newRetentionFixture(t)
|
|
doomed := f.run(t, "doomed", 0)
|
|
kept := f.run(t, "kept", 0)
|
|
active := f.run(t, "active", 0)
|
|
f.setStatus(t, active, "running")
|
|
f.seedEverything(t, doomed, filepath.Join(base, "stage-doomed"))
|
|
f.seedEverything(t, kept, filepath.Join(base, "stage-kept"))
|
|
// kept is a finished retry of doomed: deleting the original drops the
|
|
// provenance edge and checkpoint but keeps the retry itself.
|
|
f.retry(t, kept, doomed)
|
|
if _, err := f.store.db.Exec("INSERT INTO retry_checkpoints(run_id,source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash) VALUES(?,?,'resume',1,'{}',1,'h')", kept, doomed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A workspace-like directory outside the data dir must survive.
|
|
workspace := filepath.Join(t.TempDir(), "repo")
|
|
if err := os.MkdirAll(workspace, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.db.Exec("UPDATE pipeline_runs SET workspace=? WHERE id=?", workspace, doomed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
expected, err := f.store.RefreshRunSize(ctx, f.root, doomed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
result, err := f.store.DeleteRuns(ctx, []int64{doomed, doomed, active, 99999, -3}, f.root, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fmt.Sprint(result.Deleted) != fmt.Sprint([]int64{doomed}) {
|
|
t.Fatalf("deleted = %v", result.Deleted)
|
|
}
|
|
reasons := map[int64]string{}
|
|
for _, refusal := range result.Refused {
|
|
reasons[refusal.RunID] = refusal.Reason
|
|
}
|
|
if len(result.Refused) != 3 || reasons[active] == "" || reasons[99999] != "run not found" || reasons[-3] != "invalid run id" {
|
|
t.Fatalf("refused = %+v", result.Refused)
|
|
}
|
|
if result.FreedBytes != expected {
|
|
t.Fatalf("freed = %d, want measured size %d", result.FreedBytes, expected)
|
|
}
|
|
if result.Workspaces[doomed] != workspace {
|
|
t.Fatalf("workspace = %q", result.Workspaces[doomed])
|
|
}
|
|
for table, column := range runTables {
|
|
var count int
|
|
if err := f.store.db.QueryRow("SELECT COUNT(*) FROM "+table+" WHERE "+column+"=?", doomed).Scan(&count); err != nil {
|
|
t.Fatalf("%s: %v", table, err)
|
|
}
|
|
if count != 0 {
|
|
t.Fatalf("%s still holds %d rows for deleted run", table, count)
|
|
}
|
|
}
|
|
for _, check := range []string{"SELECT COUNT(*) FROM run_retries WHERE original_run_id=?", "SELECT COUNT(*) FROM retry_checkpoints WHERE source_run_id=?"} {
|
|
var count int
|
|
if err := f.store.db.QueryRow(check, doomed).Scan(&count); err != nil || count != 0 {
|
|
t.Fatalf("%s = %d %v", check, count, err)
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(doomed))); !os.IsNotExist(err) {
|
|
t.Fatalf("artifacts of deleted run remain: %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(base, "stage-doomed")); !os.IsNotExist(err) {
|
|
t.Fatalf("pi session dir of deleted run remains: %v", err)
|
|
}
|
|
if entries, _ := os.ReadDir(f.root); len(entries) != 1 {
|
|
t.Fatalf("artifacts root entries = %v, want only the kept run (no staging leftovers)", entries)
|
|
}
|
|
for _, path := range []string{workspace, filepath.Join(f.root, fmt.Sprint(kept)), filepath.Join(base, "stage-kept")} {
|
|
if _, err := os.Stat(path); err != nil {
|
|
t.Fatalf("%s must survive: %v", path, err)
|
|
}
|
|
}
|
|
for table, column := range runTables {
|
|
if table == "run_retries" || table == "retry_checkpoints" {
|
|
continue
|
|
}
|
|
var count int
|
|
if err := f.store.db.QueryRow("SELECT COUNT(*) FROM "+table+" WHERE "+column+"=?", kept).Scan(&count); err != nil || count == 0 {
|
|
t.Fatalf("%s lost the kept run's rows: %d %v", table, count, err)
|
|
}
|
|
}
|
|
violations, err := f.store.db.Query("PRAGMA foreign_key_check")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if violations.Next() {
|
|
violations.Close()
|
|
t.Fatal("foreign key violation after delete")
|
|
}
|
|
violations.Close()
|
|
}
|
|
|
|
func TestDeleteRunsRefusesActiveAndDependedOnRuns(t *testing.T) {
|
|
ctx := context.Background()
|
|
f := newRetentionFixture(t)
|
|
queued := f.run(t, "queued", 0)
|
|
f.setStatus(t, queued, "queued")
|
|
waiting := f.run(t, "waiting", 0)
|
|
f.setStatus(t, waiting, "waiting")
|
|
agent := f.run(t, "agent", 0)
|
|
f.step(t, agent, 1, "failed", true)
|
|
original := f.run(t, "original", 0)
|
|
retry := f.run(t, "retry", 0)
|
|
f.setStatus(t, retry, "running")
|
|
f.retry(t, retry, original)
|
|
busy := f.run(t, "busy", 0)
|
|
f.artifact(t, busy, 1, "keep.md", 10)
|
|
|
|
result, err := f.store.DeleteRuns(ctx, []int64{queued, waiting, agent, original, busy}, f.root, func(id int64) string {
|
|
if id == busy {
|
|
return "run is executing"
|
|
}
|
|
return ""
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(result.Deleted) != 0 || len(result.Refused) != 5 || result.FreedBytes != 0 {
|
|
t.Fatalf("result = %+v", result.RunsDeletedResult)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(busy))); err != nil {
|
|
t.Fatalf("refused run lost artifacts: %v", err)
|
|
}
|
|
if got := f.count(t, "pipeline_runs"); got != 6 {
|
|
t.Fatalf("runs = %d, want 6", got)
|
|
}
|
|
if _, err := f.store.DeleteRuns(ctx, nil, f.root, nil); err == nil {
|
|
t.Fatal("empty run_ids accepted")
|
|
}
|
|
}
|