Files
pipelines/internal/pipeline/retention_test.go
T
alan 4908452a74 retention: report removed runs and follow a symlinked artifacts root
Fix two honesty gaps in run-history retention that the implementer stage
left uncommitted on top of 14175eb:

- Populate RemovedRuns, RemovedRunCount, RemovedArtifactFiles/Bytes and
  Applied right after the row transaction commits, before staging cleanup.
  A cleanup failure must not report a successful row/artifact removal as
  zero removed, and staging leftovers are excluded from SpaceReclaimedBytes.
  A failed VACUUM after removal now reports the artifact bytes actually
  removed while claiming no database pages.
- Stage artifacts through os.Stat instead of os.Lstat so a symlinked
  artifacts root is followed: the report sized that directory, so apply
  removes through it too instead of counting bytes it never deleted.

Tests cover post-commit staging-cleanup failure reporting, staging through a
symlinked artifacts root (including no staging directory left in the link
target), and an honest reclaim failure after removal.

Checked documentation surfaces:
- docs/cli.md: measured example refreshed with the current before/apply/after
  output and the symlink/staging/VACUUM reporting described accurately.
- README.md: retention prose (explicit, never automatic, protected runs,
  artifacts go with the run) is unchanged and still correct.
- internal/author/skill/references/operations.md: operator-only retention
  summary describes the same behaviour; no user-facing surface changed, so
  no reference update was needed.
- Packaged templates: retention is a CLI/store capability with no template
  change, so nothing to update. internal/author/install.go inventory
  unchanged (no reference added or renamed).

Verification: gofmt clean on the touched Go files, git diff --check clean,
go vet ./internal/pipeline clean, and go test ./internal/pipeline -run
'TestRetention' -count=1 passing.
2026-10-17 16:19:39 +01:00

813 lines
29 KiB
Go

package pipeline
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// retentionFixture seeds one pipeline definition and a writable store, and
// returns the artifact root the server would use (artifacts/ beside the db).
type retentionFixture struct {
store *Store
definition *protocol.PipelineDetail
root string
now time.Time
}
func newRetentionFixture(t *testing.T) *retentionFixture {
t.Helper()
dir := t.TempDir()
store, err := Open(filepath.Join(dir, "pipelines.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { store.Close() })
if _, err := store.Submit(context.Background(), "retention", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(context.Background())
if err != nil {
t.Fatal(err)
}
definition, err := store.Get(context.Background(), items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
return &retentionFixture{store: store, definition: definition, root: filepath.Join(dir, "artifacts"), now: time.Now().UTC().Truncate(time.Second)}
}
// run creates one run and pins its creation time relative to the fixture's now.
func (f *retentionFixture) run(t *testing.T, key string, age time.Duration) int64 {
t.Helper()
id, _, err := f.store.CreateRun(context.Background(), f.definition, key, "ws", false, nil, map[string]any{"prompt": key})
if err != nil {
t.Fatal(err)
}
f.setCreated(t, id, f.now.Add(-age))
f.setStatus(t, id, "succeeded")
return id
}
func (f *retentionFixture) checkpoint(t *testing.T) {
t.Helper()
var busy, logFrames, checkpointed int
if err := f.store.db.QueryRowContext(context.Background(), "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil {
t.Fatal(err)
}
}
func (f *retentionFixture) setCreated(t *testing.T, id int64, at time.Time) {
t.Helper()
if _, err := f.store.db.ExecContext(context.Background(), "UPDATE pipeline_runs SET created_at=? WHERE id=?", at.UTC().Format("2006-01-02 15:04:05"), id); err != nil {
t.Fatal(err)
}
}
func (f *retentionFixture) setStatus(t *testing.T, id int64, status string) {
t.Helper()
if _, err := f.store.db.ExecContext(context.Background(), "UPDATE pipeline_runs SET status=? WHERE id=?", status, id); err != nil {
t.Fatal(err)
}
}
func (f *retentionFixture) step(t *testing.T, id int64, sequence int, status string, active bool) {
t.Helper()
activeInt := 0
if active {
activeInt = 1
}
if _, err := f.store.db.ExecContext(context.Background(), "INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?,'{}')", id, sequence, "worker", status, activeInt); err != nil {
t.Fatal(err)
}
}
func (f *retentionFixture) retry(t *testing.T, retryID, originalID int64) {
t.Helper()
if _, err := f.store.db.ExecContext(context.Background(), "INSERT INTO run_retries(run_id,original_run_id) VALUES(?,?)", retryID, originalID); err != nil {
t.Fatal(err)
}
}
func (f *retentionFixture) artifact(t *testing.T, id int64, sequence int, name string, size int) {
t.Helper()
dir := filepath.Join(f.root, fmt.Sprint(id), fmt.Sprint(sequence))
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, name), make([]byte, size), 0600); err != nil {
t.Fatal(err)
}
}
func (f *retentionFixture) report(t *testing.T, policy RetentionPolicy) *RetentionReport {
t.Helper()
report, err := f.store.RetentionReport(context.Background(), policy, f.root, f.now)
if err != nil {
t.Fatal(err)
}
return report
}
func removableIDs(report *RetentionReport) []int64 {
ids := make([]int64, 0, len(report.WouldRemove))
for _, run := range report.WouldRemove {
ids = append(ids, run.ID)
}
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
return ids
}
func TestRetentionPolicySelection(t *testing.T) {
f := newRetentionFixture(t)
// Five runs, newest first: id5..id1, at 1h, 12h, 24h, 48h and 72h old.
ages := []time.Duration{0, 72 * time.Hour, 48 * time.Hour, 24 * time.Hour, 12 * time.Hour, time.Hour}
for i := 1; i <= 5; i++ {
f.run(t, fmt.Sprintf("run-%d", i), ages[i])
}
hourly := f.report(t, RetentionPolicy{OlderThan: 35 * time.Hour})
if got := removableIDs(hourly); fmt.Sprint(got) != "[1 2]" {
t.Fatalf("age-only selection = %v, want [1 2]", got)
}
byCount := f.report(t, RetentionPolicy{Keep: 2})
if got := removableIDs(byCount); fmt.Sprint(got) != "[1 2 3]" {
t.Fatalf("count-only selection = %v, want [1 2 3]", got)
}
both := f.report(t, RetentionPolicy{OlderThan: 35 * time.Hour, Keep: 2})
if got := removableIDs(both); fmt.Sprint(got) != "[1 2]" {
t.Fatalf("combined selection = %v, want the intersection [1 2]", got)
}
keepNone := f.report(t, RetentionPolicy{Keep: 0})
if got := removableIDs(keepNone); fmt.Sprint(got) != "[1 2 3 4 5]" {
t.Fatalf("keep 0 selection = %v, want all runs", got)
}
keepAll := f.report(t, RetentionPolicy{Keep: 99})
if len(keepAll.WouldRemove) != 0 {
t.Fatalf("keep 99 selected %d runs, want none", len(keepAll.WouldRemove))
}
// Exact boundary: a run created exactly at the cutoff is retained.
f.setCreated(t, 1, f.now.Add(-30*time.Hour))
boundary := f.report(t, RetentionPolicy{OlderThan: 30 * time.Hour})
for _, id := range removableIDs(boundary) {
if id == 1 {
t.Fatal("run exactly at the age cut-off must be retained")
}
}
}
func TestRetentionPolicyValidation(t *testing.T) {
for _, policy := range []RetentionPolicy{{Keep: -1}, {OlderThan: -time.Hour, Keep: -1}, {Keep: -2}} {
if err := policy.Validate(); err == nil {
t.Fatalf("invalid policy %+v was accepted", policy)
}
}
if err := (RetentionPolicy{Keep: 0}).Validate(); err != nil {
t.Fatalf("keep 0 must be a valid explicit bound: %v", err)
}
if err := (RetentionPolicy{OlderThan: time.Hour}).Validate(); err != nil {
t.Fatalf("age bound must be valid: %v", err)
}
}
func TestRetentionProtection(t *testing.T) {
f := newRetentionFixture(t)
f.run(t, "queued", 48*time.Hour)
f.run(t, "running", 48*time.Hour)
f.run(t, "waiting-run", 48*time.Hour)
f.run(t, "waiting-step", 48*time.Hour)
f.run(t, "agent-active", 48*time.Hour)
f.setStatus(t, 1, protocol.RunStatusQueued)
f.setStatus(t, 2, protocol.RunStatusRunning)
f.setStatus(t, 3, protocol.RunStatusWaiting)
f.step(t, 4, 1, protocol.StepStatusWaiting, false)
f.step(t, 5, 1, "running", true)
report := f.report(t, RetentionPolicy{OlderThan: time.Hour})
if len(report.WouldRemove) != 0 {
t.Fatalf("protected runs must never be removable: %v", removableIDs(report))
}
if len(report.Protected) != 5 {
t.Fatalf("want 5 protected runs, got %+v", report.Protected)
}
reasons := map[int64]string{}
for _, run := range report.Protected {
reasons[run.ID] = run.Reason
if run.Reason == "" {
t.Fatalf("protected run %d has no reason", run.ID)
}
}
if !strings.Contains(reasons[4], "step is waiting") || !strings.Contains(reasons[5], "agent_active") {
t.Fatalf("protection reasons wrong: %+v", reasons)
}
}
func TestRetentionRetryProvenance(t *testing.T) {
f := newRetentionFixture(t)
original := f.run(t, "original", 48*time.Hour)
retry := f.run(t, "retry", 1*time.Hour) // retained (newest 2 kept)
f.retry(t, retry, original)
report := f.report(t, RetentionPolicy{Keep: 1})
if len(report.WouldRemove) != 0 {
t.Fatalf("an original referenced by a retained retry must be protected: %v", removableIDs(report))
}
if len(report.Protected) != 1 || report.Protected[0].ID != original {
t.Fatalf("want the original protected, got %+v", report.Protected)
}
// Transitive: middle references original, retained keeps middle alive.
f2 := newRetentionFixture(t)
a := f2.run(t, "a", 72*time.Hour)
b := f2.run(t, "b", 60*time.Hour)
c := f2.run(t, "c", 1*time.Hour)
f2.retry(t, b, a)
f2.retry(t, c, b)
report2 := f2.report(t, RetentionPolicy{Keep: 1})
if len(report2.WouldRemove) != 0 || len(report2.Protected) != 2 {
t.Fatalf("transitive chain must protect both originals: remove=%v protected=%+v", removableIDs(report2), report2.Protected)
}
// If both original and retry are selected, no retained link remains and both
// may be removed.
f3 := newRetentionFixture(t)
o := f3.run(t, "o", 72*time.Hour)
r := f3.run(t, "r", 60*time.Hour)
f3.retry(t, r, o)
report3 := f3.report(t, RetentionPolicy{Keep: 0})
if len(report3.WouldRemove) != 2 || len(report3.Protected) != 0 {
t.Fatalf("both selected must be removable: remove=%v protected=%+v", removableIDs(report3), report3.Protected)
}
}
func TestRetentionApplyRemovesHistoryAndArtifacts(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
old := f.run(t, "old", 72*time.Hour)
keep := f.run(t, "keep", 1*time.Hour)
f.artifact(t, old, 1, "report.bin", 2048)
f.artifact(t, keep, 1, "keep.bin", 16)
// Step-scoped evidence for the removed run must go with it.
f.step(t, old, 1, "completed", false)
if _, err := f.store.db.ExecContext(ctx, "INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,1,'[]')", old); err != nil {
t.Fatal(err)
}
before := f.report(t, RetentionPolicy{Keep: 1})
beforePipelines, beforeVersions, beforeSubmissions := f.counts(t)
result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now)
if err != nil {
t.Fatal(err)
}
if !result.Applied || result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{old}) {
t.Fatalf("apply removed wrong runs: %+v", result)
}
if result.RemovedArtifactBytes != 2048 || result.RemovedArtifactFiles != 1 {
t.Fatalf("apply must remove the run's artifacts: %+v", result)
}
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) {
t.Fatalf("removed run artifacts still present: %v", err)
}
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(keep), "1", "keep.bin")); err != nil {
t.Fatalf("retained run artifacts must remain: %v", err)
}
after := f.report(t, RetentionPolicy{Keep: 1})
if len(after.WouldRemove) != 0 || after.TotalRuns != 1 {
t.Fatalf("removed run still listed: %+v", after)
}
if after.Pipelines != beforePipelines || after.PipelineVersions != beforeVersions {
t.Fatalf("definitions/versions changed: %d/%d -> %d/%d", beforePipelines, beforeVersions, after.Pipelines, after.PipelineVersions)
}
afterSubmissions := f.count(t, "submissions")
if afterSubmissions != beforeSubmissions {
t.Fatalf("submissions changed: %d -> %d", beforeSubmissions, afterSubmissions)
}
if n := f.count(t, "run_steps"); n != 0 {
t.Fatalf("removed run's steps must be gone: %d", n)
}
if n := f.count(t, "step_notifications"); n != 0 {
t.Fatalf("step-scoped evidence must be removed with its run: %d", n)
}
if n := f.count(t, "run_retries"); n != 0 {
t.Fatalf("retry edges must be removed: %d", n)
}
if before.ReclaimableBytes != 2048+before.DatabaseFreeBytes {
t.Fatalf("reclaimable bytes = %d, want artifacts + free pages", before.ReclaimableBytes)
}
if result.DatabaseBytesReclaimed < 0 || result.SpaceReclaimedBytes < result.RemovedArtifactBytes {
t.Fatalf("reclaimed bytes are not measured from the removal: %+v", result)
}
if !result.Vacuumed || result.DatabaseFreeBytesAfter != 0 {
t.Fatalf("apply must reclaim free pages to zero: %+v", result)
}
}
func (f *retentionFixture) counts(t *testing.T) (int, int, int) {
t.Helper()
return f.count(t, "pipelines"), f.count(t, "pipeline_versions"), f.count(t, "submissions")
}
func (f *retentionFixture) count(t *testing.T, table string) int {
t.Helper()
var n int
if err := f.store.db.QueryRowContext(context.Background(), "SELECT COUNT(*) FROM "+table).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
func TestRetentionReportIsReadOnly(t *testing.T) {
f := newRetentionFixture(t)
f.run(t, "a", 72*time.Hour)
f.run(t, "b", 1*time.Hour)
f.artifact(t, 1, 1, "a.bin", 512)
path := f.store.path
if err := f.store.Close(); err != nil {
t.Fatal(err)
}
readOnly, err := OpenReadOnly(path)
if err != nil {
t.Fatal(err)
}
defer readOnly.Close()
beforeRuns, beforeArtifacts := countFiles(t, filepath.Join(f.root))
beforeFiles := snapshotFile(t, path)
beforeWAL := snapshotFile(t, path+"-wal")
var beforeFree int64
if err := readOnly.db.QueryRowContext(context.Background(), "PRAGMA freelist_count").Scan(&beforeFree); err != nil {
t.Fatal(err)
}
report, err := readOnly.RetentionReport(context.Background(), RetentionPolicy{Keep: 1}, f.root, f.now)
if err != nil {
t.Fatal(err)
}
if report.Applied {
t.Fatal("a report must never report applied=true")
}
afterFiles := snapshotFile(t, path)
afterWAL := snapshotFile(t, path+"-wal")
var afterFree int64
if err := readOnly.db.QueryRowContext(context.Background(), "PRAGMA freelist_count").Scan(&afterFree); err != nil {
t.Fatal(err)
}
if afterFiles.size != beforeFiles.size || afterFiles.mod != beforeFiles.mod {
t.Fatalf("report changed the database file: %+v -> %+v", beforeFiles, afterFiles)
}
if afterWAL.size != beforeWAL.size {
t.Fatalf("report changed the WAL: %d -> %d", beforeWAL.size, afterWAL.size)
}
if afterFree != beforeFree {
t.Fatalf("report changed the freelist: %d -> %d", beforeFree, afterFree)
}
afterRuns, afterArtifacts := countFiles(t, filepath.Join(f.root))
if afterRuns != beforeRuns || afterArtifacts != beforeArtifacts {
t.Fatalf("report changed rows/artifacts: %d/%d -> %d/%d", beforeRuns, beforeArtifacts, afterRuns, afterArtifacts)
}
}
type fileSnapshot struct {
size int64
mod int64
}
func snapshotFile(t *testing.T, path string) fileSnapshot {
t.Helper()
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return fileSnapshot{}
}
t.Fatal(err)
}
return fileSnapshot{size: info.Size(), mod: info.ModTime().UnixNano()}
}
func countFiles(t *testing.T, root string) (dirs, files int) {
t.Helper()
_ = filepath.WalkDir(root, func(_ string, entry os.DirEntry, err error) error {
if err != nil {
return nil
}
if entry.IsDir() {
dirs++
return nil
}
files++
return nil
})
return dirs, files
}
func TestRetentionVacuumReclaimsFreedPages(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
old := f.run(t, "large", 72*time.Hour)
f.run(t, "new", 1*time.Hour)
// A large payload leaves free pages behind after deletion.
f.step(t, old, 1, "completed", false)
if _, err := f.store.db.ExecContext(ctx, "INSERT INTO step_file_changes(run_id,sequence,changes_json) VALUES(?,1,?)", old, strings.Repeat("x", 512*1024)); err != nil {
t.Fatal(err)
}
// Flush the WAL so the "before" size reflects the payload on disk.
f.checkpoint(t)
before := f.report(t, RetentionPolicy{Keep: 1})
result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now)
if err != nil {
t.Fatal(err)
}
if result.DatabaseFreeBytesAfter != 0 {
t.Fatalf("VACUUM must leave zero free pages: %+v", result)
}
if !result.Vacuumed {
t.Fatal("apply must report vacuumed=true")
}
if result.DatabaseBytesAfter >= before.DatabaseBytes {
t.Fatalf("database did not shrink after removing a large payload: %d -> %d", before.DatabaseBytes, result.DatabaseBytesAfter)
}
if result.DatabaseBytesReclaimed != before.DatabaseBytes-result.DatabaseBytesAfter {
t.Fatalf("reclaimed bytes not measured: %+v", result)
}
}
func TestRetentionVacuumFailureIsHonest(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
f.run(t, "a", time.Hour)
// Hold the database write lock from another connection so VACUUM cannot
// proceed; a short busy timeout keeps the test fast but exercises the same
// bounded-retry failure path a busy database produces.
blocker, err := sql.Open("sqlite", f.store.path)
if err != nil {
t.Fatal(err)
}
defer blocker.Close()
conn, err := blocker.Conn(ctx)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
t.Fatal(err)
}
if _, err := f.store.db.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil {
t.Fatal(err)
}
// Nothing is selected, so only the reclaim step runs and must fail honestly.
result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 5}, f.root, f.now)
if err == nil {
t.Fatalf("apply must report a VACUUM failure: %+v", result)
}
if result == nil || result.VacuumError == "" || result.Vacuumed {
t.Fatalf("vacuum_error must be set and vacuumed false: %+v", result)
}
if result.DatabaseBytesReclaimed != 0 || result.SpaceReclaimedBytes != 0 {
t.Fatalf("a failed VACUUM must not claim reclaimed pages: %+v", result)
}
}
func TestOpenMaintenanceFailsWithoutCreating(t *testing.T) {
path := filepath.Join(t.TempDir(), "missing.db")
if _, err := OpenMaintenance(path); err == nil {
t.Fatal("OpenMaintenance must fail for a missing database")
}
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("OpenMaintenance must not create the database: %v", err)
}
}
func TestRetentionToleratesMissingProvenanceAndRoot(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
f.run(t, "a", 72*time.Hour)
if _, err := f.store.db.ExecContext(ctx, "DROP TABLE run_retries"); err != nil {
t.Fatal(err)
}
report, err := f.store.RetentionReport(ctx, RetentionPolicy{Keep: 0}, filepath.Join(t.TempDir(), "absent"), f.now)
if err != nil {
t.Fatalf("a missing run_retries table must be an empty edge set: %v", err)
}
if len(report.WouldRemove) != 1 || report.ArtifactBytes != 0 {
t.Fatalf("missing artifacts root must be a zero-byte no-op: %+v", report.WouldRemove)
}
}
func TestRetentionApplyDoesNotCreateArtifactsRoot(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
id := f.run(t, "a", 72*time.Hour)
f.run(t, "b", time.Hour)
// No artifact directory exists yet; an apply that stages no run directory
// must not create the artifacts root as a side effect.
if _, err := os.Stat(f.root); !os.IsNotExist(err) {
t.Fatalf("fixture should start without an artifacts root: %v", err)
}
result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now)
if err != nil {
t.Fatal(err)
}
if result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{id}) {
t.Fatalf("apply removed wrong runs: %+v", result)
}
if _, err := os.Stat(f.root); !os.IsNotExist(err) {
t.Fatalf("apply created the artifacts root as a side effect: %v", err)
}
if n := f.count(t, "pipeline_runs"); n != 1 {
t.Fatalf("removed run still present: %d", n)
}
}
func TestRetentionRefusesFileAtRunPathAndConfinesSymlink(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
id := f.run(t, "a", 72*time.Hour)
if err := os.MkdirAll(f.root, 0700); err != nil {
t.Fatal(err)
}
// A regular file at <root>/<id> must be refused, not deleted.
victim := filepath.Join(t.TempDir(), "victim.txt")
if err := os.WriteFile(victim, []byte("precious"), 0600); err != nil {
t.Fatal(err)
}
runPath := filepath.Join(f.root, fmt.Sprint(id))
if err := os.WriteFile(runPath, []byte("not a directory"), 0600); err != nil {
t.Fatal(err)
}
if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err == nil {
t.Fatal("apply must refuse a regular file at the run path")
}
if data, err := os.ReadFile(runPath); err != nil || string(data) != "not a directory" {
t.Fatalf("regular file at run path was touched: %q %v", data, err)
}
if err := os.Remove(runPath); err != nil {
t.Fatal(err)
}
if _, err := f.store.db.ExecContext(ctx, "SELECT COUNT(*) FROM pipeline_runs WHERE id=?", id); err != nil {
t.Fatal(err)
}
var exists int
if err := f.store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM pipeline_runs WHERE id=?", id).Scan(&exists); err != nil || exists != 1 {
t.Fatalf("refused apply must delete no rows: %d %v", exists, err)
}
// A symlink at <root>/<id> removes the link only, never its target.
if err := os.Symlink(victim, runPath); err != nil {
t.Fatal(err)
}
if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err != nil {
t.Fatal(err)
}
if data, err := os.ReadFile(victim); err != nil || string(data) != "precious" {
t.Fatalf("symlink target must survive: %q %v", data, err)
}
if _, err := os.Lstat(runPath); !os.IsNotExist(err) {
t.Fatalf("symlink at run path should be removed with the run: %v", err)
}
}
func TestRetentionRollsBackOnDeleteError(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
id := f.run(t, "a", 72*time.Hour)
f.artifact(t, id, 1, "a.bin", 256)
f.step(t, id, 1, "completed", false)
if _, err := f.store.db.ExecContext(ctx, "CREATE TRIGGER block_run_delete BEFORE DELETE ON pipeline_runs BEGIN SELECT RAISE(ABORT,'blocked'); END;"); err != nil {
t.Fatal(err)
}
if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err == nil {
t.Fatal("apply must return the delete error")
}
// Rows and staged artifacts are restored.
if n := f.count(t, "pipeline_runs"); n != 1 {
t.Fatalf("rollback lost the run: %d", n)
}
if n := f.count(t, "run_steps"); n != 1 {
t.Fatalf("rollback lost the step: %d", n)
}
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(id), "1", "a.bin")); err != nil {
t.Fatalf("rollback must restore staged artifacts: %v", err)
}
entries, err := os.ReadDir(f.root)
if err != nil {
t.Fatal(err)
}
for _, entry := range entries {
if strings.HasPrefix(entry.Name(), ".retention-staging-") {
t.Fatalf("staging directory left behind: %s", entry.Name())
}
}
}
func TestRetentionAbortsOnProtectionRace(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
original := f.run(t, "original", 72*time.Hour)
f.run(t, "keep", 1*time.Hour)
f.artifact(t, original, 1, "a.bin", 256)
report := f.report(t, RetentionPolicy{Keep: 1})
ids := removableIDs(report)
if len(ids) != 1 {
t.Fatalf("expected one removable run, got %v", ids)
}
// A retained retry appears between the report and the delete.
retry := f.run(t, "retry", time.Hour)
f.retry(t, retry, original)
result := &RetentionApplyResult{}
err := f.store.removeRetentionRuns(ctx, RetentionPolicy{Keep: 1}, f.root, f.now, ids, result)
if err == nil || !strings.Contains(err.Error(), "became protected") {
t.Fatalf("apply must abort when a selected run becomes protected: %v", err)
}
if n := f.count(t, "pipeline_runs"); n != 3 {
t.Fatalf("aborted apply deleted rows: %d", n)
}
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(original), "1", "a.bin")); err != nil {
t.Fatalf("aborted apply touched artifacts: %v", err)
}
}
// TestRetentionReportsRemovedRunsWhenStagingCleanupFails covers the post-commit
// inspection gap where a successful row/artifact removal was reported as zero
// removed when the staging directory could not be deleted: the actual removed
// run(s) and artifact totals must still be reported, and the leftover staging
// bytes must not be counted as reclaimed.
func TestRetentionReportsRemovedRunsWhenStagingCleanupFails(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
old := f.run(t, "old", 72*time.Hour)
f.run(t, "keep", time.Hour)
// A run directory whose nested subdirectory cannot be emptied: os.RemoveAll
// fails on it, so the post-commit staging cleanup returns an error.
dir := filepath.Join(f.root, fmt.Sprint(old), "1")
locked := filepath.Join(dir, "locked")
if err := os.MkdirAll(locked, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "a.bin"), make([]byte, 128), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(locked, "inside.bin"), make([]byte, 64), 0600); err != nil {
t.Fatal(err)
}
if err := os.Chmod(locked, 0500); err != nil {
t.Fatal(err)
}
// Restore write permission before t.TempDir's cleanup removes the tree.
t.Cleanup(func() {
_ = filepath.WalkDir(f.root, func(path string, entry os.DirEntry, err error) error {
if err == nil && entry.IsDir() {
_ = os.Chmod(path, 0700)
}
return nil
})
})
result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now)
if err == nil {
t.Fatal("apply must report the staging cleanup failure")
}
if result == nil {
t.Fatal("a cleanup failure must still report what was removed")
}
if result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{old}) {
t.Fatalf("rows were removed but not reported: %+v", result)
}
if !result.Applied {
t.Fatalf("run history was removed, so applied must be true: %+v", result)
}
if result.RemovedArtifactBytes != 192 || result.RemovedArtifactFiles != 2 {
t.Fatalf("removed artifact totals wrong: %+v", result)
}
if result.StagingPath == "" || result.StagingRemainingBytes != 64 || result.StagingRemainingFiles != 1 {
t.Fatalf("staging leftovers not reported: %+v", result)
}
if result.SpaceReclaimedBytes != result.RemovedArtifactBytes-result.StagingRemainingBytes {
t.Fatalf("reclaimed space must exclude staging leftovers: %+v", result)
}
if n := f.count(t, "pipeline_runs"); n != 1 {
t.Fatalf("the run rows must be gone: %d", n)
}
if _, err := os.Lstat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) {
t.Fatalf("the run tree must have been staged out of the artifacts root: %v", err)
}
}
// TestRetentionStagesThroughSymlinkedArtifactsRoot covers the inspection gap
// where a symlinked artifacts root was sized by the report but skipped by
// apply, claiming artifact bytes as removed while leaving every file in place.
// A symlinked root is followed consistently, so the artifacts really go.
func TestRetentionStagesThroughSymlinkedArtifactsRoot(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
id := f.run(t, "old", 72*time.Hour)
f.run(t, "keep", time.Hour)
real := filepath.Join(t.TempDir(), "real-artifacts")
if err := os.MkdirAll(filepath.Join(real, fmt.Sprint(id), "1"), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(real, fmt.Sprint(id), "1", "a.bin"), make([]byte, 4096), 0600); err != nil {
t.Fatal(err)
}
link := filepath.Join(t.TempDir(), "artifacts")
if err := os.Symlink(real, link); err != nil {
t.Fatal(err)
}
report, err := f.store.RetentionReport(ctx, RetentionPolicy{Keep: 1}, link, f.now)
if err != nil {
t.Fatal(err)
}
if report.ArtifactBytes != 4096 {
t.Fatalf("report must size artifacts through the symlinked root: %+v", report)
}
result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, link, f.now)
if err != nil {
t.Fatal(err)
}
if result.RemovedRunCount != 1 || result.RemovedArtifactBytes != 4096 {
t.Fatalf("a symlinked root must still remove and report the artifacts: %+v", result)
}
if result.SpaceReclaimedBytes < result.RemovedArtifactBytes {
t.Fatalf("reclaimed space must include the removed artifacts: %+v", result)
}
if _, err := os.Stat(filepath.Join(real, fmt.Sprint(id))); !os.IsNotExist(err) {
t.Fatalf("artifacts behind a symlinked root must actually be removed: %v", err)
}
entries, err := os.ReadDir(real)
if err != nil {
t.Fatal(err)
}
for _, entry := range entries {
if strings.HasPrefix(entry.Name(), ".retention-staging-") {
t.Fatalf("staging directory left in the symlink target: %s", entry.Name())
}
}
}
// TestRetentionReclaimFailureAfterRemovalIsHonest covers a reclaim failure that
// happens after the rows and artifacts are already removed: the removed
// artifact bytes must still be reported, the database pages must not be
// claimed, and the total must not be zero. A concurrent read snapshot blocks
// the final WAL checkpoint while leaving the write transaction and VACUUM free
// to run; both checkpoint and VACUUM failures route through vacuumAndCheckpoint
// and the same honest reporting path.
func TestRetentionReclaimFailureAfterRemovalIsHonest(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
old := f.run(t, "old", 72*time.Hour)
f.run(t, "keep", time.Hour)
f.artifact(t, old, 1, "a.bin", 4096)
reader, err := sql.Open("sqlite", f.store.path)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
conn, err := reader.Conn(ctx)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if _, err := conn.ExecContext(ctx, "BEGIN"); err != nil {
t.Fatal(err)
}
var n int
if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM pipeline_runs").Scan(&n); err != nil {
t.Fatal(err)
}
if _, err := f.store.db.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil {
t.Fatal(err)
}
result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now)
if err == nil {
t.Fatalf("apply must report the reclaim failure: %+v", result)
}
if result == nil || result.VacuumError == "" || result.Vacuumed {
t.Fatalf("reclaim failure must set vacuum_error and leave vacuumed false: %+v", result)
}
if result.RemovedRunCount != 1 || result.RemovedArtifactBytes != 4096 || !result.Applied {
t.Fatalf("removed artifacts must still be reported: %+v", result)
}
if result.DatabaseBytesReclaimed != 0 {
t.Fatalf("a failed reclaim must not claim database pages: %+v", result)
}
if result.SpaceReclaimedBytes != result.RemovedArtifactBytes {
t.Fatalf("space reclaimed must be the removed artifacts and no database pages: %+v", result)
}
if n := f.count(t, "pipeline_runs"); n != 1 {
t.Fatalf("the removed run must be gone: %d", n)
}
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) {
t.Fatalf("the removed run artifacts must be gone: %v", err)
}
if _, err := conn.ExecContext(ctx, "COMMIT"); err != nil {
t.Fatal(err)
}
}