package pipeline import ( "context" "database/sql" "fmt" "os" "path/filepath" "sort" "strings" "testing" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) // retentionFixture seeds one pipeline definition and a writable store, and // returns the artifact root the server would use (artifacts/ beside the db). type retentionFixture struct { store *Store definition *protocol.PipelineDetail root string now time.Time } func newRetentionFixture(t *testing.T) *retentionFixture { t.Helper() dir := t.TempDir() store, err := Open(filepath.Join(dir, "pipelines.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { store.Close() }) if _, err := store.Submit(context.Background(), "retention", Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, err := store.List(context.Background()) if err != nil { t.Fatal(err) } definition, err := store.Get(context.Background(), items[0].ID, 1) if err != nil { t.Fatal(err) } return &retentionFixture{store: store, definition: definition, root: filepath.Join(dir, "artifacts"), now: time.Now().UTC().Truncate(time.Second)} } // run creates one run and pins its creation time relative to the fixture's now. func (f *retentionFixture) run(t *testing.T, key string, age time.Duration) int64 { t.Helper() id, _, err := f.store.CreateRun(context.Background(), f.definition, key, "ws", false, nil, map[string]any{"prompt": key}) if err != nil { t.Fatal(err) } f.setCreated(t, id, f.now.Add(-age)) f.setStatus(t, id, "succeeded") return id } func (f *retentionFixture) checkpoint(t *testing.T) { t.Helper() var busy, logFrames, checkpointed int if err := f.store.db.QueryRowContext(context.Background(), "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil { t.Fatal(err) } } func (f *retentionFixture) setCreated(t *testing.T, id int64, at time.Time) { t.Helper() if _, err := f.store.db.ExecContext(context.Background(), "UPDATE pipeline_runs SET created_at=? WHERE id=?", at.UTC().Format("2006-01-02 15:04:05"), id); err != nil { t.Fatal(err) } } func (f *retentionFixture) setStatus(t *testing.T, id int64, status string) { t.Helper() if _, err := f.store.db.ExecContext(context.Background(), "UPDATE pipeline_runs SET status=? WHERE id=?", status, id); err != nil { t.Fatal(err) } } func (f *retentionFixture) step(t *testing.T, id int64, sequence int, status string, active bool) { t.Helper() activeInt := 0 if active { activeInt = 1 } if _, err := f.store.db.ExecContext(context.Background(), "INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?,'{}')", id, sequence, "worker", status, activeInt); err != nil { t.Fatal(err) } } func (f *retentionFixture) retry(t *testing.T, retryID, originalID int64) { t.Helper() if _, err := f.store.db.ExecContext(context.Background(), "INSERT INTO run_retries(run_id,original_run_id) VALUES(?,?)", retryID, originalID); err != nil { t.Fatal(err) } } func (f *retentionFixture) artifact(t *testing.T, id int64, sequence int, name string, size int) { t.Helper() dir := filepath.Join(f.root, fmt.Sprint(id), fmt.Sprint(sequence)) if err := os.MkdirAll(dir, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, name), make([]byte, size), 0600); err != nil { t.Fatal(err) } } func (f *retentionFixture) report(t *testing.T, policy RetentionPolicy) *RetentionReport { t.Helper() report, err := f.store.RetentionReport(context.Background(), policy, f.root, f.now) if err != nil { t.Fatal(err) } return report } func removableIDs(report *RetentionReport) []int64 { ids := make([]int64, 0, len(report.WouldRemove)) for _, run := range report.WouldRemove { ids = append(ids, run.ID) } sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) return ids } func TestRetentionPolicySelection(t *testing.T) { f := newRetentionFixture(t) // Five runs, newest first: id5..id1, at 1h, 12h, 24h, 48h and 72h old. ages := []time.Duration{0, 72 * time.Hour, 48 * time.Hour, 24 * time.Hour, 12 * time.Hour, time.Hour} for i := 1; i <= 5; i++ { f.run(t, fmt.Sprintf("run-%d", i), ages[i]) } hourly := f.report(t, RetentionPolicy{OlderThan: 35 * time.Hour}) if got := removableIDs(hourly); fmt.Sprint(got) != "[1 2]" { t.Fatalf("age-only selection = %v, want [1 2]", got) } byCount := f.report(t, RetentionPolicy{Keep: 2}) if got := removableIDs(byCount); fmt.Sprint(got) != "[1 2 3]" { t.Fatalf("count-only selection = %v, want [1 2 3]", got) } both := f.report(t, RetentionPolicy{OlderThan: 35 * time.Hour, Keep: 2}) if got := removableIDs(both); fmt.Sprint(got) != "[1 2]" { t.Fatalf("combined selection = %v, want the intersection [1 2]", got) } keepNone := f.report(t, RetentionPolicy{Keep: 0}) if got := removableIDs(keepNone); fmt.Sprint(got) != "[1 2 3 4 5]" { t.Fatalf("keep 0 selection = %v, want all runs", got) } keepAll := f.report(t, RetentionPolicy{Keep: 99}) if len(keepAll.WouldRemove) != 0 { t.Fatalf("keep 99 selected %d runs, want none", len(keepAll.WouldRemove)) } // Exact boundary: a run created exactly at the cutoff is retained. f.setCreated(t, 1, f.now.Add(-30*time.Hour)) boundary := f.report(t, RetentionPolicy{OlderThan: 30 * time.Hour}) for _, id := range removableIDs(boundary) { if id == 1 { t.Fatal("run exactly at the age cut-off must be retained") } } } func TestRetentionPolicyValidation(t *testing.T) { for _, policy := range []RetentionPolicy{{Keep: -1}, {OlderThan: -time.Hour, Keep: -1}, {Keep: -2}} { if err := policy.Validate(); err == nil { t.Fatalf("invalid policy %+v was accepted", policy) } } if err := (RetentionPolicy{Keep: 0}).Validate(); err != nil { t.Fatalf("keep 0 must be a valid explicit bound: %v", err) } if err := (RetentionPolicy{OlderThan: time.Hour}).Validate(); err != nil { t.Fatalf("age bound must be valid: %v", err) } } func TestRetentionProtection(t *testing.T) { f := newRetentionFixture(t) f.run(t, "queued", 48*time.Hour) f.run(t, "running", 48*time.Hour) f.run(t, "waiting-run", 48*time.Hour) f.run(t, "waiting-step", 48*time.Hour) f.run(t, "agent-active", 48*time.Hour) f.setStatus(t, 1, protocol.RunStatusQueued) f.setStatus(t, 2, protocol.RunStatusRunning) f.setStatus(t, 3, protocol.RunStatusWaiting) f.step(t, 4, 1, protocol.StepStatusWaiting, false) f.step(t, 5, 1, "running", true) report := f.report(t, RetentionPolicy{OlderThan: time.Hour}) if len(report.WouldRemove) != 0 { t.Fatalf("protected runs must never be removable: %v", removableIDs(report)) } if len(report.Protected) != 5 { t.Fatalf("want 5 protected runs, got %+v", report.Protected) } reasons := map[int64]string{} for _, run := range report.Protected { reasons[run.ID] = run.Reason if run.Reason == "" { t.Fatalf("protected run %d has no reason", run.ID) } } if !strings.Contains(reasons[4], "step is waiting") || !strings.Contains(reasons[5], "agent_active") { t.Fatalf("protection reasons wrong: %+v", reasons) } } func TestRetentionRetryProvenance(t *testing.T) { f := newRetentionFixture(t) original := f.run(t, "original", 48*time.Hour) retry := f.run(t, "retry", 1*time.Hour) // retained (newest 2 kept) f.retry(t, retry, original) report := f.report(t, RetentionPolicy{Keep: 1}) if len(report.WouldRemove) != 0 { t.Fatalf("an original referenced by a retained retry must be protected: %v", removableIDs(report)) } if len(report.Protected) != 1 || report.Protected[0].ID != original { t.Fatalf("want the original protected, got %+v", report.Protected) } // Transitive: middle references original, retained keeps middle alive. f2 := newRetentionFixture(t) a := f2.run(t, "a", 72*time.Hour) b := f2.run(t, "b", 60*time.Hour) c := f2.run(t, "c", 1*time.Hour) f2.retry(t, b, a) f2.retry(t, c, b) report2 := f2.report(t, RetentionPolicy{Keep: 1}) if len(report2.WouldRemove) != 0 || len(report2.Protected) != 2 { t.Fatalf("transitive chain must protect both originals: remove=%v protected=%+v", removableIDs(report2), report2.Protected) } // If both original and retry are selected, no retained link remains and both // may be removed. f3 := newRetentionFixture(t) o := f3.run(t, "o", 72*time.Hour) r := f3.run(t, "r", 60*time.Hour) f3.retry(t, r, o) report3 := f3.report(t, RetentionPolicy{Keep: 0}) if len(report3.WouldRemove) != 2 || len(report3.Protected) != 0 { t.Fatalf("both selected must be removable: remove=%v protected=%+v", removableIDs(report3), report3.Protected) } } func TestRetentionApplyRemovesHistoryAndArtifacts(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) old := f.run(t, "old", 72*time.Hour) keep := f.run(t, "keep", 1*time.Hour) f.artifact(t, old, 1, "report.bin", 2048) f.artifact(t, keep, 1, "keep.bin", 16) // Step-scoped evidence for the removed run must go with it. f.step(t, old, 1, "completed", false) if _, err := f.store.db.ExecContext(ctx, "INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,1,'[]')", old); err != nil { t.Fatal(err) } before := f.report(t, RetentionPolicy{Keep: 1}) beforePipelines, beforeVersions, beforeSubmissions := f.counts(t) result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) if err != nil { t.Fatal(err) } if !result.Applied || result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{old}) { t.Fatalf("apply removed wrong runs: %+v", result) } if result.RemovedArtifactBytes != 2048 || result.RemovedArtifactFiles != 1 { t.Fatalf("apply must remove the run's artifacts: %+v", result) } if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) { t.Fatalf("removed run artifacts still present: %v", err) } if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(keep), "1", "keep.bin")); err != nil { t.Fatalf("retained run artifacts must remain: %v", err) } after := f.report(t, RetentionPolicy{Keep: 1}) if len(after.WouldRemove) != 0 || after.TotalRuns != 1 { t.Fatalf("removed run still listed: %+v", after) } if after.Pipelines != beforePipelines || after.PipelineVersions != beforeVersions { t.Fatalf("definitions/versions changed: %d/%d -> %d/%d", beforePipelines, beforeVersions, after.Pipelines, after.PipelineVersions) } afterSubmissions := f.count(t, "submissions") if afterSubmissions != beforeSubmissions { t.Fatalf("submissions changed: %d -> %d", beforeSubmissions, afterSubmissions) } if n := f.count(t, "run_steps"); n != 0 { t.Fatalf("removed run's steps must be gone: %d", n) } if n := f.count(t, "step_notifications"); n != 0 { t.Fatalf("step-scoped evidence must be removed with its run: %d", n) } if n := f.count(t, "run_retries"); n != 0 { t.Fatalf("retry edges must be removed: %d", n) } if before.ReclaimableBytes != 2048+before.DatabaseFreeBytes { t.Fatalf("reclaimable bytes = %d, want artifacts + free pages", before.ReclaimableBytes) } if result.DatabaseBytesReclaimed < 0 || result.SpaceReclaimedBytes < result.RemovedArtifactBytes { t.Fatalf("reclaimed bytes are not measured from the removal: %+v", result) } if !result.Vacuumed || result.DatabaseFreeBytesAfter != 0 { t.Fatalf("apply must reclaim free pages to zero: %+v", result) } } func (f *retentionFixture) counts(t *testing.T) (int, int, int) { t.Helper() return f.count(t, "pipelines"), f.count(t, "pipeline_versions"), f.count(t, "submissions") } func (f *retentionFixture) count(t *testing.T, table string) int { t.Helper() var n int if err := f.store.db.QueryRowContext(context.Background(), "SELECT COUNT(*) FROM "+table).Scan(&n); err != nil { t.Fatal(err) } return n } func TestRetentionReportIsReadOnly(t *testing.T) { f := newRetentionFixture(t) f.run(t, "a", 72*time.Hour) f.run(t, "b", 1*time.Hour) f.artifact(t, 1, 1, "a.bin", 512) path := f.store.path if err := f.store.Close(); err != nil { t.Fatal(err) } readOnly, err := OpenReadOnly(path) if err != nil { t.Fatal(err) } defer readOnly.Close() beforeRuns, beforeArtifacts := countFiles(t, filepath.Join(f.root)) beforeFiles := snapshotFile(t, path) beforeWAL := snapshotFile(t, path+"-wal") var beforeFree int64 if err := readOnly.db.QueryRowContext(context.Background(), "PRAGMA freelist_count").Scan(&beforeFree); err != nil { t.Fatal(err) } report, err := readOnly.RetentionReport(context.Background(), RetentionPolicy{Keep: 1}, f.root, f.now) if err != nil { t.Fatal(err) } if report.Applied { t.Fatal("a report must never report applied=true") } afterFiles := snapshotFile(t, path) afterWAL := snapshotFile(t, path+"-wal") var afterFree int64 if err := readOnly.db.QueryRowContext(context.Background(), "PRAGMA freelist_count").Scan(&afterFree); err != nil { t.Fatal(err) } if afterFiles.size != beforeFiles.size || afterFiles.mod != beforeFiles.mod { t.Fatalf("report changed the database file: %+v -> %+v", beforeFiles, afterFiles) } if afterWAL.size != beforeWAL.size { t.Fatalf("report changed the WAL: %d -> %d", beforeWAL.size, afterWAL.size) } if afterFree != beforeFree { t.Fatalf("report changed the freelist: %d -> %d", beforeFree, afterFree) } afterRuns, afterArtifacts := countFiles(t, filepath.Join(f.root)) if afterRuns != beforeRuns || afterArtifacts != beforeArtifacts { t.Fatalf("report changed rows/artifacts: %d/%d -> %d/%d", beforeRuns, beforeArtifacts, afterRuns, afterArtifacts) } } type fileSnapshot struct { size int64 mod int64 } func snapshotFile(t *testing.T, path string) fileSnapshot { t.Helper() info, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return fileSnapshot{} } t.Fatal(err) } return fileSnapshot{size: info.Size(), mod: info.ModTime().UnixNano()} } func countFiles(t *testing.T, root string) (dirs, files int) { t.Helper() _ = filepath.WalkDir(root, func(_ string, entry os.DirEntry, err error) error { if err != nil { return nil } if entry.IsDir() { dirs++ return nil } files++ return nil }) return dirs, files } func TestRetentionVacuumReclaimsFreedPages(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) old := f.run(t, "large", 72*time.Hour) f.run(t, "new", 1*time.Hour) // A large payload leaves free pages behind after deletion. f.step(t, old, 1, "completed", false) if _, err := f.store.db.ExecContext(ctx, "INSERT INTO step_file_changes(run_id,sequence,changes_json) VALUES(?,1,?)", old, strings.Repeat("x", 512*1024)); err != nil { t.Fatal(err) } // Flush the WAL so the "before" size reflects the payload on disk. f.checkpoint(t) before := f.report(t, RetentionPolicy{Keep: 1}) result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) if err != nil { t.Fatal(err) } if result.DatabaseFreeBytesAfter != 0 { t.Fatalf("VACUUM must leave zero free pages: %+v", result) } if !result.Vacuumed { t.Fatal("apply must report vacuumed=true") } if result.DatabaseBytesAfter >= before.DatabaseBytes { t.Fatalf("database did not shrink after removing a large payload: %d -> %d", before.DatabaseBytes, result.DatabaseBytesAfter) } if result.DatabaseBytesReclaimed != before.DatabaseBytes-result.DatabaseBytesAfter { t.Fatalf("reclaimed bytes not measured: %+v", result) } } func TestRetentionVacuumFailureIsHonest(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) f.run(t, "a", time.Hour) // Hold the database write lock from another connection so VACUUM cannot // proceed; a short busy timeout keeps the test fast but exercises the same // bounded-retry failure path a busy database produces. blocker, err := sql.Open("sqlite", f.store.path) if err != nil { t.Fatal(err) } defer blocker.Close() conn, err := blocker.Conn(ctx) if err != nil { t.Fatal(err) } defer conn.Close() if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { t.Fatal(err) } if _, err := f.store.db.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil { t.Fatal(err) } // Nothing is selected, so only the reclaim step runs and must fail honestly. result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 5}, f.root, f.now) if err == nil { t.Fatalf("apply must report a VACUUM failure: %+v", result) } if result == nil || result.VacuumError == "" || result.Vacuumed { t.Fatalf("vacuum_error must be set and vacuumed false: %+v", result) } if result.DatabaseBytesReclaimed != 0 || result.SpaceReclaimedBytes != 0 { t.Fatalf("a failed VACUUM must not claim reclaimed pages: %+v", result) } } func TestOpenMaintenanceFailsWithoutCreating(t *testing.T) { path := filepath.Join(t.TempDir(), "missing.db") if _, err := OpenMaintenance(path); err == nil { t.Fatal("OpenMaintenance must fail for a missing database") } if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("OpenMaintenance must not create the database: %v", err) } } func TestRetentionToleratesMissingProvenanceAndRoot(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) f.run(t, "a", 72*time.Hour) if _, err := f.store.db.ExecContext(ctx, "DROP TABLE run_retries"); err != nil { t.Fatal(err) } report, err := f.store.RetentionReport(ctx, RetentionPolicy{Keep: 0}, filepath.Join(t.TempDir(), "absent"), f.now) if err != nil { t.Fatalf("a missing run_retries table must be an empty edge set: %v", err) } if len(report.WouldRemove) != 1 || report.ArtifactBytes != 0 { t.Fatalf("missing artifacts root must be a zero-byte no-op: %+v", report.WouldRemove) } } func TestRetentionApplyDoesNotCreateArtifactsRoot(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) id := f.run(t, "a", 72*time.Hour) f.run(t, "b", time.Hour) // No artifact directory exists yet; an apply that stages no run directory // must not create the artifacts root as a side effect. if _, err := os.Stat(f.root); !os.IsNotExist(err) { t.Fatalf("fixture should start without an artifacts root: %v", err) } result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) if err != nil { t.Fatal(err) } if result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{id}) { t.Fatalf("apply removed wrong runs: %+v", result) } if _, err := os.Stat(f.root); !os.IsNotExist(err) { t.Fatalf("apply created the artifacts root as a side effect: %v", err) } if n := f.count(t, "pipeline_runs"); n != 1 { t.Fatalf("removed run still present: %d", n) } } func TestRetentionRefusesFileAtRunPathAndConfinesSymlink(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) id := f.run(t, "a", 72*time.Hour) if err := os.MkdirAll(f.root, 0700); err != nil { t.Fatal(err) } // A regular file at / must be refused, not deleted. victim := filepath.Join(t.TempDir(), "victim.txt") if err := os.WriteFile(victim, []byte("precious"), 0600); err != nil { t.Fatal(err) } runPath := filepath.Join(f.root, fmt.Sprint(id)) if err := os.WriteFile(runPath, []byte("not a directory"), 0600); err != nil { t.Fatal(err) } if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err == nil { t.Fatal("apply must refuse a regular file at the run path") } if data, err := os.ReadFile(runPath); err != nil || string(data) != "not a directory" { t.Fatalf("regular file at run path was touched: %q %v", data, err) } if err := os.Remove(runPath); err != nil { t.Fatal(err) } if _, err := f.store.db.ExecContext(ctx, "SELECT COUNT(*) FROM pipeline_runs WHERE id=?", id); err != nil { t.Fatal(err) } var exists int if err := f.store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM pipeline_runs WHERE id=?", id).Scan(&exists); err != nil || exists != 1 { t.Fatalf("refused apply must delete no rows: %d %v", exists, err) } // A symlink at / removes the link only, never its target. if err := os.Symlink(victim, runPath); err != nil { t.Fatal(err) } if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err != nil { t.Fatal(err) } if data, err := os.ReadFile(victim); err != nil || string(data) != "precious" { t.Fatalf("symlink target must survive: %q %v", data, err) } if _, err := os.Lstat(runPath); !os.IsNotExist(err) { t.Fatalf("symlink at run path should be removed with the run: %v", err) } } func TestRetentionRollsBackOnDeleteError(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) id := f.run(t, "a", 72*time.Hour) f.artifact(t, id, 1, "a.bin", 256) f.step(t, id, 1, "completed", false) if _, err := f.store.db.ExecContext(ctx, "CREATE TRIGGER block_run_delete BEFORE DELETE ON pipeline_runs BEGIN SELECT RAISE(ABORT,'blocked'); END;"); err != nil { t.Fatal(err) } if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err == nil { t.Fatal("apply must return the delete error") } // Rows and staged artifacts are restored. if n := f.count(t, "pipeline_runs"); n != 1 { t.Fatalf("rollback lost the run: %d", n) } if n := f.count(t, "run_steps"); n != 1 { t.Fatalf("rollback lost the step: %d", n) } if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(id), "1", "a.bin")); err != nil { t.Fatalf("rollback must restore staged artifacts: %v", err) } entries, err := os.ReadDir(f.root) if err != nil { t.Fatal(err) } for _, entry := range entries { if strings.HasPrefix(entry.Name(), ".retention-staging-") { t.Fatalf("staging directory left behind: %s", entry.Name()) } } } func TestRetentionAbortsOnProtectionRace(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) original := f.run(t, "original", 72*time.Hour) f.run(t, "keep", 1*time.Hour) f.artifact(t, original, 1, "a.bin", 256) report := f.report(t, RetentionPolicy{Keep: 1}) ids := removableIDs(report) if len(ids) != 1 { t.Fatalf("expected one removable run, got %v", ids) } // A retained retry appears between the report and the delete. retry := f.run(t, "retry", time.Hour) f.retry(t, retry, original) result := &RetentionApplyResult{} err := f.store.removeRetentionRuns(ctx, RetentionPolicy{Keep: 1}, f.root, f.now, ids, result) if err == nil || !strings.Contains(err.Error(), "became protected") { t.Fatalf("apply must abort when a selected run becomes protected: %v", err) } if n := f.count(t, "pipeline_runs"); n != 3 { t.Fatalf("aborted apply deleted rows: %d", n) } if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(original), "1", "a.bin")); err != nil { t.Fatalf("aborted apply touched artifacts: %v", err) } } // TestRetentionReportsRemovedRunsWhenStagingCleanupFails covers the post-commit // inspection gap where a successful row/artifact removal was reported as zero // removed when the staging directory could not be deleted: the actual removed // run(s) and artifact totals must still be reported, and the leftover staging // bytes must not be counted as reclaimed. func TestRetentionReportsRemovedRunsWhenStagingCleanupFails(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) old := f.run(t, "old", 72*time.Hour) f.run(t, "keep", time.Hour) // A run directory whose nested subdirectory cannot be emptied: os.RemoveAll // fails on it, so the post-commit staging cleanup returns an error. dir := filepath.Join(f.root, fmt.Sprint(old), "1") locked := filepath.Join(dir, "locked") if err := os.MkdirAll(locked, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "a.bin"), make([]byte, 128), 0600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(locked, "inside.bin"), make([]byte, 64), 0600); err != nil { t.Fatal(err) } if err := os.Chmod(locked, 0500); err != nil { t.Fatal(err) } // Restore write permission before t.TempDir's cleanup removes the tree. t.Cleanup(func() { _ = filepath.WalkDir(f.root, func(path string, entry os.DirEntry, err error) error { if err == nil && entry.IsDir() { _ = os.Chmod(path, 0700) } return nil }) }) result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) if err == nil { t.Fatal("apply must report the staging cleanup failure") } if result == nil { t.Fatal("a cleanup failure must still report what was removed") } if result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{old}) { t.Fatalf("rows were removed but not reported: %+v", result) } if !result.Applied { t.Fatalf("run history was removed, so applied must be true: %+v", result) } if result.RemovedArtifactBytes != 192 || result.RemovedArtifactFiles != 2 { t.Fatalf("removed artifact totals wrong: %+v", result) } if result.StagingPath == "" || result.StagingRemainingBytes != 64 || result.StagingRemainingFiles != 1 { t.Fatalf("staging leftovers not reported: %+v", result) } if result.SpaceReclaimedBytes != result.RemovedArtifactBytes-result.StagingRemainingBytes { t.Fatalf("reclaimed space must exclude staging leftovers: %+v", result) } if n := f.count(t, "pipeline_runs"); n != 1 { t.Fatalf("the run rows must be gone: %d", n) } if _, err := os.Lstat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) { t.Fatalf("the run tree must have been staged out of the artifacts root: %v", err) } } // TestRetentionStagesThroughSymlinkedArtifactsRoot covers the inspection gap // where a symlinked artifacts root was sized by the report but skipped by // apply, claiming artifact bytes as removed while leaving every file in place. // A symlinked root is followed consistently, so the artifacts really go. func TestRetentionStagesThroughSymlinkedArtifactsRoot(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) id := f.run(t, "old", 72*time.Hour) f.run(t, "keep", time.Hour) real := filepath.Join(t.TempDir(), "real-artifacts") if err := os.MkdirAll(filepath.Join(real, fmt.Sprint(id), "1"), 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(real, fmt.Sprint(id), "1", "a.bin"), make([]byte, 4096), 0600); err != nil { t.Fatal(err) } link := filepath.Join(t.TempDir(), "artifacts") if err := os.Symlink(real, link); err != nil { t.Fatal(err) } report, err := f.store.RetentionReport(ctx, RetentionPolicy{Keep: 1}, link, f.now) if err != nil { t.Fatal(err) } if report.ArtifactBytes != 4096 { t.Fatalf("report must size artifacts through the symlinked root: %+v", report) } result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, link, f.now) if err != nil { t.Fatal(err) } if result.RemovedRunCount != 1 || result.RemovedArtifactBytes != 4096 { t.Fatalf("a symlinked root must still remove and report the artifacts: %+v", result) } if result.SpaceReclaimedBytes < result.RemovedArtifactBytes { t.Fatalf("reclaimed space must include the removed artifacts: %+v", result) } if _, err := os.Stat(filepath.Join(real, fmt.Sprint(id))); !os.IsNotExist(err) { t.Fatalf("artifacts behind a symlinked root must actually be removed: %v", err) } entries, err := os.ReadDir(real) if err != nil { t.Fatal(err) } for _, entry := range entries { if strings.HasPrefix(entry.Name(), ".retention-staging-") { t.Fatalf("staging directory left in the symlink target: %s", entry.Name()) } } } // TestRetentionReclaimFailureAfterRemovalIsHonest covers a reclaim failure that // happens after the rows and artifacts are already removed: the removed // artifact bytes must still be reported, the database pages must not be // claimed, and the total must not be zero. A concurrent read snapshot blocks // the final WAL checkpoint while leaving the write transaction and VACUUM free // to run; both checkpoint and VACUUM failures route through vacuumAndCheckpoint // and the same honest reporting path. func TestRetentionReclaimFailureAfterRemovalIsHonest(t *testing.T) { ctx := context.Background() f := newRetentionFixture(t) old := f.run(t, "old", 72*time.Hour) f.run(t, "keep", time.Hour) f.artifact(t, old, 1, "a.bin", 4096) reader, err := sql.Open("sqlite", f.store.path) if err != nil { t.Fatal(err) } defer reader.Close() conn, err := reader.Conn(ctx) if err != nil { t.Fatal(err) } defer conn.Close() if _, err := conn.ExecContext(ctx, "BEGIN"); err != nil { t.Fatal(err) } var n int if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM pipeline_runs").Scan(&n); err != nil { t.Fatal(err) } if _, err := f.store.db.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil { t.Fatal(err) } result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) if err == nil { t.Fatalf("apply must report the reclaim failure: %+v", result) } if result == nil || result.VacuumError == "" || result.Vacuumed { t.Fatalf("reclaim failure must set vacuum_error and leave vacuumed false: %+v", result) } if result.RemovedRunCount != 1 || result.RemovedArtifactBytes != 4096 || !result.Applied { t.Fatalf("removed artifacts must still be reported: %+v", result) } if result.DatabaseBytesReclaimed != 0 { t.Fatalf("a failed reclaim must not claim database pages: %+v", result) } if result.SpaceReclaimedBytes != result.RemovedArtifactBytes { t.Fatalf("space reclaimed must be the removed artifacts and no database pages: %+v", result) } if n := f.count(t, "pipeline_runs"); n != 1 { t.Fatalf("the removed run must be gone: %d", n) } if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) { t.Fatalf("the removed run artifacts must be gone: %v", err) } if _, err := conn.ExecContext(ctx, "COMMIT"); err != nil { t.Fatal(err) } }