diff --git a/docs/cli.md b/docs/cli.md index e2304784..7f8ab4ef 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -678,39 +678,48 @@ rename), deletes the rows for those runs in one immediate write transaction `step_verdicts`, `run_session_owners`, `run_steps`, `run_retries`, then `pipeline_runs`, batched at 500 ids), commits, and removes the staging directory. A symlink at `/` is removed as a link without touching its target; a -regular file at `/` is refused. All regular files under a removed run -directory are counted, including files beyond the server's 200-file/8-MiB/64-MiB -inventory caps, which bound the recorded inventory, not disk ownership. +regular file at `/` is refused. A symlinked artifacts root is followed +consistently: the report sizes it and apply stages through the same directory, so +the artifacts really go. All regular files under a removed run directory are +counted, including files beyond the server's 200-file/8-MiB/64-MiB inventory +caps, which bound the recorded inventory, not disk ownership. The database is then compacted with one `VACUUM` and one `PRAGMA wal_checkpoint(TRUNCATE)`, retried at most three short times on `SQLITE_BUSY`. `database_bytes_reclaimed` and `space_reclaimed_bytes` are measured, not estimated. If `VACUUM` fails the command returns an error, sets -`vacuum_error` and claims no reclaimed database pages; a later `apply` with -nothing left to remove can retry the `VACUUM`. If staging cleanup fails the -report names the staging path and remaining bytes/files and does not count them -as reclaimed. +`vacuum_error` and claims no reclaimed database pages while still reporting the +artifact bytes it did remove; a later `apply` with nothing left to remove can +retry the `VACUUM`. If staging cleanup fails the report still gives the removed +run count and totals, names the staging path and the remaining bytes/files, and +does not count the leftovers as reclaimed. ### Measured example (this machine) Reported before, applied, reported after, against the live database while its -server kept running (server PID unchanged): +server kept running (server PID 99492 unchanged throughout; the server was also +creating new runs concurrently, which is why the artifact `du` moved +independently): ```text -report --keep 200: total 279 · would remove 79 · protected 0 · retained 200 - artifacts 537 files · 499046205 bytes (476 MiB) - database 41426944 bytes · free 4096 · reclaimable 499050301 -apply --keep 200: removed 79 runs · 537 files · 499046205 bytes - vacuumed · database 41426944 -> 26898432 bytes - database_bytes_reclaimed 14528512 · space_reclaimed_bytes 513574717 +report --keep 200: total 207 · would remove 7 · protected 0 · retained 200 + artifacts 418 files · 279668627 bytes (266.7 MiB) + database 27897856 bytes · free 0 · reclaimable 279668627 +apply --keep 200: removed 7 runs · 418 files · 266.7 MiB + vacuumed · database reclaimed 1.5 MiB · now 25.1 MiB · free 0 + total space reclaimed 268.2 MiB report --keep 200: total 200 · would remove 0 · retained 200 - pipelines 18 · pipeline_versions 39 (unchanged) + artifacts 0 files · 0 bytes + database 26353664 bytes · free 0 + pipelines 19 · pipeline_versions 45 (unchanged) ``` -Independent observation: `du -sk` of the artifacts root fell from 3213324 to -2724872 KiB, the database file from 41426944 to 26898432 bytes, and the logs -stayed at 18404 KiB. A retained run's artifact still served (HTTP 200) while a -removed run's URL returned 404. The measured numbers above come from that +Independent observation: `du -sk` of the artifacts root fell from 2815316 to +2572456 KiB (the running server added artifacts during the same window), the +database file from 27897856 to 26353664 bytes, the WAL was truncated to zero and +the logs stayed at 18404 KiB. A retained run's artifact still served (HTTP 200, +`/artifacts/100/1/00_run94_artifacts.png`) while a removed run's URL returned 404 +(`/artifacts/83/1/...`). The measured numbers above come from that before/apply/after output, not an estimate. ## Verification scope diff --git a/internal/pipeline/retention.go b/internal/pipeline/retention.go index a21e0e65..26a0c533 100644 --- a/internal/pipeline/retention.go +++ b/internal/pipeline/retention.go @@ -439,6 +439,10 @@ func (s *Store) ApplyRetention(ctx context.Context, policy RetentionPolicy, arti beforeBytes := report.DatabaseBytes if err := s.vacuumAndCheckpoint(ctx); err != nil { result.VacuumError = err.Error() + // The run history and artifacts were already removed by + // removeRetentionRuns; report only the artifact space actually + // reclaimed and no database pages. + result.SpaceReclaimedBytes = result.RemovedArtifactBytes return result, fmt.Errorf("reclaim database space: %w", err) } result.Vacuumed = true @@ -509,8 +513,11 @@ func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy, // Stage only inside an artifacts root that already exists: creating the // root here would leave an empty artifacts directory behind after an // apply that had no artifacts to remove, and with no root there is no - // / directory to stage anyway. - if info, err := os.Lstat(artifactsRoot); err != nil { + // / directory to stage anyway. os.Stat (not Lstat) follows a + // symlinked root so staging happens through the same directory the + // report sized; skipping a symlinked root would claim its bytes as + // removed while leaving every artifact in place. + if info, err := os.Stat(artifactsRoot); err != nil { if !os.IsNotExist(err) { return fmt.Errorf("inspect artifacts root %s: %w", artifactsRoot, err) } @@ -573,15 +580,11 @@ func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy, } committed = true - if staging != "" { - if err := os.RemoveAll(staging); err != nil { - remainingFiles, remainingBytes, _ := retentionWalkSize(staging) - result.StagingPath = staging - result.StagingRemainingFiles = remainingFiles - result.StagingRemainingBytes = remainingBytes - return fmt.Errorf("run history removed but staged artifact cleanup failed at %s (%d files, %d bytes remain and are not counted as reclaimed): %w", staging, remainingFiles, remainingBytes, err) - } - } + // The rows are committed: these runs are gone whatever happens to the + // staged files, so record exactly what was removed before any cleanup + // error can return, and never let a cleanup failure be reported as zero + // runs removed or zero reclaimed space. SpaceReclaimedBytes is set here so + // a later VACUUM failure still reports the artifacts actually removed. result.RemovedRuns = append(result.RemovedRuns, ids...) sort.Slice(result.RemovedRuns, func(i, j int) bool { return result.RemovedRuns[i] < result.RemovedRuns[j] }) result.RemovedRunCount = len(ids) @@ -589,6 +592,25 @@ func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy, result.RemovedArtifactFiles += run.Artifacts.Files result.RemovedArtifactBytes += run.Artifacts.Bytes } + result.SpaceReclaimedBytes = result.RemovedArtifactBytes + result.Applied = result.RemovedRunCount > 0 + + if staging != "" { + if err := os.RemoveAll(staging); err != nil { + remainingFiles, remainingBytes, _ := retentionWalkSize(staging) + result.StagingPath = staging + result.StagingRemainingFiles = remainingFiles + result.StagingRemainingBytes = remainingBytes + // Files still in staging were staged out of the run tree but not + // deleted, so they are not reclaimed space. + if reclaimed := result.RemovedArtifactBytes - remainingBytes; reclaimed > 0 { + result.SpaceReclaimedBytes = reclaimed + } else { + result.SpaceReclaimedBytes = 0 + } + return fmt.Errorf("run history removed but staged artifact cleanup failed at %s (%d files, %d bytes remain and are not counted as reclaimed): %w", staging, remainingFiles, remainingBytes, err) + } + } return nil } diff --git a/internal/pipeline/retention_test.go b/internal/pipeline/retention_test.go index f6b02290..f20f48f7 100644 --- a/internal/pipeline/retention_test.go +++ b/internal/pipeline/retention_test.go @@ -631,3 +631,182 @@ func TestRetentionAbortsOnProtectionRace(t *testing.T) { t.Fatalf("aborted apply touched artifacts: %v", err) } } + +// TestRetentionReportsRemovedRunsWhenStagingCleanupFails covers the post-commit +// inspection gap where a successful row/artifact removal was reported as zero +// removed when the staging directory could not be deleted: the actual removed +// run(s) and artifact totals must still be reported, and the leftover staging +// bytes must not be counted as reclaimed. +func TestRetentionReportsRemovedRunsWhenStagingCleanupFails(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + old := f.run(t, "old", 72*time.Hour) + f.run(t, "keep", time.Hour) + // A run directory whose nested subdirectory cannot be emptied: os.RemoveAll + // fails on it, so the post-commit staging cleanup returns an error. + dir := filepath.Join(f.root, fmt.Sprint(old), "1") + locked := filepath.Join(dir, "locked") + if err := os.MkdirAll(locked, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "a.bin"), make([]byte, 128), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(locked, "inside.bin"), make([]byte, 64), 0600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0500); err != nil { + t.Fatal(err) + } + // Restore write permission before t.TempDir's cleanup removes the tree. + t.Cleanup(func() { + _ = filepath.WalkDir(f.root, func(path string, entry os.DirEntry, err error) error { + if err == nil && entry.IsDir() { + _ = os.Chmod(path, 0700) + } + return nil + }) + }) + + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) + if err == nil { + t.Fatal("apply must report the staging cleanup failure") + } + if result == nil { + t.Fatal("a cleanup failure must still report what was removed") + } + if result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{old}) { + t.Fatalf("rows were removed but not reported: %+v", result) + } + if !result.Applied { + t.Fatalf("run history was removed, so applied must be true: %+v", result) + } + if result.RemovedArtifactBytes != 192 || result.RemovedArtifactFiles != 2 { + t.Fatalf("removed artifact totals wrong: %+v", result) + } + if result.StagingPath == "" || result.StagingRemainingBytes != 64 || result.StagingRemainingFiles != 1 { + t.Fatalf("staging leftovers not reported: %+v", result) + } + if result.SpaceReclaimedBytes != result.RemovedArtifactBytes-result.StagingRemainingBytes { + t.Fatalf("reclaimed space must exclude staging leftovers: %+v", result) + } + if n := f.count(t, "pipeline_runs"); n != 1 { + t.Fatalf("the run rows must be gone: %d", n) + } + if _, err := os.Lstat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) { + t.Fatalf("the run tree must have been staged out of the artifacts root: %v", err) + } +} + +// TestRetentionStagesThroughSymlinkedArtifactsRoot covers the inspection gap +// where a symlinked artifacts root was sized by the report but skipped by +// apply, claiming artifact bytes as removed while leaving every file in place. +// A symlinked root is followed consistently, so the artifacts really go. +func TestRetentionStagesThroughSymlinkedArtifactsRoot(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + id := f.run(t, "old", 72*time.Hour) + f.run(t, "keep", time.Hour) + real := filepath.Join(t.TempDir(), "real-artifacts") + if err := os.MkdirAll(filepath.Join(real, fmt.Sprint(id), "1"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(real, fmt.Sprint(id), "1", "a.bin"), make([]byte, 4096), 0600); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "artifacts") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + report, err := f.store.RetentionReport(ctx, RetentionPolicy{Keep: 1}, link, f.now) + if err != nil { + t.Fatal(err) + } + if report.ArtifactBytes != 4096 { + t.Fatalf("report must size artifacts through the symlinked root: %+v", report) + } + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, link, f.now) + if err != nil { + t.Fatal(err) + } + if result.RemovedRunCount != 1 || result.RemovedArtifactBytes != 4096 { + t.Fatalf("a symlinked root must still remove and report the artifacts: %+v", result) + } + if result.SpaceReclaimedBytes < result.RemovedArtifactBytes { + t.Fatalf("reclaimed space must include the removed artifacts: %+v", result) + } + if _, err := os.Stat(filepath.Join(real, fmt.Sprint(id))); !os.IsNotExist(err) { + t.Fatalf("artifacts behind a symlinked root must actually be removed: %v", err) + } + entries, err := os.ReadDir(real) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".retention-staging-") { + t.Fatalf("staging directory left in the symlink target: %s", entry.Name()) + } + } +} + +// TestRetentionReclaimFailureAfterRemovalIsHonest covers a reclaim failure that +// happens after the rows and artifacts are already removed: the removed +// artifact bytes must still be reported, the database pages must not be +// claimed, and the total must not be zero. A concurrent read snapshot blocks +// the final WAL checkpoint while leaving the write transaction and VACUUM free +// to run; both checkpoint and VACUUM failures route through vacuumAndCheckpoint +// and the same honest reporting path. +func TestRetentionReclaimFailureAfterRemovalIsHonest(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + old := f.run(t, "old", 72*time.Hour) + f.run(t, "keep", time.Hour) + f.artifact(t, old, 1, "a.bin", 4096) + + reader, err := sql.Open("sqlite", f.store.path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + conn, err := reader.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, "BEGIN"); err != nil { + t.Fatal(err) + } + var n int + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM pipeline_runs").Scan(&n); err != nil { + t.Fatal(err) + } + if _, err := f.store.db.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil { + t.Fatal(err) + } + + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) + if err == nil { + t.Fatalf("apply must report the reclaim failure: %+v", result) + } + if result == nil || result.VacuumError == "" || result.Vacuumed { + t.Fatalf("reclaim failure must set vacuum_error and leave vacuumed false: %+v", result) + } + if result.RemovedRunCount != 1 || result.RemovedArtifactBytes != 4096 || !result.Applied { + t.Fatalf("removed artifacts must still be reported: %+v", result) + } + if result.DatabaseBytesReclaimed != 0 { + t.Fatalf("a failed reclaim must not claim database pages: %+v", result) + } + if result.SpaceReclaimedBytes != result.RemovedArtifactBytes { + t.Fatalf("space reclaimed must be the removed artifacts and no database pages: %+v", result) + } + if n := f.count(t, "pipeline_runs"); n != 1 { + t.Fatalf("the removed run must be gone: %d", n) + } + if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) { + t.Fatalf("the removed run artifacts must be gone: %v", err) + } + if _, err := conn.ExecContext(ctx, "COMMIT"); err != nil { + t.Fatal(err) + } +}