diff --git a/README.md b/README.md index 63b82c70..2c8523bc 100644 --- a/README.md +++ b/README.md @@ -328,9 +328,16 @@ pipelines serve # foreground server; Ctrl+C stops it pipelines ping pipelines version --json # local client identity (offline) pipelines ping --json # running server identity +pipelines retention report --keep 200 # show what run-history retention would remove +pipelines retention apply --keep 200 --yes # remove it (artifacts go with their run) pipelines help edit ``` +Run-history retention is explicit and never automatic: nothing prunes runs or +their artifacts until you run `pipelines retention apply`, protected runs (active, +waiting on a person, or referenced by a retained retry) are kept, and pipeline +definitions and versions are never touched. + CLI `new`/`edit` open the configured harness (**OpenCode** or **Pi**, native host by default; a profile is Docker-linked only when a checkout is explicitly linked) directly with the Pipelines skill. The installed skill asks diff --git a/docs/cli.md b/docs/cli.md index 8010af8d..7f8ab4ef 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -269,6 +269,7 @@ Submitting activates included schedules, so review them before publishing. | Connection and phone settings | `connection --json`, `ping --json` | | Server/ntfy status popup | `status --json` | | Usage statistics/charts | `stats --json` | +| Run-history retention | `retention report --keep N --json`, `retention apply --keep N --yes` | | Legacy operational diagnostics | managed `logs`, `doctor` | The TUI does not spawn CLI processes for each request. Both front ends share @@ -607,6 +608,120 @@ for panning and `c` to center. Secondary progress/input/evidence/execution secti are collapsed initially. Small inactive graph previews collapse instead of showing severed boxes. See [UI/UX review](ui-ux-review.md) for screen-by-screen feedback. +## Run retention + +Nothing prunes run history automatically. `pipelines retention report` and +`pipelines retention apply` are the only commands that remove run history or run +artifacts, they are local and operator-controlled, and they never run as a side +effect of any other command, server startup, `newRunner`, the scheduler, +`list_runs`, `stats`, `runs` or `inspect`. Pipeline definitions and their +immutable versions are never touched: retention is about run history and the +artifacts runs produced. + +```sh +# Read-only: show exactly what a policy would remove. +pipelines retention report --db ~/.local/share/pipelines/data/pipelines.db --keep 200 --json + +# Explicit removal of that same set (artifacts go with their run). +pipelines retention apply --db ~/.local/share/pipelines/data/pipelines.db --keep 200 --yes --json +``` + +Both commands accept `--older-than DURATION`, `--keep N`, `--db FILE`, +`--artifacts DIR` and `--json`; `apply` also accepts `--yes`. At least one policy +flag is required; an unknown subcommand, a negative count and a non-positive age +are refused. `--db` must name an existing database and is never created. With only +`--db`, the artifacts root defaults to `artifacts/` beside the database, matching +`cmd/pipelines-server/main.go`; `--artifacts` overrides it. The commands are +dispatched before server URL resolution, so they work without a running server +(for example `--url not-a-server`) and never require, restart or redeploy the +deployed server. + +### Policy + +- `--older-than` selects runs created **strictly before** `now - DURATION`. The + age is a rolling UTC duration, not a calendar-day boundary. It accepts Go + durations plus a count of days or weeks (`720h`, `30d`, `12w`). +- `--keep N` retains the newest `N` runs by run id and selects ranks `N+1` and + beyond. `--keep 0` selects every eligible run; a count larger than the history + selects none. +- Both together are the conservative **intersection**: a run is selected only + when it is both older than the age bound and outside the newest `N`. + +### Protected runs + +A policy-selected run is nonetheless never removed when it: + +- is `queued` or `running`; +- is `waiting` on a person (a `waiting` run or a `waiting` step); +- has any step with `agent_active=1` (cleanup still active); or +- is referenced by a run that will remain through retry provenance + (`run_retries.original_run_id`), propagated transitively along a retry chain. + If both an original and its retry are selected, no retained link remains and + both may be removed. + +Protected runs are reported with their reason and their artifacts are never +counted as reclaimable. Free-text run ids in prompts are not treated as +dependencies; retry provenance is the only structured cross-run dependency the +store records. + +If a selected run becomes protected between the report and the delete — a new +retry reference, a waiting/active state or new agent activity — `apply` aborts +with the run ids and reasons, deletes no rows or artifacts, and un-stages +anything already renamed. A removal never half-completes. + +### What apply removes and reclaims + +`apply` stages only the would-remove run directories into one +`artifacts/.retention-staging--/` directory (a same-filesystem +rename), deletes the rows for those runs in one immediate write transaction +(`step_notifications`, `step_file_changes`, `step_control`, `step_artifacts`, +`step_verdicts`, `run_session_owners`, `run_steps`, `run_retries`, then +`pipeline_runs`, batched at 500 ids), commits, and removes the staging directory. +A symlink at `/` is removed as a link without touching its target; a +regular file at `/` is refused. A symlinked artifacts root is followed +consistently: the report sizes it and apply stages through the same directory, so +the artifacts really go. All regular files under a removed run directory are +counted, including files beyond the server's 200-file/8-MiB/64-MiB inventory +caps, which bound the recorded inventory, not disk ownership. + +The database is then compacted with one `VACUUM` and one +`PRAGMA wal_checkpoint(TRUNCATE)`, retried at most three short times on +`SQLITE_BUSY`. `database_bytes_reclaimed` and `space_reclaimed_bytes` are +measured, not estimated. If `VACUUM` fails the command returns an error, sets +`vacuum_error` and claims no reclaimed database pages while still reporting the +artifact bytes it did remove; a later `apply` with nothing left to remove can +retry the `VACUUM`. If staging cleanup fails the report still gives the removed +run count and totals, names the staging path and the remaining bytes/files, and +does not count the leftovers as reclaimed. + +### Measured example (this machine) + +Reported before, applied, reported after, against the live database while its +server kept running (server PID 99492 unchanged throughout; the server was also +creating new runs concurrently, which is why the artifact `du` moved +independently): + +```text +report --keep 200: total 207 · would remove 7 · protected 0 · retained 200 + artifacts 418 files · 279668627 bytes (266.7 MiB) + database 27897856 bytes · free 0 · reclaimable 279668627 +apply --keep 200: removed 7 runs · 418 files · 266.7 MiB + vacuumed · database reclaimed 1.5 MiB · now 25.1 MiB · free 0 + total space reclaimed 268.2 MiB +report --keep 200: total 200 · would remove 0 · retained 200 + artifacts 0 files · 0 bytes + database 26353664 bytes · free 0 + pipelines 19 · pipeline_versions 45 (unchanged) +``` + +Independent observation: `du -sk` of the artifacts root fell from 2815316 to +2572456 KiB (the running server added artifacts during the same window), the +database file from 27897856 to 26353664 bytes, the WAL was truncated to zero and +the logs stayed at 18404 KiB. A retained run's artifact still served (HTTP 200, +`/artifacts/100/1/00_run94_artifacts.png`) while a removed run's URL returned 404 +(`/artifacts/83/1/...`). The measured numbers above come from that +before/apply/after output, not an estimate. + ## Verification scope CLI integration tests exercise draft/validate/submit/history/conflict handling and diff --git a/docs/harnesses-and-control.md b/docs/harnesses-and-control.md index 278ef5d5..392e5e7c 100644 --- a/docs/harnesses-and-control.md +++ b/docs/harnesses-and-control.md @@ -75,6 +75,24 @@ For bounded review loops, add a fenced `limits` JSON object to the reviewer: On the third rejection the server terminates at failure regardless of model counters. Approval on that visit can still succeed. Limits never enter prompts. +A reviewer that must separate a blocking defect from advisory notes declares both +edges: `rejected -> implement` (kept, with the `limits` above) is reserved for a +must-fix finding that names a concrete location, evidence and remedy, and +`advisory -> ` carries non-blocking duplication, performance or style +notes. An advisory outcome continues the run and never buys another implementer +visit. An implementer that can tell a transient environment, tooling or device +problem from an acceptance criterion that genuinely cannot be met declares a +bounded self-edge: `blocked_retryable -> implementer`, while `blocked -> #failure` +stays terminal. `max_visits` caps every visit of that stage, including re-entries +from the reviewer's `rejected` edge, so an implementer that is also the rejection +target needs a cap at least as large as the reviewer's budget; the published +maintenance, quick and focused pipelines use +`{"max_visits":5,"on":["blocked_retryable"]}`. A smaller implementer cap stops the +loop on the implementer's pre-execution limit before the reviewer's own cap can +fire, which silently makes the reviewer's `rejected` control unreachable. The runner +already counts the visits and stops at the cap, so neither loop needs new server +code and no model counts visits. + Add a fenced `memory` map such as `{"task":"input.prompt","checks":"planner.checks"}` to select reviewer input without previous verdicts. Implementers can select diff --git a/docs/stage-artifacts.md b/docs/stage-artifacts.md index 67e96c43..f1dc8969 100644 --- a/docs/stage-artifacts.md +++ b/docs/stage-artifacts.md @@ -51,3 +51,14 @@ runs/steps are refused. Responses carry the correct `Content-Type` and return `206 Partial Content` with the exact slice (or `416` when unsatisfiable), so images and video can stream. `http.ServeContent` streams from the opened file, so a request never loads the whole file into memory. + +## Removal with run history + +Run artifacts live under `//` and last as long as the run does. +`pipelines retention report`/`apply` (see [CLI and agent authoring](cli.md#run-retention)) +is the only thing that removes them, and it does so only for a run the operator +explicitly selected for removal. Removing a run removes its whole artifact +directory; protected and retained runs keep theirs, and their recorded artifact +URLs keep serving. A symlink at `/` is removed as a link without +touching its target, and a regular file at that path is refused rather than +deleted. Nothing removes artifacts automatically. diff --git a/internal/author/skill/references/control.md b/internal/author/skill/references/control.md index 73cb1b9b..24816fe8 100644 --- a/internal/author/skill/references/control.md +++ b/internal/author/skill/references/control.md @@ -16,6 +16,7 @@ does not guarantee independent review: explicitly select its input fields. ```edges approved -> #success +advisory -> #success rejected -> implement error -> #failure ``` @@ -24,6 +25,20 @@ error -> #failure These blocks accompany a normal review stage's prompt/output/tools/configs. Pipelines counts actual visits, ignoring model-generated counters. On visit three, `approved` can succeed; `rejected` retains its verdict but routes to `#failure`. +`rejected` is for a must-fix finding with a concrete location, evidence and remedy; +`advisory` carries non-blocking notes and continues the run without another +implementer visit, so an advisory-only review never buys a full re-implementation. +An implementer may likewise declare `blocked_retryable -> implementer` for a +transient environment, tooling or device problem a fresh attempt could survive, +keeping `blocked -> #failure` for an acceptance criterion that genuinely cannot be +met. `max_visits` caps every visit of that stage, including re-entries from the +reviewer's `rejected` edge, so an implementer that is also the rejection target must +use a cap at least as large as the reviewer's budget; the published pipelines use +`{"max_visits":5,"on":["blocked_retryable"]}`. A lower implementer cap stops the loop +before the reviewer's own cap fires, making the reviewer's `rejected` control +unreachable. Both edges need only the existing visit counting and cap; neither the +reviewer nor the implementer counts visits, and no stage invents edits or checks to +appear busy. Visit count, limit decision and original target are retained in `inspect --json` and run metadata. `max_visits` accepts 1–5. Every `on` outcome must declare a nonterminal edge. Without `on`, entering beyond the cap fails before execution. diff --git a/internal/author/skill/references/operations.md b/internal/author/skill/references/operations.md index f01f5ff2..fad5e9af 100644 --- a/internal/author/skill/references/operations.md +++ b/internal/author/skill/references/operations.md @@ -224,6 +224,22 @@ traversal, absolute paths, symlink escapes and unknown runs/steps. - Do not invent artifact files or claim a stage produced one without the recorded evidence; the server observes the directory after the stage. +### Operator-only retention + +Run history and its artifacts are never pruned automatically. The operator-only +`pipelines retention report` (strictly read-only) and `pipelines retention apply` +(removal after explicit approval) are the only commands that remove them; no +server startup, scheduler, `list_runs`, `stats`, `runs` or `inspect` path calls +retention, and there is no background goroutine, timer or startup hook. A policy +selects by rolling UTC age (`--older-than`), newest-N count (`--keep`), or their +intersection. Runs that are queued/running, waiting on a person, running a cleanup +agent, or referenced by a retained run's retry provenance are protected and +reported with a reason; their artifacts are kept. Removing a run removes its +`/` artifact directory and reclaims database space with one `VACUUM` +and one WAL checkpoint, reporting measured bytes. Pipeline definitions and their +immutable versions are never touched. The commands are local, require an existing +`--db`, and need no running server. + ## Brokered OpenCode chat The protocol declares the brokered `chat_*` surface (`chat_sessions`, diff --git a/internal/cli/app.go b/internal/cli/app.go index ab8cc852..d8caf7be 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -40,6 +40,9 @@ type options struct { output string conflict string kind string + db, artifacts, olderThan string + keep int + keepSet bool all bool json, source, noOpen, yes, allow, createOnly, force, wait bool version, expected, limit int @@ -151,6 +154,19 @@ func (a App) flags(command string) (*flag.FlagSet, *options) { f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait (0 waits until the run ends); independent of --timeout") case "cancel", "draft-delete": f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt") + case "retention": + f.StringVar(&o.db, "db", "", "Existing database path (default: standard data directory)") + f.StringVar(&o.artifacts, "artifacts", "", "Artifacts root (default: artifacts/ beside --db)") + f.StringVar(&o.olderThan, "older-than", "", "Select runs created before this rolling UTC duration ago (e.g. 720h, 30d, 12w)") + f.Func("keep", "Keep the newest N runs; older runs are selected (0 selects every eligible run)", func(value string) error { + n, err := strconv.Atoi(strings.TrimSpace(value)) + if err != nil { + return fmt.Errorf("--keep requires an integer count") + } + o.keep, o.keepSet = n, true + return nil + }) + f.BoolVar(&o.yes, "yes", false, "Approve removal without an interactive prompt (apply only)") case "decide": f.StringVar(&o.option, "option", "", "Option key to choose among the waiting stage's edges (required)") f.StringVar(&o.feedback, "feedback", "", "Optional free-text feedback recorded with the decision") @@ -389,6 +405,9 @@ func (a App) Run(ctx context.Context, args []string) error { _, err := fmt.Fprintln(a.Out, "Deleted draft:", pos[0]) return err } + if command == "retention" { + return a.retention(ctx, o, pos) + } if command != "drafts" && os.Getenv("PIPELINES_URL") == "" { explicit := false f.Visit(func(value *flag.Flag) { diff --git a/internal/cli/commands.go b/internal/cli/commands.go index be4603c3..aec2945f 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -31,6 +31,7 @@ var usage = map[string]string{ "status": "status [--json]", "stats": "stats [--json]", "rpc": "rpc ", + "retention": "retention report|apply [--older-than DURATION] [--keep N] [--db FILE] [--artifacts DIR] [--json] [--yes]", "tui": "tui", } @@ -51,6 +52,7 @@ var notes = map[string]string{ "decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.", "debug": "Open a fresh configured harness conversation to investigate a run using inspect/show/runs CLI queries. --harness opencode|pi|nvim chooses another configured harness or read-only nvim packet. Does not resume implementation or retry. --json returns the prompt without launching anything.", "rpc": "Accepts a JSON protocol envelope; '-' reads stdin. Every TUI server operation is available here too. Mutating commands are not dry runs. The omni-search command is `search` (fields query, optional kind and limit) and answers with ranked `search_results`.", + "retention": "Local, explicit run-history retention; never automatic and never a side effect of another command. `report` is strictly read-only and prints what a policy would remove; `apply` removes it after approval (--yes for scripted use). Policy: --older-than DURATION selects runs created before now minus that rolling UTC duration (Go durations plus d/w; not a calendar boundary), --keep N keeps the newest N runs by run id and selects older ranks (--keep 0 keeps none). Both together are the conservative intersection; at least one is required. Protected and never removed: queued/running runs, runs waiting on a person (waiting run or step), runs with an active cleanup agent, and any run referenced by a retained run's retry provenance (transitively). A run that becomes protected before deletion aborts the whole apply. Artifacts at / are removed with their run and kept with protected/retained runs; the database is VACUUMed and its WAL checkpointed, with measured reclaimed bytes. Definitions and versions are immutable and never touched. --db must name an existing database (default: the standard data directory); --artifacts defaults to artifacts/ beside --db. Works without a running server.", "search": "One read-only query over pipelines, versions, runs, steps, field text and artifacts. QUERY is free text, a bare status word (queued, running, succeeded, failed, canceled, interrupted) selecting those runs, or a lone date expression (today, yesterday, this week [Monday 00:00 local], or YYYY-MM-DD) selecting the runs created in that window. --type is the only narrowing mechanism and takes one kind: pipeline, version, run, step, field or artifact; an unknown kind is an error. Ranking: an exact name match (3) beats a name substring (2) beats a body-text match (1); equal tiers break by recency, then a stable kind/target tie-break, and every hit reports score, matched_field and recency so a client can re-rank. Bounds: the result cap defaults to 50 and clamps to 200 (0 = default); every kind scans at most 2000 rows and field-text scanning is additionally capped at 2000 steps / 1 MiB; Truncated plus scanned_steps, scanned_bytes and scan_bound_bytes report when a bound applied. Targets are stable: pipeline:, pipeline::v, run:, run::, run:::, or an artifact URL.", } diff --git a/internal/cli/help.go b/internal/cli/help.go index 98bea72e..7f7f6f7c 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -39,6 +39,10 @@ EXECUTE version Identify this CLI build without connecting (--version) rpc Send any protocol command (automation escape hatch) +MAINTAIN + retention report Show what a run-history policy would remove (read-only) + retention apply Remove it after approval; artifacts go with their run + SERVER / INTERFACE harness [opencode|pi] Launch the configured default harness in this directory harness setup opencode|pi Login/select separate harness and new-pipeline defaults @@ -71,6 +75,10 @@ Examples: pipelines run "bug fix" "Fix the layout" --json pipelines run "bug fix" "Fix the layout" --model openai/gpt-5 --stage-model review=openai/o4 --json +Run retention (local, explicit, never automatic): + pipelines retention report --db ~/.local/share/pipelines/data/pipelines.db --keep 200 --json + pipelines retention apply --db ~/.local/share/pipelines/data/pipelines.db --keep 200 --yes + Launch-time model overrides: 'pipelines run --model P/M' covers every model stage; '--stage-model STAGE=P/M' (repeatable; a repeated key is last-wins) overrides one stage and wins over --model, which wins over the model pinned in the pipeline diff --git a/internal/cli/retention.go b/internal/cli/retention.go new file mode 100644 index 00000000..eef2ad67 --- /dev/null +++ b/internal/cli/retention.go @@ -0,0 +1,218 @@ +package cli + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/paths" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/tui" +) + +// retention implements the local maintenance command: `report` describes what a +// policy would remove without touching anything, and `apply` removes it after +// explicit approval. It is dispatched before server URL resolution, so it works +// against a local database without a running server and never mutates the +// deployed server's configuration or definitions. +func (a App) retention(ctx context.Context, o *options, pos []string) error { + if len(pos) != 1 { + return fmt.Errorf("retention requires a subcommand: report or apply; use pipelines help retention") + } + sub := pos[0] + if sub != "report" && sub != "apply" { + return fmt.Errorf("unknown retention subcommand %q; use report or apply", sub) + } + policy, err := o.retentionPolicy() + if err != nil { + return err + } + db := strings.TrimSpace(o.db) + if db == "" { + db, err = paths.Database() + if err != nil { + return err + } + } + if _, err := os.Stat(db); err != nil { + return fmt.Errorf("retention database %s is unavailable: %w", db, err) + } + artifacts := strings.TrimSpace(o.artifacts) + if artifacts == "" { + artifacts = filepath.Join(filepath.Dir(db), "artifacts") + } + now := time.Now().UTC() + + if sub == "report" { + store, err := pipeline.OpenReadOnly(db) + if err != nil { + return err + } + defer store.Close() + report, err := store.RetentionReport(ctx, policy, artifacts, now) + if err != nil { + return err + } + return a.printRetentionReport(report, o.json) + } + + store, err := pipeline.OpenMaintenance(db) + if err != nil { + return err + } + defer store.Close() + if !o.yes { + preview, err := store.RetentionReport(ctx, policy, artifacts, now) + if err != nil { + return err + } + question := fmt.Sprintf("Permanently remove %d run(s) and %s of artifacts? Definitions and versions are never touched.", + len(preview.WouldRemove), tui.HumanBytes(preview.ArtifactBytes)) + if err := a.confirmAction(question, false); err != nil { + if printErr := a.printRetentionReport(preview, o.json); printErr != nil { + return printErr + } + return err + } + } + result, err := store.ApplyRetention(ctx, policy, artifacts, now) + if result != nil { + // Print the report even on a refusal, cleanup or vacuum error so the + // selection and reasons stay visible. + if printErr := a.printRetentionResult(result, o.json); printErr != nil { + return printErr + } + } + return err +} + +// retentionPolicy converts the parsed flags into a validated policy. An unset +// --keep is -1 (no count bound) so it is distinguishable from --keep 0. +func (o *options) retentionPolicy() (pipeline.RetentionPolicy, error) { + policy := pipeline.RetentionPolicy{Keep: -1} + if raw := strings.TrimSpace(o.olderThan); raw != "" { + duration, err := parseRetentionDuration(raw) + if err != nil { + return policy, err + } + if duration <= 0 { + return policy, fmt.Errorf("--older-than must be a positive duration") + } + policy.OlderThan = duration + } + if o.keepSet { + if o.keep < 0 { + return policy, fmt.Errorf("--keep must be nonnegative") + } + policy.Keep = o.keep + } + if err := policy.Validate(); err != nil { + return policy, err + } + return policy, nil +} + +// parseRetentionDuration accepts Go durations plus a count of days or weeks, +// so an operator can write 720h, 30d or 12w. The result is a rolling duration, +// never a calendar-day boundary. +func parseRetentionDuration(value string) (time.Duration, error) { + value = strings.TrimSpace(value) + if value == "" { + return 0, fmt.Errorf("duration is empty") + } + unit, number := time.Duration(0), value + switch { + case strings.HasSuffix(value, "d"): + unit, number = 24*time.Hour, strings.TrimSuffix(value, "d") + case strings.HasSuffix(value, "w"): + unit, number = 7*24*time.Hour, strings.TrimSuffix(value, "w") + } + if unit != 0 { + count, err := strconv.ParseFloat(number, 64) + if err != nil { + return 0, fmt.Errorf("invalid duration %q: use Go durations or a count of d/w", value) + } + return time.Duration(count * float64(unit)), nil + } + duration, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("invalid duration %q: %w", value, err) + } + return duration, nil +} + +func (a App) printRetentionReport(report *pipeline.RetentionReport, jsonOutput bool) error { + if jsonOutput { + return a.output(report) + } + writeRetention(a.Out, report) + fmt.Fprintln(a.Out, "\nNo changes made.") + return nil +} + +func (a App) printRetentionResult(result *pipeline.RetentionApplyResult, jsonOutput bool) error { + if jsonOutput { + return a.output(result) + } + writeRetention(a.Out, &result.RetentionReport) + fmt.Fprintf(a.Out, "\nRemoved %d run(s) · %d files · %s\n", result.RemovedRunCount, result.RemovedArtifactFiles, tui.HumanBytes(result.RemovedArtifactBytes)) + if result.Vacuumed { + fmt.Fprintf(a.Out, "Database reclaimed: %s · now %s · free %s\n", tui.HumanBytes(result.DatabaseBytesReclaimed), tui.HumanBytes(result.DatabaseBytesAfter), tui.HumanBytes(result.DatabaseFreeBytesAfter)) + } + if result.VacuumError != "" { + fmt.Fprintf(a.Out, "Database reclaim failed: %s\n", result.VacuumError) + } + fmt.Fprintf(a.Out, "Total space reclaimed: %s\n", tui.HumanBytes(result.SpaceReclaimedBytes)) + if result.StagingPath != "" { + fmt.Fprintf(a.Out, "Staging cleanup incomplete: %s · %d files · %s remain and are not reclaimed\n", result.StagingPath, result.StagingRemainingFiles, tui.HumanBytes(result.StagingRemainingBytes)) + } + return nil +} + +// writeRetention renders the shared report body used by both report and apply +// output, so the selection, sizes and reasons are presented identically. +func writeRetention(w interface{ Write([]byte) (int, error) }, report *pipeline.RetentionReport) { + fmt.Fprintf(w, "Retention · %s\n", report.Database) + fmt.Fprintf(w, "Artifacts root: %s\n", report.ArtifactsRoot) + fmt.Fprintf(w, "Policy: older-than %s · keep newest %s\n", retentionAge(report.Policy), retentionKeep(report.Policy)) + fmt.Fprintf(w, "Runs: %d total · %d would remove · %d protected · %d retained\n", report.TotalRuns, len(report.WouldRemove), len(report.Protected), report.RetainedRuns) + fmt.Fprintf(w, "Artifacts: %d files · %s\n", report.ArtifactFiles, tui.HumanBytes(report.ArtifactBytes)) + fmt.Fprintf(w, "Database: %s · free %s\n", tui.HumanBytes(report.DatabaseBytes), tui.HumanBytes(report.DatabaseFreeBytes)) + fmt.Fprintf(w, "Reclaimable: %s\n", tui.HumanBytes(report.ReclaimableBytes)) + if len(report.WouldRemove) > 0 { + fmt.Fprintln(w, "\nWould remove:") + for _, run := range report.WouldRemove { + fmt.Fprintf(w, " #%d %s %s %d files %s\n", run.ID, run.Status, run.CreatedAt, run.Artifacts.Files, tui.HumanBytes(run.Artifacts.Bytes)) + } + } + if len(report.Protected) > 0 { + fmt.Fprintln(w, "\nProtected:") + for _, run := range report.Protected { + fmt.Fprintf(w, " #%d %s\n", run.ID, run.Reason) + } + } + if report.ArtifactsPartial { + fmt.Fprintln(w, "\nArtifact sizes are partial; some directories could not be fully walked:") + for _, err := range report.ArtifactWalkErrors { + fmt.Fprintf(w, " %s\n", err) + } + } +} + +func retentionAge(policy pipeline.RetentionPolicy) string { + if policy.OlderThan <= 0 { + return "unset" + } + return policy.OlderThan.String() +} + +func retentionKeep(policy pipeline.RetentionPolicy) string { + if policy.Keep < 0 { + return "unset" + } + return strconv.Itoa(policy.Keep) +} diff --git a/internal/cli/retention_test.go b/internal/cli/retention_test.go new file mode 100644 index 00000000..e5182392 --- /dev/null +++ b/internal/cli/retention_test.go @@ -0,0 +1,179 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/pipeline" +) + +// seedRetentionStore creates a scratch database with a definition and the given +// number of runs, plus one artifact under the first run. It returns the database +// path and the directory that holds the default artifacts root. +func seedRetentionStore(t *testing.T, runs int) (db, dir string) { + t.Helper() + dir = t.TempDir() + db = filepath.Join(dir, "pipelines.db") + ctx := context.Background() + store, err := pipeline.Open(db) + if err != nil { + t.Fatal(err) + } + if _, err := store.Submit(ctx, "cli-retention", pipeline.Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + definition, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + for i := 0; i < runs; i++ { + id, _, err := store.CreateRun(ctx, definition, fmt.Sprintf("run-%d", i), "ws", false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + if err := store.FinishRun(ctx, id, "succeeded", "success", "", map[string]any{}); err != nil { + t.Fatal(err) + } + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + artifact := filepath.Join(dir, "artifacts", "1", "1", "report.bin") + if err := os.MkdirAll(filepath.Dir(artifact), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(artifact, []byte("artifact bytes"), 0600); err != nil { + t.Fatal(err) + } + return db, dir +} + +func runCount(t *testing.T, db string) int { + t.Helper() + store, err := pipeline.OpenReadOnly(db) + if err != nil { + t.Fatal(err) + } + defer store.Close() + runs, err := store.ListRuns(context.Background()) + if err != nil { + t.Fatal(err) + } + return len(runs) +} + +func newRetentionApp() (App, *bytes.Buffer) { + var out bytes.Buffer + return App{Out: &out, Err: &out, In: strings.NewReader("")}, &out +} + +func TestRetentionReportIsReadOnlyAndLocal(t *testing.T) { + ctx := context.Background() + db, dir := seedRetentionStore(t, 3) + app, out := newRetentionApp() + // --url not-a-server proves the command never dials a server. + if err := app.Run(ctx, []string{"retention", "report", "--db", db, "--keep", "1", "--json", "--url", "not-a-server"}); err != nil { + t.Fatal(err) + } + var report pipeline.RetentionReport + if err := json.Unmarshal(out.Bytes(), &report); err != nil { + t.Fatalf("report is not valid JSON: %v\n%s", err, out.String()) + } + if report.Applied { + t.Fatal("a report must never be applied") + } + if len(report.WouldRemove) != 2 || report.TotalRuns != 3 { + t.Fatalf("report selection wrong: %+v", report) + } + if report.ArtifactsRoot != filepath.Join(dir, "artifacts") { + t.Fatalf("default artifacts root = %q, want artifacts/ beside the db", report.ArtifactsRoot) + } + if report.ReclaimableBytes < int64(len("artifact bytes")) { + t.Fatalf("reclaimable bytes must include the run artifacts: %d", report.ReclaimableBytes) + } + if runCount(t, db) != 3 { + t.Fatal("report must not change the run history") + } + if _, err := os.Stat(filepath.Join(dir, "artifacts", "1", "1", "report.bin")); err != nil { + t.Fatalf("report must not touch artifacts: %v", err) + } +} + +func TestRetentionApplyRequiresApprovalAndRemovesRuns(t *testing.T) { + ctx := context.Background() + db, dir := seedRetentionStore(t, 3) + app, out := newRetentionApp() + // Without --yes and with no interactive input, apply refuses and changes nothing. + if err := app.Run(ctx, []string{"retention", "apply", "--db", db, "--keep", "1", "--url", "not-a-server"}); err == nil { + t.Fatal("apply must refuse without --yes or interactive approval") + } + if runCount(t, db) != 3 { + t.Fatal("refused apply changed the run history") + } + if _, err := os.Stat(filepath.Join(dir, "artifacts", "1", "1", "report.bin")); err != nil { + t.Fatalf("refused apply touched artifacts: %v", err) + } + + out.Reset() + if err := app.Run(ctx, []string{"retention", "apply", "--db", db, "--keep", "1", "--yes", "--json", "--url", "not-a-server"}); err != nil { + t.Fatal(err) + } + var result pipeline.RetentionApplyResult + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatalf("apply is not valid JSON: %v\n%s", err, out.String()) + } + if !result.Applied || result.RemovedRunCount != 2 || !result.Vacuumed { + t.Fatalf("apply did not remove and reclaim: %+v", result) + } + if runCount(t, db) != 1 { + t.Fatalf("apply removed the wrong number of runs: %d", runCount(t, db)) + } + if _, err := os.Stat(filepath.Join(dir, "artifacts", "1")); !os.IsNotExist(err) { + t.Fatalf("removed run's artifacts must be gone: %v", err) + } +} + +func TestRetentionValidationAndMissingDatabase(t *testing.T) { + ctx := context.Background() + db, dir := seedRetentionStore(t, 1) + app, _ := newRetentionApp() + for _, args := range [][]string{ + {"retention", "report", "--db", db, "--url", "not-a-server"}, // no policy + {"retention", "report", "--db", db, "--keep=-1", "--url", "not-a-server"}, // negative count + {"retention", "report", "--db", db, "--older-than", "0h", "--url", "not-a-server"}, + {"retention", "frobnicate", "--db", db, "--keep", "1", "--url", "not-a-server"}, // unknown subcommand + } { + if err := app.Run(ctx, args); err == nil { + t.Fatalf("%v was accepted", args) + } + } + missing := filepath.Join(dir, "missing.db") + if err := app.Run(ctx, []string{"retention", "report", "--db", missing, "--keep", "1", "--url", "not-a-server"}); err == nil { + t.Fatal("a missing database must be an error") + } + if _, err := os.Stat(missing); !os.IsNotExist(err) { + t.Fatal("retention must not create a database") + } +} + +func TestRetentionHelp(t *testing.T) { + app, out := newRetentionApp() + if err := app.Run(context.Background(), []string{"help", "retention"}); err != nil { + t.Fatal(err) + } + for _, want := range []string{"retention report|apply", "--older-than", "--keep", "--artifacts"} { + if !strings.Contains(out.String(), want) { + t.Fatalf("help retention missing %q:\n%s", want, out.String()) + } + } +} diff --git a/internal/pipeline/retention.go b/internal/pipeline/retention.go new file mode 100644 index 00000000..26a0c533 --- /dev/null +++ b/internal/pipeline/retention.go @@ -0,0 +1,760 @@ +package pipeline + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// RetentionPolicy selects run history an operator has asked to remove. At least +// one bound must be set: +// +// - OlderThan selects runs created strictly before now-OlderThan (a rolling +// UTC duration, never a calendar-day boundary). Zero means no age bound. +// - Keep retains the newest Keep runs by run id and selects every older rank +// (N+1 and beyond). -1 means no count bound; 0 is the explicit "keep none". +// +// When both are set the selection is their conservative intersection: a run is +// selected only when it is both older than the age bound and outside the newest +// Keep. Selecting a run is not the same as removing it: a protected run is +// never in the removal set. +type RetentionPolicy struct { + OlderThan time.Duration `json:"older_than"` + Keep int `json:"keep"` +} + +// Validate reports whether a policy is well-formed: at least one bound is set, +// the age is not negative and the count is not below the -1 "unbounded" sentinel. +func (p RetentionPolicy) Validate() error { + if p.OlderThan < 0 { + return fmt.Errorf("retention age must not be negative") + } + if p.Keep < -1 { + return fmt.Errorf("retention count must not be negative") + } + if p.OlderThan == 0 && p.Keep < 0 { + return fmt.Errorf("retention policy requires an age bound, a count bound, or both") + } + return nil +} + +// RetentionArtifacts is the stat-only size of one run's artifact tree. +type RetentionArtifacts struct { + Files int64 `json:"files"` + Bytes int64 `json:"bytes"` + Partial bool `json:"partial,omitempty"` +} + +// RetentionRun is one run selected by the policy for removal, with the measured +// size of the artifacts that would be removed with it. +type RetentionRun struct { + ID int64 `json:"id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + Artifacts RetentionArtifacts `json:"artifacts"` +} + +// ProtectedRun is a run the policy selected but that must not be removed, +// with the reason it is protected. +type ProtectedRun struct { + ID int64 `json:"id"` + Reason string `json:"reason"` +} + +// RetentionReport is the read-only description of what a policy would remove. +// WouldRemove and Protected partition the policy-selected runs; RetainedRuns +// counts the runs the policy never selected. +type RetentionReport struct { + Now string `json:"now"` + Policy RetentionPolicy `json:"policy"` + Database string `json:"database"` + ArtifactsRoot string `json:"artifacts_root"` + TotalRuns int `json:"total_runs"` + RetainedRuns int `json:"retained_runs"` + WouldRemove []RetentionRun `json:"would_remove"` + Protected []ProtectedRun `json:"protected"` + ArtifactFiles int64 `json:"artifact_files"` + ArtifactBytes int64 `json:"artifact_bytes"` + ArtifactsPartial bool `json:"artifacts_partial,omitempty"` + ArtifactWalkErrors []string `json:"artifact_walk_errors,omitempty"` + DatabaseBytes int64 `json:"database_bytes"` + WALBytes int64 `json:"wal_bytes"` + DatabaseFreeBytes int64 `json:"database_free_bytes"` + Pipelines int `json:"pipelines"` + PipelineVersions int `json:"pipeline_versions"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` + // Applied is always false for a report; an apply sets it true once it has + // actually removed run history. + Applied bool `json:"applied"` +} + +// RetentionApplyResult is the report plus what an apply actually removed and +// the measured database space it reclaimed. It embeds the pre-removal report so +// refusal and vacuum errors still carry the selection and reasons. +type RetentionApplyResult struct { + RetentionReport + RemovedRuns []int64 `json:"removed_runs"` + RemovedRunCount int `json:"removed_run_count"` + RemovedArtifactFiles int64 `json:"removed_artifact_files"` + RemovedArtifactBytes int64 `json:"removed_artifact_bytes"` + Vacuumed bool `json:"vacuumed"` + VacuumError string `json:"vacuum_error,omitempty"` + DatabaseBytesAfter int64 `json:"database_bytes_after"` + DatabaseFreeBytesAfter int64 `json:"database_free_bytes_after"` + WALBytesAfter int64 `json:"wal_bytes_after"` + DatabaseBytesReclaimed int64 `json:"database_bytes_reclaimed"` + SpaceReclaimedBytes int64 `json:"space_reclaimed_bytes"` + StagingPath string `json:"staging_path,omitempty"` + StagingRemainingFiles int64 `json:"staging_remaining_files,omitempty"` + StagingRemainingBytes int64 `json:"staging_remaining_bytes,omitempty"` +} + +// retentionQuerier is the read surface shared by *sql.DB, *sql.Tx and *sql.Conn, +// so the selection runs identically against the report connection and inside +// the apply write transaction without a second query projection. +type retentionQuerier interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// retentionRunInfo is one pipeline_runs row plus the step aggregates protection +// needs, read in the same statement so the report never issues a per-run query. +type retentionRunInfo struct { + id int64 + status string + createdAt time.Time + waitingSteps int + activeSteps int +} + +// protectionReasons lists why a run must not be removed for a reason intrinsic +// to itself: it is active, waiting on a person, or still running a cleanup agent. +func (r retentionRunInfo) protectionReasons() []string { + var reasons []string + switch r.status { + case protocol.RunStatusQueued: + reasons = append(reasons, "run is queued") + case protocol.RunStatusRunning: + reasons = append(reasons, "run is running") + case protocol.RunStatusWaiting: + reasons = append(reasons, "run is waiting on a person") + } + if r.waitingSteps > 0 { + reasons = append(reasons, "a step is waiting on a person") + } + if r.activeSteps > 0 { + reasons = append(reasons, "cleanup agent still active (agent_active=1)") + } + return reasons +} + +// retentionSelection is the raw run rows and retry edges the policy is applied +// to. It is deliberately independent of the removal decision so the same +// selection can be recomputed inside the apply transaction. +type retentionSelection struct { + runs []retentionRunInfo // ordered by run id DESC + edges [][2]int64 // [retry run id, original run id] +} + +// selectRetentionRuns reads every run's status, creation time and step +// protection aggregates in one statement (plus one retry-edge statement), so the +// report is a bounded number of round trips rather than one query per run. +func selectRetentionRuns(ctx context.Context, q retentionQuerier, policy RetentionPolicy, now time.Time) (retentionSelection, error) { + var sel retentionSelection + rows, err := q.QueryContext(ctx, `SELECT r.id,r.status,strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at), +COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.status=?),0), +COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0) `+ + runFrom+` ORDER BY r.id DESC`, protocol.StepStatusWaiting) + if err != nil { + return sel, err + } + defer rows.Close() + for rows.Next() { + var run retentionRunInfo + var created string + if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps); err != nil { + return sel, err + } + run.createdAt = parseRetentionTime(created) + sel.runs = append(sel.runs, run) + } + if err := rows.Err(); err != nil { + return sel, err + } + edges, err := q.QueryContext(ctx, `SELECT run_id,original_run_id FROM run_retries`) + if err != nil { + // A database predating retry provenance has no run_retries table; that is + // an empty edge set, not a report failure. + if !strings.Contains(err.Error(), "no such table") { + return sel, err + } + return sel, nil + } + defer edges.Close() + for edges.Next() { + var retry, original int64 + if err := edges.Scan(&retry, &original); err != nil { + return sel, err + } + sel.edges = append(sel.edges, [2]int64{retry, original}) + } + return sel, edges.Err() +} + +// parseRetentionTime reads the ISO timestamp the selection quotes, accepting the +// stored space-separated UTC form as a fallback. Values without a zone are UTC. +func parseRetentionTime(value string) time.Time { + for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05Z", "2006-01-02 15:04:05"} { + if parsed, err := time.Parse(layout, value); err == nil { + return parsed.UTC() + } + } + return time.Time{} +} + +// policySelects reports whether the run at the given newest-first rank is chosen +// by the policy. Rank 0 is the newest run. +func policySelects(policy RetentionPolicy, info retentionRunInfo, rank int, now time.Time) bool { + if policy.OlderThan > 0 && !info.createdAt.Before(now.Add(-policy.OlderThan)) { + return false + } + if policy.Keep >= 0 && rank < policy.Keep { + return false + } + return true +} + +// retentionDecision is the policy selection split into runs that would be +// removed and runs that are protected, with each protected run's reason. +type retentionDecision struct { + wouldRemove []retentionRunInfo + protected []ProtectedRun +} + +// decide applies the policy and then protection. Protection is intrinsic state +// (active/waiting/agent) plus retry provenance propagated to a fixpoint: a run +// that remains and references a selected run protects it; if both a retry and +// its original are selected, no retained link remains and both may be removed. +func (s retentionSelection) decide(policy RetentionPolicy, now time.Time) retentionDecision { + selected := map[int64]bool{} + for rank, run := range s.runs { + if policySelects(policy, run, rank, now) { + selected[run.id] = true + } + } + reasons := map[int64][]string{} + remaining := map[int64]bool{} + for _, run := range s.runs { + if !selected[run.id] { + remaining[run.id] = true + continue + } + if found := run.protectionReasons(); len(found) > 0 { + reasons[run.id] = append(reasons[run.id], found...) + remaining[run.id] = true + } + } + for changed := true; changed; { + changed = false + for _, edge := range s.edges { + retry, original := edge[0], edge[1] + if !remaining[retry] || !selected[original] { + continue + } + if _, done := reasons[original]; done { + continue + } + reasons[original] = append(reasons[original], fmt.Sprintf("referenced by retained retry run #%d", retry)) + remaining[original] = true + changed = true + } + } + decision := retentionDecision{} + for _, run := range s.runs { + if !selected[run.id] { + continue + } + if found, ok := reasons[run.id]; ok { + decision.protected = append(decision.protected, ProtectedRun{ID: run.id, Reason: strings.Join(found, "; ")}) + continue + } + decision.wouldRemove = append(decision.wouldRemove, run) + } + return decision +} + +// RetentionReport describes what a policy would remove without changing +// anything: no table write, VACUUM, checkpoint or temp table runs. It uses one +// run query, one retry-edge query, one definition/version count query and the +// page/freelist PRAGMAs plus file stats, so it is a bounded read-only pass. +func (s *Store) RetentionReport(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time) (*RetentionReport, error) { + if err := policy.Validate(); err != nil { + return nil, err + } + now = now.UTC() + selection, err := selectRetentionRuns(ctx, s.db, policy, now) + if err != nil { + return nil, err + } + decision := selection.decide(policy, now) + report := &RetentionReport{ + Now: now.Format(time.RFC3339), + Policy: policy, + Database: s.path, + ArtifactsRoot: artifactsRoot, + TotalRuns: len(selection.runs), + WouldRemove: []RetentionRun{}, + Protected: decision.protected, + } + if report.Protected == nil { + report.Protected = []ProtectedRun{} + } + partial := false + for _, run := range decision.wouldRemove { + files, bytes, errs := retentionArtifactsSize(artifactsRoot, run.id) + entry := RetentionRun{ + ID: run.id, + Status: run.status, + CreatedAt: run.createdAt.Format(time.RFC3339), + Artifacts: RetentionArtifacts{Files: files, Bytes: bytes, Partial: len(errs) > 0}, + } + if len(errs) > 0 { + partial = true + report.ArtifactWalkErrors = append(report.ArtifactWalkErrors, errs...) + } + report.ArtifactFiles += files + report.ArtifactBytes += bytes + report.WouldRemove = append(report.WouldRemove, entry) + } + report.ArtifactsPartial = partial + report.RetainedRuns = report.TotalRuns - len(report.WouldRemove) - len(report.Protected) + if report.DatabaseBytes, report.WALBytes, report.DatabaseFreeBytes, err = s.retentionDBSize(ctx); err != nil { + return nil, err + } + if err := s.db.QueryRowContext(ctx, `SELECT (SELECT COUNT(*) FROM pipelines),(SELECT COUNT(*) FROM pipeline_versions)`). + Scan(&report.Pipelines, &report.PipelineVersions); err != nil { + return nil, err + } + report.ReclaimableBytes = report.ArtifactBytes + report.DatabaseFreeBytes + return report, nil +} + +// retentionArtifactsSize stat-walks one run's artifact directory, counting every +// regular file including files the server's inventory caps did not record. It +// skips symlinks and non-regular entries, never reads file contents, and treats +// a missing directory as zero bytes. A symlink or file at the run path owns no +// tree under the root, so it is not followed. Walk errors are returned so the +// report can mark the size partial rather than fail. +func retentionArtifactsSize(root string, id int64) (int64, int64, []string) { + if root == "" { + return 0, 0, nil + } + dir := filepath.Join(root, strconv.FormatInt(id, 10)) + info, err := os.Lstat(dir) + if err != nil { + if os.IsNotExist(err) { + return 0, 0, nil + } + return 0, 0, []string{fmt.Sprintf("run %d: %v", id, err)} + } + // A symlink or file at the run path owns no tree under the root, so it is + // not walked; only a real run directory contributes artifact bytes. + if !info.IsDir() { + return 0, 0, nil + } + files, bytes, errs := retentionWalkSize(dir) + for i, walkErr := range errs { + errs[i] = fmt.Sprintf("run %d: %s", id, walkErr) + } + return files, bytes, errs +} + +// retentionWalkSize stat-walks a directory counting every regular file and its +// bytes without reading any file contents. It skips directories, symlinks and +// other non-regular entries, accumulates only what it could stat, and returns +// every walk error so a caller can report a partial size rather than fail. It is +// the one walk shared by artifact sizing and the staging-cleanup report. +func retentionWalkSize(dir string) (int64, int64, []string) { + var files, bytes int64 + var errs []string + walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + errs = append(errs, fmt.Sprintf("%s: %v", path, err)) + return nil + } + if entry.IsDir() || !entry.Type().IsRegular() { + return nil + } + stat, statErr := entry.Info() + if statErr != nil { + errs = append(errs, fmt.Sprintf("%s: %v", path, statErr)) + return nil + } + files++ + bytes += stat.Size() + return nil + }) + if walkErr != nil { + errs = append(errs, fmt.Sprintf("%s: %v", dir, walkErr)) + } + return files, bytes, errs +} + +// ApplyRetention removes exactly the runs the policy selected and is not +// protecting, together with their artifacts, then reclaims database space. It +// is the explicit operator command: nothing here runs automatically. A run that +// becomes protected between the report and the delete aborts the whole apply +// with the run ids and reasons, so a removal never half-completes. +func (s *Store) ApplyRetention(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time) (*RetentionApplyResult, error) { + if err := policy.Validate(); err != nil { + return nil, err + } + now = now.UTC() + report, err := s.RetentionReport(ctx, policy, artifactsRoot, now) + if err != nil { + return nil, err + } + result := &RetentionApplyResult{RetentionReport: *report, RemovedRuns: []int64{}} + ids := make([]int64, 0, len(report.WouldRemove)) + for _, run := range report.WouldRemove { + ids = append(ids, run.ID) + } + if len(ids) > 0 { + if err := s.removeRetentionRuns(ctx, policy, artifactsRoot, now, ids, result); err != nil { + return result, err + } + } + // Apply is the explicit reclaim command: even with nothing to remove it may + // compact pages freed earlier. VACUUM failure is reported, never hidden. + beforeBytes := report.DatabaseBytes + if err := s.vacuumAndCheckpoint(ctx); err != nil { + result.VacuumError = err.Error() + // The run history and artifacts were already removed by + // removeRetentionRuns; report only the artifact space actually + // reclaimed and no database pages. + result.SpaceReclaimedBytes = result.RemovedArtifactBytes + return result, fmt.Errorf("reclaim database space: %w", err) + } + result.Vacuumed = true + afterBytes, afterWAL, afterFree, err := s.retentionDBSize(ctx) + if err != nil { + return result, err + } + result.DatabaseBytesAfter = afterBytes + result.DatabaseFreeBytesAfter = afterFree + result.WALBytesAfter = afterWAL + if reclaimed := beforeBytes - afterBytes; reclaimed > 0 { + result.DatabaseBytesReclaimed = reclaimed + } + result.SpaceReclaimedBytes = result.RemovedArtifactBytes + result.DatabaseBytesReclaimed + result.Applied = result.RemovedRunCount > 0 + return result, nil +} + +// removeRetentionRuns performs the staged, single-transaction removal of the +// selected runs: re-check protection under an immediate write lock, rename the +// run artifact directories into one staging directory, delete the rows, commit, +// then delete the staged files. Every failure path restores what it can and +// leaves nothing half-removed. +func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time, ids []int64, result *RetentionApplyResult) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin retention transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + + // Re-check protection inside the write transaction: a run that gained a + // retry reference, a waiting state or agent activity since the report must + // abort the whole apply. + selection, err := selectRetentionRuns(ctx, tx, policy, now) + if err != nil { + return fmt.Errorf("re-check retention selection: %w", err) + } + decision := selection.decide(policy, now) + stillRemove := map[int64]bool{} + for _, run := range decision.wouldRemove { + stillRemove[run.id] = true + } + became := []string{} + for _, id := range ids { + if stillRemove[id] { + continue + } + reason := "run is no longer selected for removal" + for _, protected := range decision.protected { + if protected.ID == id { + reason = protected.Reason + } + } + became = append(became, fmt.Sprintf("run #%d: %s", id, reason)) + } + if len(became) > 0 { + return fmt.Errorf("retention aborted: selected run(s) became protected before deletion; nothing was removed: %s", strings.Join(became, "; ")) + } + + staging := "" + staged := []string{} + if artifactsRoot != "" { + // Stage only inside an artifacts root that already exists: creating the + // root here would leave an empty artifacts directory behind after an + // apply that had no artifacts to remove, and with no root there is no + // / directory to stage anyway. os.Stat (not Lstat) follows a + // symlinked root so staging happens through the same directory the + // report sized; skipping a symlinked root would claim its bytes as + // removed while leaving every artifact in place. + if info, err := os.Stat(artifactsRoot); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("inspect artifacts root %s: %w", artifactsRoot, err) + } + } else if info.IsDir() { + staging = filepath.Join(artifactsRoot, fmt.Sprintf(".retention-staging-%d-%d", now.Unix(), os.Getpid())) + if err := os.MkdirAll(staging, 0700); err != nil { + return fmt.Errorf("create retention staging directory: %w", err) + } + } + } + // unstage restores every renamed run directory and removes the staging + // directory, returning any restore/cleanup error so the rollback path cannot + // silently leave directories behind while only the original failure is named. + unstage := func() error { + var problems []string + for _, original := range staged { + if err := os.Rename(filepath.Join(staging, filepath.Base(original)), original); err != nil { + problems = append(problems, fmt.Sprintf("restore %s: %v", original, err)) + } + } + if staging != "" { + if err := os.RemoveAll(staging); err != nil { + problems = append(problems, fmt.Sprintf("remove staging directory %s: %v", staging, err)) + } + } + if len(problems) > 0 { + return fmt.Errorf("un-stage artifacts: %s", strings.Join(problems, "; ")) + } + return nil + } + for _, id := range ids { + if staging == "" { + break + } + original := filepath.Join(artifactsRoot, strconv.FormatInt(id, 10)) + info, lerr := os.Lstat(original) + if lerr != nil { + if os.IsNotExist(lerr) { + continue + } + return errors.Join(fmt.Errorf("inspect artifacts for run %d: %w", id, lerr), unstage()) + } + // Only the exact numeric run entry is staged. A regular file at that path + // is refused rather than deleted; a symlink is renamed (removing the link + // only), never its target. + if !info.IsDir() && info.Mode()&os.ModeSymlink == 0 { + return errors.Join(fmt.Errorf("refusing to remove run %d: %s is a regular file, not a run artifact directory", id, original), unstage()) + } + if err := os.Rename(original, filepath.Join(staging, strconv.FormatInt(id, 10))); err != nil { + return errors.Join(fmt.Errorf("stage artifacts for run %d: %w", id, err), unstage()) + } + staged = append(staged, original) + } + + if err := deleteRetentionRows(ctx, tx, ids); err != nil { + return errors.Join(fmt.Errorf("delete run history: %w", err), unstage()) + } + if err := tx.Commit(); err != nil { + return errors.Join(fmt.Errorf("commit retention removal: %w", err), unstage()) + } + committed = true + + // The rows are committed: these runs are gone whatever happens to the + // staged files, so record exactly what was removed before any cleanup + // error can return, and never let a cleanup failure be reported as zero + // runs removed or zero reclaimed space. SpaceReclaimedBytes is set here so + // a later VACUUM failure still reports the artifacts actually removed. + result.RemovedRuns = append(result.RemovedRuns, ids...) + sort.Slice(result.RemovedRuns, func(i, j int) bool { return result.RemovedRuns[i] < result.RemovedRuns[j] }) + result.RemovedRunCount = len(ids) + for _, run := range result.WouldRemove { + result.RemovedArtifactFiles += run.Artifacts.Files + result.RemovedArtifactBytes += run.Artifacts.Bytes + } + result.SpaceReclaimedBytes = result.RemovedArtifactBytes + result.Applied = result.RemovedRunCount > 0 + + if staging != "" { + if err := os.RemoveAll(staging); err != nil { + remainingFiles, remainingBytes, _ := retentionWalkSize(staging) + result.StagingPath = staging + result.StagingRemainingFiles = remainingFiles + result.StagingRemainingBytes = remainingBytes + // Files still in staging were staged out of the run tree but not + // deleted, so they are not reclaimed space. + if reclaimed := result.RemovedArtifactBytes - remainingBytes; reclaimed > 0 { + result.SpaceReclaimedBytes = reclaimed + } else { + result.SpaceReclaimedBytes = 0 + } + return fmt.Errorf("run history removed but staged artifact cleanup failed at %s (%d files, %d bytes remain and are not counted as reclaimed): %w", staging, remainingFiles, remainingBytes, err) + } + } + return nil +} + +// deleteRetentionRows removes every table row scoped to the selected runs, in +// foreign-key order, batched at 500 ids per statement in the caller's single +// transaction: step-scoped evidence first, then run_steps, then retry edges, +// then the runs. +func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error { + for _, table := range []struct{ name, column string }{ + {"step_notifications", "run_id"}, + {"step_file_changes", "run_id"}, + {"step_control", "run_id"}, + {"step_artifacts", "run_id"}, + {"step_verdicts", "run_id"}, + {"run_session_owners", "run_id"}, + {"run_steps", "run_id"}, + } { + if err := deleteRetentionBatch(ctx, tx, table.name, table.column, ids); err != nil { + return err + } + } + // Retry provenance is removed when either end is selected, so a removed + // original leaves no dangling edge and a removed retry leaves none either. + for start := 0; start < len(ids); start += retentionDeleteBatch { + batch := ids[start:min(start+retentionDeleteBatch, len(ids))] + placeholders, args := retentionBindings(batch, 2) + if _, err := tx.ExecContext(ctx, "DELETE FROM run_retries WHERE run_id IN ("+placeholders+") OR original_run_id IN ("+placeholders+")", args...); err != nil { + return err + } + } + return deleteRetentionBatch(ctx, tx, "pipeline_runs", "id", ids) +} + +// retentionDeleteBatch bounds each IN clause so a large removal does not build +// an unbounded statement. +const retentionDeleteBatch = 500 + +// retentionBindings builds an "?,?,..." placeholder list for len(ids) values and +// returns args as copies repeats of ids, so a statement with several IN clauses +// over the same id set shares one construction instead of duplicating it. +func retentionBindings(ids []int64, copies int) (string, []any) { + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") + args := make([]any, 0, len(ids)*copies) + for i := 0; i < copies; i++ { + for _, id := range ids { + args = append(args, id) + } + } + return placeholders, args +} + +func deleteRetentionBatch(ctx context.Context, tx *sql.Tx, table, column string, ids []int64) error { + for start := 0; start < len(ids); start += retentionDeleteBatch { + batch := ids[start:min(start+retentionDeleteBatch, len(ids))] + placeholders, args := retentionBindings(batch, 1) + if _, err := tx.ExecContext(ctx, "DELETE FROM "+table+" WHERE "+column+" IN ("+placeholders+")", args...); err != nil { + return err + } + } + return nil +} + +// retentionDBSize measures the database file, its -wal sidecar and the free +// pages SQLite reports, in bytes. It is a measurement, never an estimate. +func (s *Store) retentionDBSize(ctx context.Context) (int64, int64, int64, error) { + var free int64 + var pageSize, freelist int64 + if err := s.db.QueryRowContext(ctx, "PRAGMA page_size").Scan(&pageSize); err != nil { + return 0, 0, 0, err + } + if err := s.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&freelist); err != nil { + return 0, 0, 0, err + } + free = pageSize * freelist + bytes, err := fileSize(s.path) + if err != nil { + return 0, 0, 0, err + } + wal, err := fileSize(s.path + "-wal") + if err != nil { + return 0, 0, 0, err + } + return bytes, wal, free, nil +} + +func fileSize(path string) (int64, error) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + if !info.Mode().IsRegular() { + return 0, nil + } + return info.Size(), nil +} + +// vacuumAndCheckpoint reclaims freed pages and truncates the write-ahead log. +// SQLITE_BUSY is retried a bounded number of times; any other failure, or a +// still-busy database, is returned as an honest error rather than a hang. +func (s *Store) vacuumAndCheckpoint(ctx context.Context) error { + if err := s.retryBusy(ctx, func() error { + _, err := s.db.ExecContext(ctx, "VACUUM") + return err + }); err != nil { + return err + } + return s.retryBusy(ctx, func() error { + var busy, logFrames, checkpointed int + if err := s.db.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil { + return err + } + if busy != 0 { + return fmt.Errorf("wal checkpoint busy") + } + return nil + }) +} + +// retentionBusyAttempts bounds VACUUM/checkpoint retries; a locked database is +// reported, never retried forever. +const retentionBusyAttempts = 3 + +func (s *Store) retryBusy(ctx context.Context, action func() error) error { + var err error + for attempt := 0; attempt < retentionBusyAttempts; attempt++ { + if err = action(); err == nil || !isBusyError(err) { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(attempt+1) * 200 * time.Millisecond): + } + } + return err +} + +func isBusyError(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "busy") || strings.Contains(message, "locked") +} diff --git a/internal/pipeline/retention_test.go b/internal/pipeline/retention_test.go new file mode 100644 index 00000000..f20f48f7 --- /dev/null +++ b/internal/pipeline/retention_test.go @@ -0,0 +1,812 @@ +package pipeline + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// retentionFixture seeds one pipeline definition and a writable store, and +// returns the artifact root the server would use (artifacts/ beside the db). +type retentionFixture struct { + store *Store + definition *protocol.PipelineDetail + root string + now time.Time +} + +func newRetentionFixture(t *testing.T) *retentionFixture { + t.Helper() + dir := t.TempDir() + store, err := Open(filepath.Join(dir, "pipelines.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + if _, err := store.Submit(context.Background(), "retention", Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, err := store.List(context.Background()) + if err != nil { + t.Fatal(err) + } + definition, err := store.Get(context.Background(), items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + return &retentionFixture{store: store, definition: definition, root: filepath.Join(dir, "artifacts"), now: time.Now().UTC().Truncate(time.Second)} +} + +// run creates one run and pins its creation time relative to the fixture's now. +func (f *retentionFixture) run(t *testing.T, key string, age time.Duration) int64 { + t.Helper() + id, _, err := f.store.CreateRun(context.Background(), f.definition, key, "ws", false, nil, map[string]any{"prompt": key}) + if err != nil { + t.Fatal(err) + } + f.setCreated(t, id, f.now.Add(-age)) + f.setStatus(t, id, "succeeded") + return id +} + +func (f *retentionFixture) checkpoint(t *testing.T) { + t.Helper() + var busy, logFrames, checkpointed int + if err := f.store.db.QueryRowContext(context.Background(), "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil { + t.Fatal(err) + } +} + +func (f *retentionFixture) setCreated(t *testing.T, id int64, at time.Time) { + t.Helper() + if _, err := f.store.db.ExecContext(context.Background(), "UPDATE pipeline_runs SET created_at=? WHERE id=?", at.UTC().Format("2006-01-02 15:04:05"), id); err != nil { + t.Fatal(err) + } +} + +func (f *retentionFixture) setStatus(t *testing.T, id int64, status string) { + t.Helper() + if _, err := f.store.db.ExecContext(context.Background(), "UPDATE pipeline_runs SET status=? WHERE id=?", status, id); err != nil { + t.Fatal(err) + } +} + +func (f *retentionFixture) step(t *testing.T, id int64, sequence int, status string, active bool) { + t.Helper() + activeInt := 0 + if active { + activeInt = 1 + } + if _, err := f.store.db.ExecContext(context.Background(), "INSERT INTO run_steps(run_id,sequence,stage_key,status,agent_active,input_json) VALUES(?,?,?,?,?,'{}')", id, sequence, "worker", status, activeInt); err != nil { + t.Fatal(err) + } +} + +func (f *retentionFixture) retry(t *testing.T, retryID, originalID int64) { + t.Helper() + if _, err := f.store.db.ExecContext(context.Background(), "INSERT INTO run_retries(run_id,original_run_id) VALUES(?,?)", retryID, originalID); err != nil { + t.Fatal(err) + } +} + +func (f *retentionFixture) artifact(t *testing.T, id int64, sequence int, name string, size int) { + t.Helper() + dir := filepath.Join(f.root, fmt.Sprint(id), fmt.Sprint(sequence)) + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), make([]byte, size), 0600); err != nil { + t.Fatal(err) + } +} + +func (f *retentionFixture) report(t *testing.T, policy RetentionPolicy) *RetentionReport { + t.Helper() + report, err := f.store.RetentionReport(context.Background(), policy, f.root, f.now) + if err != nil { + t.Fatal(err) + } + return report +} + +func removableIDs(report *RetentionReport) []int64 { + ids := make([]int64, 0, len(report.WouldRemove)) + for _, run := range report.WouldRemove { + ids = append(ids, run.ID) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids +} + +func TestRetentionPolicySelection(t *testing.T) { + f := newRetentionFixture(t) + // Five runs, newest first: id5..id1, at 1h, 12h, 24h, 48h and 72h old. + ages := []time.Duration{0, 72 * time.Hour, 48 * time.Hour, 24 * time.Hour, 12 * time.Hour, time.Hour} + for i := 1; i <= 5; i++ { + f.run(t, fmt.Sprintf("run-%d", i), ages[i]) + } + hourly := f.report(t, RetentionPolicy{OlderThan: 35 * time.Hour}) + if got := removableIDs(hourly); fmt.Sprint(got) != "[1 2]" { + t.Fatalf("age-only selection = %v, want [1 2]", got) + } + byCount := f.report(t, RetentionPolicy{Keep: 2}) + if got := removableIDs(byCount); fmt.Sprint(got) != "[1 2 3]" { + t.Fatalf("count-only selection = %v, want [1 2 3]", got) + } + both := f.report(t, RetentionPolicy{OlderThan: 35 * time.Hour, Keep: 2}) + if got := removableIDs(both); fmt.Sprint(got) != "[1 2]" { + t.Fatalf("combined selection = %v, want the intersection [1 2]", got) + } + keepNone := f.report(t, RetentionPolicy{Keep: 0}) + if got := removableIDs(keepNone); fmt.Sprint(got) != "[1 2 3 4 5]" { + t.Fatalf("keep 0 selection = %v, want all runs", got) + } + keepAll := f.report(t, RetentionPolicy{Keep: 99}) + if len(keepAll.WouldRemove) != 0 { + t.Fatalf("keep 99 selected %d runs, want none", len(keepAll.WouldRemove)) + } + // Exact boundary: a run created exactly at the cutoff is retained. + f.setCreated(t, 1, f.now.Add(-30*time.Hour)) + boundary := f.report(t, RetentionPolicy{OlderThan: 30 * time.Hour}) + for _, id := range removableIDs(boundary) { + if id == 1 { + t.Fatal("run exactly at the age cut-off must be retained") + } + } +} + +func TestRetentionPolicyValidation(t *testing.T) { + for _, policy := range []RetentionPolicy{{Keep: -1}, {OlderThan: -time.Hour, Keep: -1}, {Keep: -2}} { + if err := policy.Validate(); err == nil { + t.Fatalf("invalid policy %+v was accepted", policy) + } + } + if err := (RetentionPolicy{Keep: 0}).Validate(); err != nil { + t.Fatalf("keep 0 must be a valid explicit bound: %v", err) + } + if err := (RetentionPolicy{OlderThan: time.Hour}).Validate(); err != nil { + t.Fatalf("age bound must be valid: %v", err) + } +} + +func TestRetentionProtection(t *testing.T) { + f := newRetentionFixture(t) + f.run(t, "queued", 48*time.Hour) + f.run(t, "running", 48*time.Hour) + f.run(t, "waiting-run", 48*time.Hour) + f.run(t, "waiting-step", 48*time.Hour) + f.run(t, "agent-active", 48*time.Hour) + f.setStatus(t, 1, protocol.RunStatusQueued) + f.setStatus(t, 2, protocol.RunStatusRunning) + f.setStatus(t, 3, protocol.RunStatusWaiting) + f.step(t, 4, 1, protocol.StepStatusWaiting, false) + f.step(t, 5, 1, "running", true) + + report := f.report(t, RetentionPolicy{OlderThan: time.Hour}) + if len(report.WouldRemove) != 0 { + t.Fatalf("protected runs must never be removable: %v", removableIDs(report)) + } + if len(report.Protected) != 5 { + t.Fatalf("want 5 protected runs, got %+v", report.Protected) + } + reasons := map[int64]string{} + for _, run := range report.Protected { + reasons[run.ID] = run.Reason + if run.Reason == "" { + t.Fatalf("protected run %d has no reason", run.ID) + } + } + if !strings.Contains(reasons[4], "step is waiting") || !strings.Contains(reasons[5], "agent_active") { + t.Fatalf("protection reasons wrong: %+v", reasons) + } +} + +func TestRetentionRetryProvenance(t *testing.T) { + f := newRetentionFixture(t) + original := f.run(t, "original", 48*time.Hour) + retry := f.run(t, "retry", 1*time.Hour) // retained (newest 2 kept) + f.retry(t, retry, original) + + report := f.report(t, RetentionPolicy{Keep: 1}) + if len(report.WouldRemove) != 0 { + t.Fatalf("an original referenced by a retained retry must be protected: %v", removableIDs(report)) + } + if len(report.Protected) != 1 || report.Protected[0].ID != original { + t.Fatalf("want the original protected, got %+v", report.Protected) + } + + // Transitive: middle references original, retained keeps middle alive. + f2 := newRetentionFixture(t) + a := f2.run(t, "a", 72*time.Hour) + b := f2.run(t, "b", 60*time.Hour) + c := f2.run(t, "c", 1*time.Hour) + f2.retry(t, b, a) + f2.retry(t, c, b) + report2 := f2.report(t, RetentionPolicy{Keep: 1}) + if len(report2.WouldRemove) != 0 || len(report2.Protected) != 2 { + t.Fatalf("transitive chain must protect both originals: remove=%v protected=%+v", removableIDs(report2), report2.Protected) + } + + // If both original and retry are selected, no retained link remains and both + // may be removed. + f3 := newRetentionFixture(t) + o := f3.run(t, "o", 72*time.Hour) + r := f3.run(t, "r", 60*time.Hour) + f3.retry(t, r, o) + report3 := f3.report(t, RetentionPolicy{Keep: 0}) + if len(report3.WouldRemove) != 2 || len(report3.Protected) != 0 { + t.Fatalf("both selected must be removable: remove=%v protected=%+v", removableIDs(report3), report3.Protected) + } +} + +func TestRetentionApplyRemovesHistoryAndArtifacts(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + old := f.run(t, "old", 72*time.Hour) + keep := f.run(t, "keep", 1*time.Hour) + f.artifact(t, old, 1, "report.bin", 2048) + f.artifact(t, keep, 1, "keep.bin", 16) + // Step-scoped evidence for the removed run must go with it. + f.step(t, old, 1, "completed", false) + if _, err := f.store.db.ExecContext(ctx, "INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,1,'[]')", old); err != nil { + t.Fatal(err) + } + before := f.report(t, RetentionPolicy{Keep: 1}) + + beforePipelines, beforeVersions, beforeSubmissions := f.counts(t) + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) + if err != nil { + t.Fatal(err) + } + if !result.Applied || result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{old}) { + t.Fatalf("apply removed wrong runs: %+v", result) + } + if result.RemovedArtifactBytes != 2048 || result.RemovedArtifactFiles != 1 { + t.Fatalf("apply must remove the run's artifacts: %+v", result) + } + if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) { + t.Fatalf("removed run artifacts still present: %v", err) + } + if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(keep), "1", "keep.bin")); err != nil { + t.Fatalf("retained run artifacts must remain: %v", err) + } + after := f.report(t, RetentionPolicy{Keep: 1}) + if len(after.WouldRemove) != 0 || after.TotalRuns != 1 { + t.Fatalf("removed run still listed: %+v", after) + } + if after.Pipelines != beforePipelines || after.PipelineVersions != beforeVersions { + t.Fatalf("definitions/versions changed: %d/%d -> %d/%d", beforePipelines, beforeVersions, after.Pipelines, after.PipelineVersions) + } + afterSubmissions := f.count(t, "submissions") + if afterSubmissions != beforeSubmissions { + t.Fatalf("submissions changed: %d -> %d", beforeSubmissions, afterSubmissions) + } + if n := f.count(t, "run_steps"); n != 0 { + t.Fatalf("removed run's steps must be gone: %d", n) + } + if n := f.count(t, "step_notifications"); n != 0 { + t.Fatalf("step-scoped evidence must be removed with its run: %d", n) + } + if n := f.count(t, "run_retries"); n != 0 { + t.Fatalf("retry edges must be removed: %d", n) + } + if before.ReclaimableBytes != 2048+before.DatabaseFreeBytes { + t.Fatalf("reclaimable bytes = %d, want artifacts + free pages", before.ReclaimableBytes) + } + if result.DatabaseBytesReclaimed < 0 || result.SpaceReclaimedBytes < result.RemovedArtifactBytes { + t.Fatalf("reclaimed bytes are not measured from the removal: %+v", result) + } + if !result.Vacuumed || result.DatabaseFreeBytesAfter != 0 { + t.Fatalf("apply must reclaim free pages to zero: %+v", result) + } +} + +func (f *retentionFixture) counts(t *testing.T) (int, int, int) { + t.Helper() + return f.count(t, "pipelines"), f.count(t, "pipeline_versions"), f.count(t, "submissions") +} + +func (f *retentionFixture) count(t *testing.T, table string) int { + t.Helper() + var n int + if err := f.store.db.QueryRowContext(context.Background(), "SELECT COUNT(*) FROM "+table).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func TestRetentionReportIsReadOnly(t *testing.T) { + f := newRetentionFixture(t) + f.run(t, "a", 72*time.Hour) + f.run(t, "b", 1*time.Hour) + f.artifact(t, 1, 1, "a.bin", 512) + path := f.store.path + if err := f.store.Close(); err != nil { + t.Fatal(err) + } + readOnly, err := OpenReadOnly(path) + if err != nil { + t.Fatal(err) + } + defer readOnly.Close() + + beforeRuns, beforeArtifacts := countFiles(t, filepath.Join(f.root)) + beforeFiles := snapshotFile(t, path) + beforeWAL := snapshotFile(t, path+"-wal") + var beforeFree int64 + if err := readOnly.db.QueryRowContext(context.Background(), "PRAGMA freelist_count").Scan(&beforeFree); err != nil { + t.Fatal(err) + } + report, err := readOnly.RetentionReport(context.Background(), RetentionPolicy{Keep: 1}, f.root, f.now) + if err != nil { + t.Fatal(err) + } + if report.Applied { + t.Fatal("a report must never report applied=true") + } + afterFiles := snapshotFile(t, path) + afterWAL := snapshotFile(t, path+"-wal") + var afterFree int64 + if err := readOnly.db.QueryRowContext(context.Background(), "PRAGMA freelist_count").Scan(&afterFree); err != nil { + t.Fatal(err) + } + if afterFiles.size != beforeFiles.size || afterFiles.mod != beforeFiles.mod { + t.Fatalf("report changed the database file: %+v -> %+v", beforeFiles, afterFiles) + } + if afterWAL.size != beforeWAL.size { + t.Fatalf("report changed the WAL: %d -> %d", beforeWAL.size, afterWAL.size) + } + if afterFree != beforeFree { + t.Fatalf("report changed the freelist: %d -> %d", beforeFree, afterFree) + } + afterRuns, afterArtifacts := countFiles(t, filepath.Join(f.root)) + if afterRuns != beforeRuns || afterArtifacts != beforeArtifacts { + t.Fatalf("report changed rows/artifacts: %d/%d -> %d/%d", beforeRuns, beforeArtifacts, afterRuns, afterArtifacts) + } +} + +type fileSnapshot struct { + size int64 + mod int64 +} + +func snapshotFile(t *testing.T, path string) fileSnapshot { + t.Helper() + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return fileSnapshot{} + } + t.Fatal(err) + } + return fileSnapshot{size: info.Size(), mod: info.ModTime().UnixNano()} +} + +func countFiles(t *testing.T, root string) (dirs, files int) { + t.Helper() + _ = filepath.WalkDir(root, func(_ string, entry os.DirEntry, err error) error { + if err != nil { + return nil + } + if entry.IsDir() { + dirs++ + return nil + } + files++ + return nil + }) + return dirs, files +} + +func TestRetentionVacuumReclaimsFreedPages(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + old := f.run(t, "large", 72*time.Hour) + f.run(t, "new", 1*time.Hour) + // A large payload leaves free pages behind after deletion. + f.step(t, old, 1, "completed", false) + if _, err := f.store.db.ExecContext(ctx, "INSERT INTO step_file_changes(run_id,sequence,changes_json) VALUES(?,1,?)", old, strings.Repeat("x", 512*1024)); err != nil { + t.Fatal(err) + } + // Flush the WAL so the "before" size reflects the payload on disk. + f.checkpoint(t) + before := f.report(t, RetentionPolicy{Keep: 1}) + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) + if err != nil { + t.Fatal(err) + } + if result.DatabaseFreeBytesAfter != 0 { + t.Fatalf("VACUUM must leave zero free pages: %+v", result) + } + if !result.Vacuumed { + t.Fatal("apply must report vacuumed=true") + } + if result.DatabaseBytesAfter >= before.DatabaseBytes { + t.Fatalf("database did not shrink after removing a large payload: %d -> %d", before.DatabaseBytes, result.DatabaseBytesAfter) + } + if result.DatabaseBytesReclaimed != before.DatabaseBytes-result.DatabaseBytesAfter { + t.Fatalf("reclaimed bytes not measured: %+v", result) + } +} + +func TestRetentionVacuumFailureIsHonest(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + f.run(t, "a", time.Hour) + // Hold the database write lock from another connection so VACUUM cannot + // proceed; a short busy timeout keeps the test fast but exercises the same + // bounded-retry failure path a busy database produces. + blocker, err := sql.Open("sqlite", f.store.path) + if err != nil { + t.Fatal(err) + } + defer blocker.Close() + conn, err := blocker.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + t.Fatal(err) + } + if _, err := f.store.db.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil { + t.Fatal(err) + } + // Nothing is selected, so only the reclaim step runs and must fail honestly. + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 5}, f.root, f.now) + if err == nil { + t.Fatalf("apply must report a VACUUM failure: %+v", result) + } + if result == nil || result.VacuumError == "" || result.Vacuumed { + t.Fatalf("vacuum_error must be set and vacuumed false: %+v", result) + } + if result.DatabaseBytesReclaimed != 0 || result.SpaceReclaimedBytes != 0 { + t.Fatalf("a failed VACUUM must not claim reclaimed pages: %+v", result) + } +} + +func TestOpenMaintenanceFailsWithoutCreating(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing.db") + if _, err := OpenMaintenance(path); err == nil { + t.Fatal("OpenMaintenance must fail for a missing database") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("OpenMaintenance must not create the database: %v", err) + } +} + +func TestRetentionToleratesMissingProvenanceAndRoot(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + f.run(t, "a", 72*time.Hour) + if _, err := f.store.db.ExecContext(ctx, "DROP TABLE run_retries"); err != nil { + t.Fatal(err) + } + report, err := f.store.RetentionReport(ctx, RetentionPolicy{Keep: 0}, filepath.Join(t.TempDir(), "absent"), f.now) + if err != nil { + t.Fatalf("a missing run_retries table must be an empty edge set: %v", err) + } + if len(report.WouldRemove) != 1 || report.ArtifactBytes != 0 { + t.Fatalf("missing artifacts root must be a zero-byte no-op: %+v", report.WouldRemove) + } +} + +func TestRetentionApplyDoesNotCreateArtifactsRoot(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + id := f.run(t, "a", 72*time.Hour) + f.run(t, "b", time.Hour) + // No artifact directory exists yet; an apply that stages no run directory + // must not create the artifacts root as a side effect. + if _, err := os.Stat(f.root); !os.IsNotExist(err) { + t.Fatalf("fixture should start without an artifacts root: %v", err) + } + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) + if err != nil { + t.Fatal(err) + } + if result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{id}) { + t.Fatalf("apply removed wrong runs: %+v", result) + } + if _, err := os.Stat(f.root); !os.IsNotExist(err) { + t.Fatalf("apply created the artifacts root as a side effect: %v", err) + } + if n := f.count(t, "pipeline_runs"); n != 1 { + t.Fatalf("removed run still present: %d", n) + } +} + +func TestRetentionRefusesFileAtRunPathAndConfinesSymlink(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + id := f.run(t, "a", 72*time.Hour) + if err := os.MkdirAll(f.root, 0700); err != nil { + t.Fatal(err) + } + // A regular file at / must be refused, not deleted. + victim := filepath.Join(t.TempDir(), "victim.txt") + if err := os.WriteFile(victim, []byte("precious"), 0600); err != nil { + t.Fatal(err) + } + runPath := filepath.Join(f.root, fmt.Sprint(id)) + if err := os.WriteFile(runPath, []byte("not a directory"), 0600); err != nil { + t.Fatal(err) + } + if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err == nil { + t.Fatal("apply must refuse a regular file at the run path") + } + if data, err := os.ReadFile(runPath); err != nil || string(data) != "not a directory" { + t.Fatalf("regular file at run path was touched: %q %v", data, err) + } + if err := os.Remove(runPath); err != nil { + t.Fatal(err) + } + if _, err := f.store.db.ExecContext(ctx, "SELECT COUNT(*) FROM pipeline_runs WHERE id=?", id); err != nil { + t.Fatal(err) + } + var exists int + if err := f.store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM pipeline_runs WHERE id=?", id).Scan(&exists); err != nil || exists != 1 { + t.Fatalf("refused apply must delete no rows: %d %v", exists, err) + } + + // A symlink at / removes the link only, never its target. + if err := os.Symlink(victim, runPath); err != nil { + t.Fatal(err) + } + if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(victim); err != nil || string(data) != "precious" { + t.Fatalf("symlink target must survive: %q %v", data, err) + } + if _, err := os.Lstat(runPath); !os.IsNotExist(err) { + t.Fatalf("symlink at run path should be removed with the run: %v", err) + } +} + +func TestRetentionRollsBackOnDeleteError(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + id := f.run(t, "a", 72*time.Hour) + f.artifact(t, id, 1, "a.bin", 256) + f.step(t, id, 1, "completed", false) + if _, err := f.store.db.ExecContext(ctx, "CREATE TRIGGER block_run_delete BEFORE DELETE ON pipeline_runs BEGIN SELECT RAISE(ABORT,'blocked'); END;"); err != nil { + t.Fatal(err) + } + if _, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 0}, f.root, f.now); err == nil { + t.Fatal("apply must return the delete error") + } + // Rows and staged artifacts are restored. + if n := f.count(t, "pipeline_runs"); n != 1 { + t.Fatalf("rollback lost the run: %d", n) + } + if n := f.count(t, "run_steps"); n != 1 { + t.Fatalf("rollback lost the step: %d", n) + } + if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(id), "1", "a.bin")); err != nil { + t.Fatalf("rollback must restore staged artifacts: %v", err) + } + entries, err := os.ReadDir(f.root) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".retention-staging-") { + t.Fatalf("staging directory left behind: %s", entry.Name()) + } + } +} + +func TestRetentionAbortsOnProtectionRace(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + original := f.run(t, "original", 72*time.Hour) + f.run(t, "keep", 1*time.Hour) + f.artifact(t, original, 1, "a.bin", 256) + report := f.report(t, RetentionPolicy{Keep: 1}) + ids := removableIDs(report) + if len(ids) != 1 { + t.Fatalf("expected one removable run, got %v", ids) + } + // A retained retry appears between the report and the delete. + retry := f.run(t, "retry", time.Hour) + f.retry(t, retry, original) + result := &RetentionApplyResult{} + err := f.store.removeRetentionRuns(ctx, RetentionPolicy{Keep: 1}, f.root, f.now, ids, result) + if err == nil || !strings.Contains(err.Error(), "became protected") { + t.Fatalf("apply must abort when a selected run becomes protected: %v", err) + } + if n := f.count(t, "pipeline_runs"); n != 3 { + t.Fatalf("aborted apply deleted rows: %d", n) + } + if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(original), "1", "a.bin")); err != nil { + t.Fatalf("aborted apply touched artifacts: %v", err) + } +} + +// TestRetentionReportsRemovedRunsWhenStagingCleanupFails covers the post-commit +// inspection gap where a successful row/artifact removal was reported as zero +// removed when the staging directory could not be deleted: the actual removed +// run(s) and artifact totals must still be reported, and the leftover staging +// bytes must not be counted as reclaimed. +func TestRetentionReportsRemovedRunsWhenStagingCleanupFails(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + old := f.run(t, "old", 72*time.Hour) + f.run(t, "keep", time.Hour) + // A run directory whose nested subdirectory cannot be emptied: os.RemoveAll + // fails on it, so the post-commit staging cleanup returns an error. + dir := filepath.Join(f.root, fmt.Sprint(old), "1") + locked := filepath.Join(dir, "locked") + if err := os.MkdirAll(locked, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "a.bin"), make([]byte, 128), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(locked, "inside.bin"), make([]byte, 64), 0600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0500); err != nil { + t.Fatal(err) + } + // Restore write permission before t.TempDir's cleanup removes the tree. + t.Cleanup(func() { + _ = filepath.WalkDir(f.root, func(path string, entry os.DirEntry, err error) error { + if err == nil && entry.IsDir() { + _ = os.Chmod(path, 0700) + } + return nil + }) + }) + + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) + if err == nil { + t.Fatal("apply must report the staging cleanup failure") + } + if result == nil { + t.Fatal("a cleanup failure must still report what was removed") + } + if result.RemovedRunCount != 1 || fmt.Sprint(result.RemovedRuns) != fmt.Sprint([]int64{old}) { + t.Fatalf("rows were removed but not reported: %+v", result) + } + if !result.Applied { + t.Fatalf("run history was removed, so applied must be true: %+v", result) + } + if result.RemovedArtifactBytes != 192 || result.RemovedArtifactFiles != 2 { + t.Fatalf("removed artifact totals wrong: %+v", result) + } + if result.StagingPath == "" || result.StagingRemainingBytes != 64 || result.StagingRemainingFiles != 1 { + t.Fatalf("staging leftovers not reported: %+v", result) + } + if result.SpaceReclaimedBytes != result.RemovedArtifactBytes-result.StagingRemainingBytes { + t.Fatalf("reclaimed space must exclude staging leftovers: %+v", result) + } + if n := f.count(t, "pipeline_runs"); n != 1 { + t.Fatalf("the run rows must be gone: %d", n) + } + if _, err := os.Lstat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) { + t.Fatalf("the run tree must have been staged out of the artifacts root: %v", err) + } +} + +// TestRetentionStagesThroughSymlinkedArtifactsRoot covers the inspection gap +// where a symlinked artifacts root was sized by the report but skipped by +// apply, claiming artifact bytes as removed while leaving every file in place. +// A symlinked root is followed consistently, so the artifacts really go. +func TestRetentionStagesThroughSymlinkedArtifactsRoot(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + id := f.run(t, "old", 72*time.Hour) + f.run(t, "keep", time.Hour) + real := filepath.Join(t.TempDir(), "real-artifacts") + if err := os.MkdirAll(filepath.Join(real, fmt.Sprint(id), "1"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(real, fmt.Sprint(id), "1", "a.bin"), make([]byte, 4096), 0600); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "artifacts") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + report, err := f.store.RetentionReport(ctx, RetentionPolicy{Keep: 1}, link, f.now) + if err != nil { + t.Fatal(err) + } + if report.ArtifactBytes != 4096 { + t.Fatalf("report must size artifacts through the symlinked root: %+v", report) + } + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, link, f.now) + if err != nil { + t.Fatal(err) + } + if result.RemovedRunCount != 1 || result.RemovedArtifactBytes != 4096 { + t.Fatalf("a symlinked root must still remove and report the artifacts: %+v", result) + } + if result.SpaceReclaimedBytes < result.RemovedArtifactBytes { + t.Fatalf("reclaimed space must include the removed artifacts: %+v", result) + } + if _, err := os.Stat(filepath.Join(real, fmt.Sprint(id))); !os.IsNotExist(err) { + t.Fatalf("artifacts behind a symlinked root must actually be removed: %v", err) + } + entries, err := os.ReadDir(real) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".retention-staging-") { + t.Fatalf("staging directory left in the symlink target: %s", entry.Name()) + } + } +} + +// TestRetentionReclaimFailureAfterRemovalIsHonest covers a reclaim failure that +// happens after the rows and artifacts are already removed: the removed +// artifact bytes must still be reported, the database pages must not be +// claimed, and the total must not be zero. A concurrent read snapshot blocks +// the final WAL checkpoint while leaving the write transaction and VACUUM free +// to run; both checkpoint and VACUUM failures route through vacuumAndCheckpoint +// and the same honest reporting path. +func TestRetentionReclaimFailureAfterRemovalIsHonest(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + old := f.run(t, "old", 72*time.Hour) + f.run(t, "keep", time.Hour) + f.artifact(t, old, 1, "a.bin", 4096) + + reader, err := sql.Open("sqlite", f.store.path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + conn, err := reader.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, "BEGIN"); err != nil { + t.Fatal(err) + } + var n int + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM pipeline_runs").Scan(&n); err != nil { + t.Fatal(err) + } + if _, err := f.store.db.ExecContext(ctx, "PRAGMA busy_timeout=50"); err != nil { + t.Fatal(err) + } + + result, err := f.store.ApplyRetention(ctx, RetentionPolicy{Keep: 1}, f.root, f.now) + if err == nil { + t.Fatalf("apply must report the reclaim failure: %+v", result) + } + if result == nil || result.VacuumError == "" || result.Vacuumed { + t.Fatalf("reclaim failure must set vacuum_error and leave vacuumed false: %+v", result) + } + if result.RemovedRunCount != 1 || result.RemovedArtifactBytes != 4096 || !result.Applied { + t.Fatalf("removed artifacts must still be reported: %+v", result) + } + if result.DatabaseBytesReclaimed != 0 { + t.Fatalf("a failed reclaim must not claim database pages: %+v", result) + } + if result.SpaceReclaimedBytes != result.RemovedArtifactBytes { + t.Fatalf("space reclaimed must be the removed artifacts and no database pages: %+v", result) + } + if n := f.count(t, "pipeline_runs"); n != 1 { + t.Fatalf("the removed run must be gone: %d", n) + } + if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(old))); !os.IsNotExist(err) { + t.Fatalf("the removed run artifacts must be gone: %v", err) + } + if _, err := conn.ExecContext(ctx, "COMMIT"); err != nil { + t.Fatal(err) + } +} diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 0e309d68..17be1942 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -18,17 +18,29 @@ import ( ) type Store struct { - db *sql.DB - dir string - mu sync.Mutex + db *sql.DB + dir string + path string + mu sync.Mutex } -func Open(path string) (*Store, error) { - db, err := sql.Open("sqlite", path) +// openDB opens one SQLite connection pool for a DSN. Every open path shares it +// so the connection count and pool settings cannot drift between the writable, +// read-only and maintenance stores. +func openDB(dsn string) (*sql.DB, error) { + db, err := sql.Open("sqlite", dsn) if err != nil { return nil, err } db.SetMaxOpenConns(1) + return db, nil +} + +func Open(path string) (*Store, error) { + db, err := openDB(path) + if err != nil { + return nil, err + } _, err = db.Exec(`PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000; CREATE TABLE IF NOT EXISTS pipelines (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); CREATE TABLE IF NOT EXISTS pipeline_versions (id INTEGER PRIMARY KEY, pipeline_id INTEGER NOT NULL REFERENCES pipelines(id), version INTEGER NOT NULL, source TEXT NOT NULL, content_hash TEXT NOT NULL, graph_json TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(pipeline_id, version)); @@ -37,7 +49,7 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU db.Close() return nil, err } - s := &Store{db: db, dir: filepath.Dir(path)} + s := &Store{db: db, dir: filepath.Dir(path), path: path} if err := s.initRuns(); err != nil { db.Close() return nil, err @@ -63,16 +75,36 @@ func OpenReadOnly(path string) (*Store, error) { if _, err := os.Stat(path); err != nil { return nil, err } - db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") + db, err := openDB("file:" + path + "?mode=ro") if err != nil { return nil, err } - db.SetMaxOpenConns(1) if _, err := db.Exec("PRAGMA busy_timeout=5000"); err != nil { db.Close() return nil, err } - return &Store{db: db, dir: filepath.Dir(path)}, nil + return &Store{db: db, dir: filepath.Dir(path), path: path}, nil +} + +// OpenMaintenance opens an existing database read-write for an explicit +// operator maintenance action such as retention apply. It never creates a file +// and never runs schema DDL or changes the journal mode, so it cannot silently +// create or migrate a live database. It uses an immediate transaction lock and +// a longer busy timeout, so a concurrent server write produces an honest busy +// failure instead of a hang. The deployed server is never started or changed. +func OpenMaintenance(path string) (*Store, error) { + if _, err := os.Stat(path); err != nil { + return nil, err + } + db, err := openDB("file:" + path + "?_txlock=immediate") + if err != nil { + return nil, err + } + if _, err := db.Exec("PRAGMA busy_timeout=30000; PRAGMA foreign_keys=ON"); err != nil { + db.Close() + return nil, err + } + return &Store{db: db, dir: filepath.Dir(path), path: path}, nil } // Directory is the resolved directory holding the database, used to place diff --git a/internal/server/concurrency_test.go b/internal/server/concurrency_test.go index 354f13b2..33d51d05 100644 --- a/internal/server/concurrency_test.go +++ b/internal/server/concurrency_test.go @@ -20,6 +20,46 @@ func concurrentStore(t *testing.T) (context.Context, *runner, int64) { return newConcurrentRunner(t, pipeline.Template([]string{"one"}, true)) } +// submitRunner opens a fresh store, submits source through the parser and returns +// the runner, its store and the pipeline id. Callers own the returned store's +// lifetime and cleanup so tests can inspect persisted runs; newConcurrentRunner and +// the review-contract tests share this setup instead of copying it. +func submitRunner(t *testing.T, ctx context.Context, source string) (*runner, *pipeline.Store, int64) { + t.Helper() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + if _, err := store.Submit(ctx, "test", source); err != nil { + store.Close() + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + store.Close() + t.Fatal(err) + } + r, err := newRunner(ctx, store) + if err != nil { + store.Close() + t.Fatal(err) + } + return r, store, items[0].ID +} + +// startRun launches one run of a submitted pipeline and waits for it to stop, +// returning the started run detail. It is the single start helper for tests that +// only need a completed run. +func startRun(t *testing.T, r *runner, pipelineID int64, key string) *protocol.RunDetail { + t.Helper() + run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + return run +} + // newConcurrentRunner builds the shared test store/runner around a submitted // pipeline source. Tests that need an agent stage (useAgent true) must pass a // source with a non-empty tools block: the default template's empty tools block @@ -28,34 +68,13 @@ func concurrentStore(t *testing.T) (context.Context, *runner, int64) { func newConcurrentRunner(t *testing.T, source string) (context.Context, *runner, int64) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) - store, err := pipeline.Open(filepath.Join(t.TempDir(), "concurrent.db")) - if err != nil { - cancel() - t.Fatal(err) - } - if _, err := store.Submit(ctx, "test", source); err != nil { - cancel() - store.Close() - t.Fatal(err) - } - items, err := store.List(ctx) - if err != nil { - cancel() - store.Close() - t.Fatal(err) - } - r, err := newRunner(ctx, store) - if err != nil { - cancel() - store.Close() - t.Fatal(err) - } + r, store, id := submitRunner(t, ctx, source) t.Cleanup(func() { cancel() r.wg.Wait() store.Close() }) - return ctx, r, items[0].ID + return ctx, r, id } // agentStore returns a runner whose pipeline has an agent stage (a non-empty, diff --git a/internal/server/control_test.go b/internal/server/control_test.go index e8cc1f27..141916b2 100644 --- a/internal/server/control_test.go +++ b/internal/server/control_test.go @@ -59,11 +59,7 @@ func TestRuntimeLimitAndSelectedMemory(t *testing.T) { return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "fix layout", "ui_revision": 0}}, nil } } - run, err := runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "test", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) - if err != nil { - t.Fatal(err) - } - runner.wg.Wait() + run := startRun(t, runner, items[0].ID, "test") retained, err := store.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) diff --git a/internal/server/human_test.go b/internal/server/human_test.go index 8daa4ca8..c5ac1dfd 100644 --- a/internal/server/human_test.go +++ b/internal/server/human_test.go @@ -48,12 +48,7 @@ func startHumanRun(t *testing.T, ctx context.Context, r *runner, store *pipeline } } } - run, err := r.start(protocol.Message{PipelineID: pipelineID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) - if err != nil { - t.Fatal(err) - } - r.wg.Wait() - return run + return startRun(t, r, pipelineID, key) } func TestHumanStageWaitsAndDecisionResumes(t *testing.T) { diff --git a/internal/server/retention_test.go b/internal/server/retention_test.go new file mode 100644 index 00000000..d2143cba --- /dev/null +++ b/internal/server/retention_test.go @@ -0,0 +1,144 @@ +package server + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" +) + +// TestRetentionRemovesServedRunAndArtifact proves the artifact endpoint serves a +// live run's artifact, that an explicit retention apply removes the run and its +// artifact, and that the endpoint then 404s while a still-defined pipeline and +// version load as before. +func TestRetentionRemovesServedRunAndArtifact(t *testing.T) { + ctx := context.Background() + handler, step := runWithArtifact(t) + store := handler.runner.store + root := handler.runner.artifactsRoot + runID := runIDFromURL(step.Artifacts[0].URL) + + srv := httptest.NewServer(handler) + defer srv.Close() + resp, err := http.Get(srv.URL + step.Artifacts[0].URL) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("recorded artifact must serve before removal: %d", resp.StatusCode) + } + + result, err := store.ApplyRetention(ctx, pipeline.RetentionPolicy{Keep: 0}, root, time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + if !result.Applied || result.RemovedRunCount != 1 { + t.Fatalf("retention did not remove the served run: %+v", result) + } + if _, err := os.Stat(filepath.Join(root, fmt.Sprint(runID))); !os.IsNotExist(err) { + t.Fatalf("removed run artifacts must be gone: %v", err) + } + gone, err := http.Get(srv.URL + step.Artifacts[0].URL) + if err != nil { + t.Fatal(err) + } + gone.Body.Close() + if gone.StatusCode != http.StatusNotFound { + t.Fatalf("removed run's artifact endpoint must 404: %d", gone.StatusCode) + } + + runs, err := store.ListRuns(ctx) + if err != nil { + t.Fatal(err) + } + for _, run := range runs { + if run.ID == runID { + t.Fatalf("removed run still listed: %+v", run) + } + } + // The pipeline definition and its immutable version still load, and a run of + // another definition would be unaffected. + items, err := store.List(ctx) + if err != nil || len(items) == 0 { + t.Fatalf("definitions must survive retention: %v %v", items, err) + } + definition, err := store.Get(ctx, items[0].ID, 1) + if err != nil || definition.Source == "" { + t.Fatalf("pipeline version must still load after its run history is removed: %+v %v", definition, err) + } +} + +// TestServerStartupAndListRunsNeverPrune pins the no-automatic-pruning rule: +// constructing the handler and issuing a list_runs leaves old runs untouched, +// and a run that is still active is protected from an explicit apply. +func TestServerStartupAndListRunsNeverPrune(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "server-retention.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err := store.Submit(ctx, "server-retention", pipeline.Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + definition, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + id, _, err := store.CreateRun(ctx, definition, fmt.Sprintf("old-%d", i), "ws", false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + if err := store.FinishRun(ctx, id, "succeeded", "success", "", map[string]any{}); err != nil { + t.Fatal(err) + } + } + + handler := Handler(ctx, store).(*runHandler) + defer func() { cancel(); WaitRuns(handler) }() + + // Startup plus a list_runs request must not prune the old runs. + runs, err := store.ListRuns(ctx) + if err != nil || len(runs) != 3 { + t.Fatalf("startup/list_runs pruned history: %d %v", len(runs), err) + } + + // A run created after startup and left active is protected and kept. + active, _, err := store.CreateRun(ctx, definition, "active", "ws", false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + result, err := store.ApplyRetention(ctx, pipeline.RetentionPolicy{Keep: 0}, handler.runner.artifactsRoot, time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + if result.RemovedRunCount != 3 { + t.Fatalf("only the finished runs may be removed: %d", result.RemovedRunCount) + } + found := false + for _, protected := range result.Protected { + if protected.ID == active { + found = true + } + } + if !found { + t.Fatalf("active run must be reported protected: %+v", result.Protected) + } + runs, err = store.ListRuns(ctx) + if err != nil || len(runs) != 1 || runs[0].ID != active { + t.Fatalf("active run must survive retention: %+v %v", runs, err) + } +} diff --git a/internal/server/review_contract_test.go b/internal/server/review_contract_test.go new file mode 100644 index 00000000..5aae5da8 --- /dev/null +++ b/internal/server/review_contract_test.go @@ -0,0 +1,214 @@ +package server + +import ( + "context" + "strconv" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// reviewLoopRunner submits a definition through the parser and opens a runner +// over it, returning the runner, its store, the pipeline id and a per-stage call +// counter. It reuses the shared submitRunner seam so every assertion below runs +// the real runLoop, not a copy of its setup or routing. +func reviewLoopRunner(t *testing.T, ctx context.Context, source string) (*runner, *pipeline.Store, int64, map[string]int) { + t.Helper() + r, store, id := submitRunner(t, ctx, source) + t.Cleanup(func() { store.Close() }) + return r, store, id, map[string]int{} +} + +func getRun(t *testing.T, store *pipeline.Store, ctx context.Context, id int64) *protocol.RunDetail { + t.Helper() + detail, err := store.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + return detail +} + +// TestReviewerAdvisoryDoesNotRevisitImplementer proves an advisory-only review +// outcome moves the run forward and never buys another worker visit, while the +// rejected edge and its limits stay declared. The advisory route is the single +// definition-level mechanism that keeps a non-blocking rejection from looping. +func TestReviewerAdvisoryDoesNotRevisitImplementer(t *testing.T) { + ctx := context.Background() + r, store, id, calls := reviewLoopRunner(t, ctx, reviewerLoopSource()) + r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + calls[def.Key]++ + if def.Key == "review" { + return stage.Result{Outcome: "advisory", Output: stage.Values{"findings": "non-blocking nits only"}}, nil + } + return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil + } + run := startRun(t, r, id, "advisory") + detail := getRun(t, store, ctx, run.ID) + if calls["worker"] != 1 { + t.Fatalf("advisory review bought %d worker visits, want 1", calls["worker"]) + } + if detail.Status != "succeeded" || len(detail.Steps) != 3 { + t.Fatalf("advisory route ended %s with %d steps, want succeeded with 3", detail.Status, len(detail.Steps)) + } + if detail.Steps[1].Outcome != "advisory" || detail.Steps[1].Target != "commit" { + t.Fatalf("advisory step = %+v", detail.Steps[1]) + } +} + +// TestReviewerMustFixRejectionLoopsAndStopsAtLimit proves a rejection that names a +// real defect still routes back to the implementer and still terminates at the +// reviewer's limits cap with the retained diagnostic, so the rejection path is +// preserved. It runs the exact published combination - the implementer's own +// blocked_retryable limits block plus the reviewer's rejected -> implementer edge - +// rather than the reviewer limits alone. +func TestReviewerMustFixRejectionLoopsAndStopsAtLimit(t *testing.T) { + ctx := context.Background() + r, store, id, calls := reviewLoopRunner(t, ctx, publishedReviewLoopSource(5)) + r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + calls[def.Key]++ + if def.Key == "reviewer" { + return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "unmet acceptance check at worker.go:12"}}, nil + } + return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil + } + run := startRun(t, r, id, "mustfix") + detail := getRun(t, store, ctx, run.ID) + if calls["reviewer"] != 3 || calls["implementer"] != 3 || len(detail.Steps) != 6 { + t.Fatalf("must-fix loop = calls %v, %d steps", calls, len(detail.Steps)) + } + last := detail.Steps[5] + if detail.Status != "failed" || last.Outcome != "rejected" || last.Control == nil || !last.Control.LimitReached || last.Target != protocol.FailureEnd { + t.Fatalf("limit did not terminate the must-fix loop: %+v", detail) + } + if last.Diagnostic != "Pipelines stopped reviewer after 3 visits on outcome rejected" { + t.Fatalf("limit diagnostic = %q", last.Diagnostic) + } +} + +// TestImplementerVisitCapMustNotPreemptReviewerLimit pins the regression the +// published shape had: an implementer limits cap below the reviewer's rejection +// budget stops the loop on the implementer's pre-execution visit cap, so the +// reviewer's third visit and its "Pipelines stopped reviewer..." diagnostic are +// unreachable. The cap must be at least the reviewer's budget (the published +// pipelines use 5) or the reviewer's rejected control is dead. +func TestImplementerVisitCapMustNotPreemptReviewerLimit(t *testing.T) { + ctx := context.Background() + r, store, id, calls := reviewLoopRunner(t, ctx, publishedReviewLoopSource(2)) + r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + calls[def.Key]++ + if def.Key == "reviewer" { + return stage.Result{Outcome: "rejected", Output: stage.Values{"findings": "unmet acceptance check"}}, nil + } + return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil + } + run := startRun(t, r, id, "preempted") + detail := getRun(t, store, ctx, run.ID) + if calls["reviewer"] != 2 || calls["implementer"] != 2 || len(detail.Steps) != 4 { + t.Fatalf("implementer cap did not preempt the reviewer: calls %v, %d steps", calls, len(detail.Steps)) + } + if detail.Status != "failed" || detail.Diagnostic != "stage implementer visit limit reached (2)" { + t.Fatalf("preempted limit diagnostic = %+v", detail) + } +} + +// TestBlockedRetryableRetriesWithinBudgetThenStops proves a transient block is +// retried only inside its declared visit bound and then fails closed, so a +// retryable implementer block cannot become a retry storm. +func TestBlockedRetryableRetriesWithinBudgetThenStops(t *testing.T) { + ctx := context.Background() + r, store, id, calls := reviewLoopRunner(t, ctx, retryableBlockSource(2)) + r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + calls[def.Key]++ + return stage.Result{Outcome: "blocked_retryable", Output: stage.Values{"blocked_reason": "device lock held"}}, nil + } + run := startRun(t, r, id, "retryable") + detail := getRun(t, store, ctx, run.ID) + if calls["worker"] != 2 || len(detail.Steps) != 2 { + t.Fatalf("retry bound = calls %v, %d steps, want 2", calls, len(detail.Steps)) + } + last := detail.Steps[1] + if detail.Status != "failed" || last.Outcome != "blocked_retryable" || last.Control == nil || !last.Control.LimitReached || last.Target != protocol.FailureEnd { + t.Fatalf("retryable block not bounded: %+v", last) + } + if last.Diagnostic != "Pipelines stopped worker after 2 visits on outcome blocked_retryable" { + t.Fatalf("retry bound diagnostic = %q", last.Diagnostic) + } +} + +// TestBlockedRetryableRetryCanRecover proves the bounded retry is useful: a stage +// that blocks once on a transient cause and succeeds on the retry continues to +// its normal next stage instead of failing the run. +func TestBlockedRetryableRetryCanRecover(t *testing.T) { + ctx := context.Background() + r, store, id, calls := reviewLoopRunner(t, ctx, retryableBlockSource(3)) + r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + calls[def.Key]++ + if def.Key == "worker" && calls["worker"] == 1 { + return stage.Result{Outcome: "blocked_retryable", Output: stage.Values{"blocked_reason": "held device lock"}}, nil + } + return stage.Result{Outcome: "success", Output: stage.Values{"summary": def.Key}}, nil + } + run := startRun(t, r, id, "recovered") + detail := getRun(t, store, ctx, run.ID) + if calls["worker"] != 2 || calls["review"] != 1 || detail.Status != "succeeded" { + t.Fatalf("retry did not recover: calls %v, status %s", calls, detail.Status) + } +} + +// TestTerminalBlockedFailsImmediately proves a terminal block (an acceptance +// criterion another attempt cannot change) stops on the first visit with no +// retry and a meaningful run diagnostic. +func TestTerminalBlockedFailsImmediately(t *testing.T) { + ctx := context.Background() + source := strings.Replace(pipeline.Template([]string{"worker", "review"}, true), "success -> review", "success -> review\nblocked -> #failure", 1) + r, store, id, calls := reviewLoopRunner(t, ctx, source) + r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + calls[def.Key]++ + return stage.Result{Outcome: "blocked", Output: stage.Values{"blocked_reason": "absent capability"}}, nil + } + run := startRun(t, r, id, "terminal") + detail := getRun(t, store, ctx, run.ID) + if calls["worker"] != 1 || len(detail.Steps) != 1 { + t.Fatalf("terminal block was retried: calls %v, %d steps", calls, len(detail.Steps)) + } + if detail.Status != "failed" || detail.Diagnostic != `stage worker returned outcome "blocked"` { + t.Fatalf("terminal block diagnostic = %+v", detail) + } + if detail.Steps[0].Target != protocol.FailureEnd { + t.Fatalf("terminal block target = %q", detail.Steps[0].Target) + } +} + +// reviewerLoopSource is the worker/review/commit shape the maintenance pipelines +// use: the reviewer keeps its bounded rejection edge and gains an advisory edge +// that continues the run without another worker visit. +func reviewerLoopSource() string { + source := pipeline.Template([]string{"worker", "review", "commit"}, true) + source = strings.Replace(source, "# stage review", "# stage review\n\n```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```", 1) + return strings.Replace(source, "success -> commit", "success -> commit\nadvisory -> commit\nrejected -> worker", 1) +} + +// publishedReviewLoopSource reconstructs the published maintenance/quick/focused +// control shape with the real stage keys: the implementer carries its own +// blocked_retryable limits block and self-edge, and the reviewer's rejected edge +// returns to that same implementer. implementerVisits is the implementer's +// max_visits, which also caps its re-entries from the reviewer, so it must be at +// least the reviewer's rejection budget. +func publishedReviewLoopSource(implementerVisits int) string { + source := pipeline.Template([]string{"implementer", "reviewer", "commit"}, true) + source = strings.Replace(source, "# stage implementer", "# stage implementer\n\n```limits\n{\"max_visits\":"+strconv.Itoa(implementerVisits)+",\"on\":[\"blocked_retryable\"]}\n```", 1) + source = strings.Replace(source, "success -> reviewer", "success -> reviewer\nblocked_retryable -> implementer\nblocked -> #failure", 1) + source = strings.Replace(source, "# stage reviewer", "# stage reviewer\n\n```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```", 1) + return strings.Replace(source, "success -> commit", "success -> commit\nadvisory -> commit\nrejected -> implementer", 1) +} + +// retryableBlockSource is a worker/review shape whose worker declares a bounded +// blocked_retryable self-edge, mirroring the published definitions. +func retryableBlockSource(maxVisits int) string { + source := pipeline.Template([]string{"worker", "review"}, true) + source = strings.Replace(source, "# stage worker", "# stage worker\n\n```limits\n{\"max_visits\":"+strconv.Itoa(maxVisits)+",\"on\":[\"blocked_retryable\"]}\n```", 1) + return strings.Replace(source, "success -> review", "success -> review\nblocked_retryable -> worker", 1) +} diff --git a/internal/tui/artifacts.go b/internal/tui/artifacts.go index bb3d3a5f..2088ba26 100644 --- a/internal/tui/artifacts.go +++ b/internal/tui/artifacts.go @@ -104,7 +104,7 @@ func artifactWidthsFor(visits []artifactVisit) artifactWidths { for _, artifact := range visit.artifacts { widths.name = min(artifactMaxName, max(widths.name, lipgloss.Width(ansi.Strip(artifact.Name)))) widths.typ = min(artifactMaxType, max(widths.typ, lipgloss.Width(ansi.Strip(artifact.ContentType)))) - widths.size = max(widths.size, lipgloss.Width(humanBytes(artifact.SizeBytes))) + widths.size = max(widths.size, lipgloss.Width(HumanBytes(artifact.SizeBytes))) } } return widths @@ -182,7 +182,7 @@ func artifactRowText(artifact protocol.StageArtifact, widths artifactWidths, wid if strings.TrimSpace(contentType) == "" { contentType = "unknown type" } - size := humanBytes(artifact.SizeBytes) + size := HumanBytes(artifact.SizeBytes) gap := strings.Repeat(" ", artifactColumnGap) nameCell := func(w int) string { return padRight(ansi.Truncate(name, max(1, w), "…"), w) } typeCell := padRight(ansi.Truncate(contentType, max(1, widths.typ), "…"), widths.typ) diff --git a/internal/tui/file_changes.go b/internal/tui/file_changes.go index 2319c6b2..c0d17e54 100644 --- a/internal/tui/file_changes.go +++ b/internal/tui/file_changes.go @@ -44,7 +44,9 @@ func artifactsSummary(step protocol.RunStep) string { return label } -func humanBytes(size int64) string { +// HumanBytes renders a byte size with binary units. It is the one byte +// formatter shared by the TUI and the CLI, so sizes never diverge. +func HumanBytes(size int64) string { const unit = 1024 if size < unit { return fmt.Sprintf("%d B", size)