From 31e89e872b6557c8babb16185e63ca8770e773b9 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 31 Oct 2026 22:11:14 +0000 Subject: [PATCH] Add run size_bytes and delete_runs to free disk space Every run summary carries size_bytes (artifact tree, server-created Pi session directories and an estimate of the run's database rows), cached in pipeline_runs.size_bytes, invalidated by triggers, measured when a run finishes and backfilled in the background. delete_runs removes terminal runs with every scoped row and server-owned file, refuses active ones with a reason, and pushes runs_removed to subscribers. pipelines runs rm is the CLI form; retention apply now shares the complete row deletion. --- docs/cli.md | 24 ++- internal/cli/app.go | 4 + internal/cli/commands.go | 3 +- internal/cli/runs_rm.go | 70 ++++++ internal/cli/runs_rm_test.go | 60 ++++++ internal/pipeline/retention.go | 38 +++- internal/pipeline/run_delete.go | 292 +++++++++++++++++++++++++ internal/pipeline/run_delete_test.go | 308 +++++++++++++++++++++++++++ internal/pipeline/run_size.go | 264 +++++++++++++++++++++++ internal/pipeline/runs.go | 5 +- internal/pipeline/store.go | 4 + internal/protocol/message.go | 3 + internal/protocol/runevents.go | 4 + internal/protocol/runs.go | 27 +++ internal/server/retry_test.go | 8 + internal/server/run_delete.go | 132 ++++++++++++ internal/server/run_delete_test.go | 126 +++++++++++ internal/server/runevents.go | 37 ++++ internal/server/runner.go | 11 + internal/server/websocket.go | 13 +- 20 files changed, 1421 insertions(+), 12 deletions(-) create mode 100644 internal/cli/runs_rm.go create mode 100644 internal/cli/runs_rm_test.go create mode 100644 internal/pipeline/run_delete.go create mode 100644 internal/pipeline/run_delete_test.go create mode 100644 internal/pipeline/run_size.go create mode 100644 internal/server/run_delete.go create mode 100644 internal/server/run_delete_test.go diff --git a/docs/cli.md b/docs/cli.md index 79e9ec61..9ede8d3e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -319,7 +319,29 @@ and matched field. `list_runs` also returns all-history `usage` aggregates. `attention_acks` reads and `ack_attention` adds to the operator's shared in-app acknowledgements (dismissed Needs-you entries and chat read watermarks; see [ntfy.md](ntfy.md#clearing-notifications-across-devices)); other -connections receive an `attention_acks_changed` push. See +connections receive an `attention_acks_changed` push. + +Every run summary (`list_runs` rows, `get_run`, subscribe snapshots and terminal +`run_event` frames) carries `size_bytes`: bytes on disk for the run (artifact tree +with turn documents and session exports, server-created Pi session directories, +plus an estimate of its database rows). It is cached in `pipeline_runs.size_bytes`, +invalidated by triggers when the run changes, measured on completion, and +backfilled in the background; 0 means not measured yet. `delete_runs` removes +finished runs to free space: + +```json +{"type":"delete_runs","id":"1","run_ids":[12,13,99]} +{"type":"runs_deleted","id":"1","runs_deleted":{"deleted":[12],"refused":[{"run_id":13,"reason":"run is running; cancel it or wait for it to finish"},{"run_id":99,"reason":"run not found"}],"freed_bytes":48213}} +{"type":"runs_removed","run_ids":[12]} +``` + +Only terminal runs (succeeded, failed, canceled, interrupted) are deleted; active +runs, unconfirmed cleanup, a running session export, a run an unfinished retry +depends on, and unknown ids are refused individually. Deletion removes every +row scoped to the run in one transaction plus `/` and owned Pi +session directories; user workspaces are never touched. Subscribed connections +whose scope covers the runs receive the `runs_removed` push. `pipelines runs rm +` is the CLI form. See `internal/protocol/message.go` for JSON envelopes. Mutating RPCs have real effects; they are not previews. TUI-only animation, scrolling, and transient client logs are presentation, not separate server capabilities. diff --git a/internal/cli/app.go b/internal/cli/app.go index 40a50689..cfe3897e 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -158,6 +158,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) { f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait after the run is admitted (0 waits until terminal); separate from per-request --timeout") case "runs": f.StringVar(&o.name, "pipeline", "", "Filter by exact pipeline name or ID") + f.BoolVar(&o.yes, "yes", false, "Confirm `runs rm` without an interactive prompt") case "search": f.StringVar(&o.kind, "type", "", "Narrow to one kind: "+strings.Join(protocol.SearchKinds, ", ")) f.IntVar(&o.limit, "limit", 0, "Maximum results (default 50, max 200)") @@ -751,6 +752,9 @@ func (a App) Run(ctx context.Context, args []string) error { } return nil case "list", "templates", "runs", "ping", "connection": + if command == "runs" && len(pos) > 0 && pos[0] == "rm" { + return a.removeRuns(o, pos[1:], call) + } if len(pos) > 0 { return fmt.Errorf("%s takes no positional arguments", command) } diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 4daef5f2..1487ae9e 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -19,7 +19,7 @@ var usage = map[string]string{ "templates": "templates [--json]", "template": "template [--source | --json]", "run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]", - "runs": "runs [--pipeline NAME] [--json]", + "runs": "runs [--pipeline NAME] [--json] | runs rm [run-id ...] [--yes] [--json]", "inspect": "inspect [--json]", "wait": "wait [run-id ...] [--timeout DURATION] [--json]", "cancel": "cancel [--json]", @@ -52,6 +52,7 @@ var notes = map[string]string{ "submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.", "run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).", "retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. --stage selects a recorded stage (use --visit for repeated stages). --input-file accepts a bounded JSON object, merged with replacement at the selected stage; '-' reads stdin. Original history and pinned identity are retained; --version selects an explicit published version of the same pipeline only. --wait-timeout leaves an active retry for later read-only inspection or reattachment with the original --request-key, rather than starting a duplicate retry.", + "runs": "Lists runs newest first; --json includes each run's size_bytes (bytes on disk: artifacts, session exports, server-created Pi session directories and an estimate of its database rows; 0 until first measured). `runs rm ` permanently deletes finished runs (succeeded, failed, canceled, interrupted) with every database row and server-owned file for them, after confirmation (--yes for scripts). Active runs, runs an unfinished retry depends on and unknown ids are refused individually with a reason, and the command then exits 1. User workspaces are never touched. Database pages are reused by SQLite; `retention apply` additionally VACUUMs to shrink the file.", "title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.", "wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).", "decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.", diff --git a/internal/cli/runs_rm.go b/internal/cli/runs_rm.go new file mode 100644 index 00000000..74b892e8 --- /dev/null +++ b/internal/cli/runs_rm.go @@ -0,0 +1,70 @@ +package cli + +import ( + "fmt" + "strconv" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// removeRuns implements `pipelines runs rm `: one delete_runs request +// for every named run. The server refuses active and unknown runs individually; +// any refusal makes the command exit non-zero after reporting the result. +func (a App) removeRuns(o *options, args []string, call callFunc) error { + if len(args) == 0 { + return fmt.Errorf("runs rm requires at least one run ID; use pipelines help runs") + } + ids := make([]int64, 0, len(args)) + for _, value := range args { + id, err := strconv.ParseInt(value, 10, 64) + if err != nil || id <= 0 { + return fmt.Errorf("run ID must be positive: %q", value) + } + ids = append(ids, id) + } + if err := a.confirmAction(fmt.Sprintf("Permanently delete %d run(s) with their artifacts and history?", len(ids)), o.yes); err != nil { + return err + } + r, err := call(protocol.Message{Type: protocol.DeleteRuns, RunIDs: ids}) + if err != nil { + return err + } + if r.RunsDeleted == nil { + return fmt.Errorf("server did not return a deletion result; update pipelines-server") + } + result := r.RunsDeleted + if o.json { + if err := a.output(result); err != nil { + return err + } + } else { + for _, id := range result.Deleted { + fmt.Fprintf(a.Out, "deleted\t%d\n", id) + } + for _, refusal := range result.Refused { + fmt.Fprintf(a.Out, "refused\t%d\t%s\n", refusal.RunID, refusal.Reason) + } + fmt.Fprintf(a.Out, "Freed %s\n", formatSize(result.FreedBytes)) + } + if len(result.Refused) > 0 { + return &ExitError{Code: 1} + } + return nil +} + +// formatSize renders a byte count with a binary unit for human output. +func formatSize(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + value, suffix := float64(bytes), "B" + for _, next := range []string{"KiB", "MiB", "GiB", "TiB"} { + if value < unit { + break + } + value /= unit + suffix = next + } + return fmt.Sprintf("%.1f %s", value, suffix) +} diff --git a/internal/cli/runs_rm_test.go b/internal/cli/runs_rm_test.go new file mode 100644 index 00000000..4b2ea810 --- /dev/null +++ b/internal/cli/runs_rm_test.go @@ -0,0 +1,60 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/wstest" +) + +func TestCLIRunsRmSendsDeleteRuns(t *testing.T) { + var got protocol.Message + endpoint := wstest.Server(t, func(request protocol.Message) protocol.Message { + got = request + return protocol.Message{Type: protocol.RunsDeleted, RunsDeleted: &protocol.RunsDeletedResult{ + Deleted: []int64{3}, + Refused: []protocol.RunDeleteRefusal{{RunID: 4, Reason: "run is running; cancel it or wait for it to finish"}}, + FreedBytes: 2048, + }} + }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var out, errOut bytes.Buffer + app := App{Out: &out, Err: &errOut, In: strings.NewReader("")} + err := app.Run(ctx, []string{"--url", endpoint, "runs", "rm", "3", "4", "--yes", "--json"}) + var exit *ExitError + if !errors.As(err, &exit) || exit.Code != 1 { + t.Fatalf("err = %v, want exit 1 for a refusal", err) + } + if got.Type != protocol.DeleteRuns || fmt.Sprint(got.RunIDs) != "[3 4]" { + t.Fatalf("wire request = %+v", got) + } + var decoded protocol.RunsDeletedResult + if err := json.Unmarshal(out.Bytes(), &decoded); err != nil { + t.Fatalf("stdout is not JSON: %v\n%s", err, out.String()) + } + if fmt.Sprint(decoded.Deleted) != "[3]" || decoded.FreedBytes != 2048 || len(decoded.Refused) != 1 { + t.Fatalf("decoded = %+v", decoded) + } + + // Without --yes a declined prompt sends nothing. + got = protocol.Message{} + declined := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}, In: strings.NewReader("n\n")} + if err := declined.Run(ctx, []string{"--url", endpoint, "runs", "rm", "3"}); err == nil { + t.Fatal("declined deletion succeeded") + } + if got.Type != "" { + t.Fatalf("declined deletion sent %+v", got) + } + if err := declined.Run(ctx, []string{"--url", endpoint, "runs", "rm", "x", "--yes"}); err == nil { + t.Fatal("non-numeric id accepted") + } +} diff --git a/internal/pipeline/retention.go b/internal/pipeline/retention.go index bbbb0dab..fa75c240 100644 --- a/internal/pipeline/retention.go +++ b/internal/pipeline/retention.go @@ -614,11 +614,21 @@ func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy, return nil } -// deleteRetentionRows removes every table row scoped to the selected runs, in -// foreign-key order, batched at 500 ids per statement in the caller's single -// transaction: step-scoped evidence first, then run_steps, then retry edges, -// then the runs. +// deleteRetentionRows removes every table row scoped to the selected runs. It +// is the same removal delete_runs performs (deleteRunRows). func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error { + return deleteRunRows(ctx, tx, ids) +} + +// deleteRunRows removes every table row scoped to the given runs, in +// foreign-key order, batched at 500 ids per statement in the caller's single +// transaction: step-scoped evidence first, then run_steps, then retry +// provenance, then the runs. Tables with ON DELETE CASCADE are deleted +// explicitly too, so the removal does not depend on PRAGMA foreign_keys. A table +// missing from an older database is skipped. Schedule event history +// (saved_schedule_events, schedule_events) is audit history of the schedule, +// not of the run, and is kept. +func deleteRunRows(ctx context.Context, tx *sql.Tx, ids []int64) error { for _, table := range []struct{ name, column string }{ {"step_notifications", "run_id"}, {"run_notifications", "run_id"}, @@ -628,10 +638,15 @@ func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error { {"step_verdicts", "run_id"}, {"step_session_exports", "run_id"}, {"step_usage", "run_id"}, + {"session_export_repairs", "run_id"}, + {"session_export_jobs", "run_id"}, + {"session_export_gates", "run_id"}, {"run_session_owners", "run_id"}, + {"attention_dismissals", "run_id"}, + {"run_schedule_origin", "run_id"}, {"run_steps", "run_id"}, } { - if err := deleteRetentionBatch(ctx, tx, table.name, table.column, ids); err != nil { + if err := deleteRetentionBatch(ctx, tx, table.name, table.column, ids); err != nil && !missingTable(err) { return err } } @@ -640,13 +655,22 @@ func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error { for start := 0; start < len(ids); start += retentionDeleteBatch { batch := ids[start:min(start+retentionDeleteBatch, len(ids))] placeholders, args := retentionBindings(batch, 2) - if _, err := tx.ExecContext(ctx, "DELETE FROM run_retries WHERE run_id IN ("+placeholders+") OR original_run_id IN ("+placeholders+")", args...); err != nil { - return err + for _, statement := range []string{ + "DELETE FROM run_retries WHERE run_id IN (" + placeholders + ") OR original_run_id IN (" + placeholders + ")", + "DELETE FROM retry_checkpoints WHERE run_id IN (" + placeholders + ") OR source_run_id IN (" + placeholders + ")", + } { + if _, err := tx.ExecContext(ctx, statement, args...); err != nil && !missingTable(err) { + return err + } } } return deleteRetentionBatch(ctx, tx, "pipeline_runs", "id", ids) } +func missingTable(err error) bool { + return err != nil && strings.Contains(err.Error(), "no such table") +} + // retentionDeleteBatch bounds each IN clause so a large removal does not build // an unbounded statement. const retentionDeleteBatch = 500 diff --git a/internal/pipeline/run_delete.go b/internal/pipeline/run_delete.go new file mode 100644 index 00000000..95b6fd12 --- /dev/null +++ b/internal/pipeline/run_delete.go @@ -0,0 +1,292 @@ +package pipeline + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// MaxDeleteRuns bounds one delete_runs request. +const MaxDeleteRuns = 1000 + +// terminalRunStatuses are the only run statuses delete_runs removes. Anything +// else (queued, running, waiting, or a status this build does not know) is +// refused. +var terminalRunStatuses = map[string]bool{"succeeded": true, "failed": true, "canceled": true, "interrupted": true} + +// DeleteRunsResult is what DeleteRuns removed. Workspaces maps each deleted run +// to its workspace so the server can scope the runs_removed push. +type DeleteRunsResult struct { + protocol.RunsDeletedResult + Workspaces map[int64]string +} + +// DeleteRuns removes each requested terminal run and everything the server +// stores for it: every database row scoped to it (see deleteRunRows), its +// artifact tree /, and Pi session directories the server +// created for its stages. User workspaces and repositories are never touched. +// +// Eligibility is checked inside the write transaction. busy, when non-nil, lets +// the caller veto a run it knows to be executing; its non-empty reason refuses +// the run. Unknown ids, invalid ids and duplicates are refused individually; an +// I/O or database failure aborts the whole request with nothing removed (the +// artifact trees are staged by rename and restored on failure). +func (s *Store) DeleteRuns(ctx context.Context, ids []int64, artifactsRoot string, busy func(int64) string) (*DeleteRunsResult, error) { + if len(ids) == 0 { + return nil, fmt.Errorf("run_ids is required") + } + if len(ids) > MaxDeleteRuns { + return nil, fmt.Errorf("at most %d runs can be deleted at once", MaxDeleteRuns) + } + result := &DeleteRunsResult{ + RunsDeletedResult: protocol.RunsDeletedResult{Deleted: []int64{}, Refused: []protocol.RunDeleteRefusal{}}, + Workspaces: map[int64]string{}, + } + refuse := func(id int64, reason string) { + result.Refused = append(result.Refused, protocol.RunDeleteRefusal{RunID: id, Reason: reason}) + } + seen := map[int64]bool{} + var candidates []int64 + for _, id := range ids { + switch { + case id <= 0: + refuse(id, "invalid run id") + case seen[id]: + // A repeated id is the same request; report it once. + default: + seen[id] = true + candidates = append(candidates, id) + } + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + + var deletable []int64 + sizes := map[int64]int64{} + for _, id := range candidates { + reason, workspace, err := deleteRefusal(ctx, tx, id) + if err != nil { + return nil, err + } + if reason == "" && busy != nil { + reason = busy(id) + } + if reason != "" { + refuse(id, reason) + continue + } + size, err := computeRunSize(ctx, tx, artifactsRoot, id) + if err != nil { + return nil, err + } + sizes[id] = size + result.Workspaces[id] = workspace + deletable = append(deletable, id) + } + if len(deletable) == 0 { + return result, nil + } + + piDirs, err := deletablePiDirs(ctx, tx, deletable) + if err != nil { + return nil, err + } + stage, err := stageRunArtifacts(artifactsRoot, deletable) + if err != nil { + return nil, err + } + if err := deleteRunRows(ctx, tx, deletable); err != nil { + return nil, errors.Join(fmt.Errorf("delete run history: %w", err), stage.restore()) + } + if err := tx.Commit(); err != nil { + return nil, errors.Join(fmt.Errorf("commit run deletion: %w", err), stage.restore()) + } + committed = true + + // The rows are gone; file cleanup failures cannot resurrect the runs, so + // they are reported only in the freed byte count, never as an error. + for _, id := range deletable { + result.Deleted = append(result.Deleted, id) + result.FreedBytes += sizes[id] + } + if stage.dir != "" { + if err := os.RemoveAll(stage.dir); err != nil { + _, remaining, _ := retentionWalkSize(stage.dir) + result.FreedBytes = max(result.FreedBytes-remaining, 0) + } + } + for _, dir := range piDirs { + if err := os.RemoveAll(dir); err != nil { + _, remaining, _ := retentionWalkSize(dir) + result.FreedBytes = max(result.FreedBytes-remaining, 0) + } + } + return result, nil +} + +// deleteRefusal returns why a run must not be deleted ("" when it may be), and +// its workspace. It refuses unknown runs, every non-terminal status, runs with +// unconfirmed cleanup, active or waiting visits, a running session export, and +// runs an unfinished retry still depends on. +func deleteRefusal(ctx context.Context, q sizeQuerier, id int64) (string, string, error) { + var status, workspace string + var cleanupUnconfirmed bool + var activeSteps, runningExports, dependents int + err := q.QueryRowContext(ctx, `SELECT r.status,r.workspace,r.cleanup_unconfirmed, +(SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND (s.status IN ('running','waiting') OR s.agent_active<>0)), +(SELECT COUNT(*) FROM session_export_jobs j WHERE j.run_id=r.id AND j.status='running'), +(SELECT COUNT(*) FROM pipeline_runs d WHERE d.id<>r.id AND d.status NOT IN ('succeeded','failed','canceled','interrupted') + AND (d.id IN (SELECT run_id FROM run_retries WHERE original_run_id=r.id) OR d.id IN (SELECT run_id FROM retry_checkpoints WHERE source_run_id=r.id))) +FROM pipeline_runs r WHERE r.id=?`, id).Scan(&status, &workspace, &cleanupUnconfirmed, &activeSteps, &runningExports, &dependents) + if err == sql.ErrNoRows { + return "run not found", "", nil + } + if err != nil { + return "", "", err + } + switch { + case !terminalRunStatuses[status]: + return fmt.Sprintf("run is %s; cancel it or wait for it to finish", status), workspace, nil + case cleanupUnconfirmed: + return "run cleanup is not confirmed yet", workspace, nil + case activeSteps > 0: + return "a stage of this run is still active or waiting", workspace, nil + case runningExports > 0: + return "a session export for this run is in progress", workspace, nil + case dependents > 0: + return "an unfinished retry depends on this run", workspace, nil + } + return "", workspace, nil +} + +// deletablePiDirs lists the owned Pi session directories of the runs being +// deleted that no surviving run also references. +func deletablePiDirs(ctx context.Context, q sizeQuerier, ids []int64) ([]string, error) { + deleting := map[int64]bool{} + for _, id := range ids { + deleting[id] = true + } + var dirs []string + for _, id := range ids { + found, err := runPiSessionDirs(ctx, q, id) + if err != nil { + return nil, err + } + for _, dir := range found { + rows, err := q.QueryContext(ctx, `SELECT DISTINCT run_id FROM run_session_owners WHERE reference=?`, dir) + if err != nil { + return nil, err + } + shared := false + for rows.Next() { + var owner int64 + if err := rows.Scan(&owner); err != nil { + rows.Close() + return nil, err + } + if !deleting[owner] { + shared = true + } + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + if !shared { + dirs = append(dirs, dir) + } + } + } + return dirs, nil +} + +// stagedArtifacts is a set of run artifact trees renamed into one staging +// directory under the artifacts root, so the database removal and the file +// removal commit or roll back together. +type stagedArtifacts struct { + dir string + staged []string +} + +// stageRunArtifacts renames / for each run into a fresh staging +// directory. Only the exact numeric entry is staged; a symlink is moved (the +// link, never its target) and a regular file there aborts with an error. +func stageRunArtifacts(root string, ids []int64) (*stagedArtifacts, error) { + stage := &stagedArtifacts{} + if root == "" { + return stage, nil + } + info, err := os.Stat(root) + if os.IsNotExist(err) { + return stage, nil + } + if err != nil { + return nil, fmt.Errorf("inspect artifacts root %s: %w", root, err) + } + if !info.IsDir() { + return stage, nil + } + for _, id := range ids { + original := filepath.Join(root, strconv.FormatInt(id, 10)) + entry, err := os.Lstat(original) + if os.IsNotExist(err) { + continue + } + if err != nil { + return nil, errors.Join(fmt.Errorf("inspect artifacts for run %d: %w", id, err), stage.restore()) + } + if !entry.IsDir() && entry.Mode()&os.ModeSymlink == 0 { + return nil, errors.Join(fmt.Errorf("refusing to remove run %d: %s is a regular file, not a run artifact directory", id, original), stage.restore()) + } + if stage.dir == "" { + stage.dir = filepath.Join(root, fmt.Sprintf(".delete-staging-%d-%d", time.Now().UnixNano(), os.Getpid())) + if err := os.Mkdir(stage.dir, 0o700); err != nil { + stage.dir = "" + return nil, fmt.Errorf("create delete staging directory: %w", err) + } + } + if err := os.Rename(original, filepath.Join(stage.dir, filepath.Base(original))); err != nil { + return nil, errors.Join(fmt.Errorf("stage artifacts for run %d: %w", id, err), stage.restore()) + } + stage.staged = append(stage.staged, original) + } + return stage, nil +} + +// restore moves every staged tree back and removes the staging directory. +func (s *stagedArtifacts) restore() error { + var problems []string + for _, original := range s.staged { + if err := os.Rename(filepath.Join(s.dir, filepath.Base(original)), original); err != nil { + problems = append(problems, fmt.Sprintf("restore %s: %v", original, err)) + } + } + s.staged = nil + if s.dir != "" { + if err := os.Remove(s.dir); err != nil && !os.IsNotExist(err) { + problems = append(problems, fmt.Sprintf("remove staging directory %s: %v", s.dir, err)) + } + } + if len(problems) > 0 { + return fmt.Errorf("restore artifacts: %s", strings.Join(problems, "; ")) + } + return nil +} diff --git a/internal/pipeline/run_delete_test.go b/internal/pipeline/run_delete_test.go new file mode 100644 index 00000000..d6092a25 --- /dev/null +++ b/internal/pipeline/run_delete_test.go @@ -0,0 +1,308 @@ +package pipeline + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" +) + +func (f *retentionFixture) sizeColumn(t *testing.T, id int64) int64 { + t.Helper() + var size int64 + if err := f.store.db.QueryRow("SELECT size_bytes FROM pipeline_runs WHERE id=?", id).Scan(&size); err != nil { + t.Fatal(err) + } + return size +} + +func (f *retentionFixture) listedSize(t *testing.T, id int64) int64 { + t.Helper() + runs, err := f.store.ListRuns(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, run := range runs { + if run.ID == id { + return run.SizeBytes + } + } + t.Fatalf("run %d not listed", id) + return 0 +} + +// usePiSessionsBase points Pi session ownership at a temporary directory so a +// test never reads or removes the real user cache. +func usePiSessionsBase(t *testing.T) string { + t.Helper() + base := filepath.Join(t.TempDir(), "pi-sessions") + if err := os.MkdirAll(base, 0o700); err != nil { + t.Fatal(err) + } + previous := piSessionsBase + piSessionsBase = func() (string, error) { return base, nil } + t.Cleanup(func() { piSessionsBase = previous }) + return base +} + +func TestRunSizeComputedCachedAndInvalidated(t *testing.T) { + ctx := context.Background() + base := usePiSessionsBase(t) + f := newRetentionFixture(t) + id := f.run(t, "sized", 0) + f.step(t, id, 1, "completed", false) + f.artifact(t, id, 1, "report.md", 10_000) + piDir := filepath.Join(base, "stage-1") + if err := os.MkdirAll(piDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(piDir, "rpc.jsonl"), make([]byte, 3_000), 0o600); err != nil { + t.Fatal(err) + } + if _, err := f.store.db.Exec("INSERT INTO run_session_owners(run_id,sequence,backend,reference) VALUES(?,1,'pi',?)", id, piDir); err != nil { + t.Fatal(err) + } + if got := f.sizeColumn(t, id); got != -1 { + t.Fatalf("new run size = %d, want -1 (unmeasured)", got) + } + if got := f.listedSize(t, id); got != 0 { + t.Fatalf("unmeasured listed size = %d, want 0", got) + } + measured, err := f.store.RefreshRunSizes(ctx, f.root, 0) + if err != nil { + t.Fatal(err) + } + if len(measured) != 1 || measured[0].ID != id { + t.Fatalf("measured = %+v", measured) + } + size := measured[0].SizeBytes + if size < 13_000 || size > 20_000 { + t.Fatalf("size = %d, want artifacts + pi dir + modest DB estimate", size) + } + if got := f.listedSize(t, id); got != size { + t.Fatalf("listed size = %d, want cached %d", got, size) + } + again, err := f.store.RefreshRunSizes(ctx, f.root, 0) + if err != nil || len(again) != 0 { + t.Fatalf("cached run re-measured: %+v %v", again, err) + } + // A visit change invalidates the cache; the refreshed size includes it. + if _, err := f.store.db.Exec("UPDATE run_steps SET transcript=? WHERE run_id=?", string(make([]byte, 5_000)), id); err != nil { + t.Fatal(err) + } + if got := f.sizeColumn(t, id); got != -1 { + t.Fatalf("size after step update = %d, want -1", got) + } + grown, err := f.store.RefreshRunSize(ctx, f.root, id) + if err != nil { + t.Fatal(err) + } + if grown < size+5_000 { + t.Fatalf("size after transcript = %d, want >= %d", grown, size+5_000) + } + // A run status change invalidates it too. + f.setStatus(t, id, "failed") + if got := f.sizeColumn(t, id); got != -1 { + t.Fatalf("size after status change = %d, want -1", got) + } +} + +func TestRunSizeNotStoredWhenRunChangedDuringMeasurement(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + id := f.run(t, "racy", 0) + var gen int64 + if err := f.store.db.QueryRow("SELECT size_gen FROM pipeline_runs WHERE id=?", id).Scan(&gen); err != nil { + t.Fatal(err) + } + f.step(t, id, 1, "completed", false) // a change after the generation was read + if _, err := f.store.db.ExecContext(ctx, "UPDATE pipeline_runs SET size_bytes=? WHERE id=? AND size_gen=?", 1, id, gen); err != nil { + t.Fatal(err) + } + if got := f.sizeColumn(t, id); got != -1 { + t.Fatalf("stale measurement stored: %d", got) + } +} + +// runTables lists every table delete_runs must clear, with its run column. +var runTables = map[string]string{ + "pipeline_runs": "id", "run_steps": "run_id", "step_notifications": "run_id", "step_file_changes": "run_id", + "step_control": "run_id", "step_artifacts": "run_id", "step_verdicts": "run_id", "step_session_exports": "run_id", + "step_usage": "run_id", "run_session_owners": "run_id", "session_export_repairs": "run_id", + "session_export_jobs": "run_id", "session_export_gates": "run_id", "run_notifications": "run_id", + "attention_dismissals": "run_id", "run_schedule_origin": "run_id", "run_retries": "run_id", + "retry_checkpoints": "run_id", +} + +func (f *retentionFixture) seedEverything(t *testing.T, id int64, piDir string) { + t.Helper() + f.step(t, id, 1, "completed", false) + statements := []string{ + "INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,1,'[]')", + "INSERT INTO step_file_changes(run_id,sequence,changes_json) VALUES(?,1,'[]')", + "INSERT INTO step_control(run_id,sequence,evidence_json) VALUES(?,1,'{}')", + "INSERT INTO step_artifacts(run_id,sequence,artifacts_json) VALUES(?,1,'{}')", + "INSERT INTO step_verdicts(run_id,sequence,verdict_json) VALUES(?,1,'{}')", + "INSERT INTO step_session_exports(run_id,sequence,export_json) VALUES(?,1,'{}')", + "INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,1,'{}')", + "INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,1,'{}')", + "INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at,status) VALUES(?,1,'s','pi','/tmp','r',0,'complete')", + "INSERT INTO session_export_gates(run_id,sequence) VALUES(?,1)", + "INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click) VALUES(?,1,'finished','system','t','b','c')", + "INSERT INTO attention_dismissals(key,run_id,dismissed_at) VALUES('run:'||?1,?1,0)", + "INSERT INTO run_schedule_origin(run_id,schedule_id,name) VALUES(?,1,'nightly')", + } + for _, statement := range statements { + if _, err := f.store.db.Exec(statement, id); err != nil { + t.Fatalf("%s: %v", statement, err) + } + } + if _, err := f.store.db.Exec("INSERT INTO run_session_owners(run_id,sequence,backend,reference) VALUES(?,1,'pi',?)", id, piDir); err != nil { + t.Fatal(err) + } + f.artifact(t, id, 1, "out.md", 4_096) + if err := os.MkdirAll(piDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(piDir, "rpc.jsonl"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestDeleteRunsRemovesRowsAndFiles(t *testing.T) { + ctx := context.Background() + base := usePiSessionsBase(t) + f := newRetentionFixture(t) + doomed := f.run(t, "doomed", 0) + kept := f.run(t, "kept", 0) + active := f.run(t, "active", 0) + f.setStatus(t, active, "running") + f.seedEverything(t, doomed, filepath.Join(base, "stage-doomed")) + f.seedEverything(t, kept, filepath.Join(base, "stage-kept")) + // kept is a finished retry of doomed: deleting the original drops the + // provenance edge and checkpoint but keeps the retry itself. + f.retry(t, kept, doomed) + if _, err := f.store.db.Exec("INSERT INTO retry_checkpoints(run_id,source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash) VALUES(?,?,'resume',1,'{}',1,'h')", kept, doomed); err != nil { + t.Fatal(err) + } + // A workspace-like directory outside the data dir must survive. + workspace := filepath.Join(t.TempDir(), "repo") + if err := os.MkdirAll(workspace, 0o700); err != nil { + t.Fatal(err) + } + if _, err := f.store.db.Exec("UPDATE pipeline_runs SET workspace=? WHERE id=?", workspace, doomed); err != nil { + t.Fatal(err) + } + expected, err := f.store.RefreshRunSize(ctx, f.root, doomed) + if err != nil { + t.Fatal(err) + } + + result, err := f.store.DeleteRuns(ctx, []int64{doomed, doomed, active, 99999, -3}, f.root, nil) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(result.Deleted) != fmt.Sprint([]int64{doomed}) { + t.Fatalf("deleted = %v", result.Deleted) + } + reasons := map[int64]string{} + for _, refusal := range result.Refused { + reasons[refusal.RunID] = refusal.Reason + } + if len(result.Refused) != 3 || reasons[active] == "" || reasons[99999] != "run not found" || reasons[-3] != "invalid run id" { + t.Fatalf("refused = %+v", result.Refused) + } + if result.FreedBytes != expected { + t.Fatalf("freed = %d, want measured size %d", result.FreedBytes, expected) + } + if result.Workspaces[doomed] != workspace { + t.Fatalf("workspace = %q", result.Workspaces[doomed]) + } + for table, column := range runTables { + var count int + if err := f.store.db.QueryRow("SELECT COUNT(*) FROM "+table+" WHERE "+column+"=?", doomed).Scan(&count); err != nil { + t.Fatalf("%s: %v", table, err) + } + if count != 0 { + t.Fatalf("%s still holds %d rows for deleted run", table, count) + } + } + for _, check := range []string{"SELECT COUNT(*) FROM run_retries WHERE original_run_id=?", "SELECT COUNT(*) FROM retry_checkpoints WHERE source_run_id=?"} { + var count int + if err := f.store.db.QueryRow(check, doomed).Scan(&count); err != nil || count != 0 { + t.Fatalf("%s = %d %v", check, count, err) + } + } + if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(doomed))); !os.IsNotExist(err) { + t.Fatalf("artifacts of deleted run remain: %v", err) + } + if _, err := os.Stat(filepath.Join(base, "stage-doomed")); !os.IsNotExist(err) { + t.Fatalf("pi session dir of deleted run remains: %v", err) + } + if entries, _ := os.ReadDir(f.root); len(entries) != 1 { + t.Fatalf("artifacts root entries = %v, want only the kept run (no staging leftovers)", entries) + } + for _, path := range []string{workspace, filepath.Join(f.root, fmt.Sprint(kept)), filepath.Join(base, "stage-kept")} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("%s must survive: %v", path, err) + } + } + for table, column := range runTables { + if table == "run_retries" || table == "retry_checkpoints" { + continue + } + var count int + if err := f.store.db.QueryRow("SELECT COUNT(*) FROM "+table+" WHERE "+column+"=?", kept).Scan(&count); err != nil || count == 0 { + t.Fatalf("%s lost the kept run's rows: %d %v", table, count, err) + } + } + violations, err := f.store.db.Query("PRAGMA foreign_key_check") + if err != nil { + t.Fatal(err) + } + if violations.Next() { + violations.Close() + t.Fatal("foreign key violation after delete") + } + violations.Close() +} + +func TestDeleteRunsRefusesActiveAndDependedOnRuns(t *testing.T) { + ctx := context.Background() + f := newRetentionFixture(t) + queued := f.run(t, "queued", 0) + f.setStatus(t, queued, "queued") + waiting := f.run(t, "waiting", 0) + f.setStatus(t, waiting, "waiting") + agent := f.run(t, "agent", 0) + f.step(t, agent, 1, "failed", true) + original := f.run(t, "original", 0) + retry := f.run(t, "retry", 0) + f.setStatus(t, retry, "running") + f.retry(t, retry, original) + busy := f.run(t, "busy", 0) + f.artifact(t, busy, 1, "keep.md", 10) + + result, err := f.store.DeleteRuns(ctx, []int64{queued, waiting, agent, original, busy}, f.root, func(id int64) string { + if id == busy { + return "run is executing" + } + return "" + }) + if err != nil { + t.Fatal(err) + } + if len(result.Deleted) != 0 || len(result.Refused) != 5 || result.FreedBytes != 0 { + t.Fatalf("result = %+v", result.RunsDeletedResult) + } + if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(busy))); err != nil { + t.Fatalf("refused run lost artifacts: %v", err) + } + if got := f.count(t, "pipeline_runs"); got != 6 { + t.Fatalf("runs = %d, want 6", got) + } + if _, err := f.store.DeleteRuns(ctx, nil, f.root, nil); err == nil { + t.Fatal("empty run_ids accepted") + } +} diff --git a/internal/pipeline/run_size.go b/internal/pipeline/run_size.go new file mode 100644 index 00000000..26b588d7 --- /dev/null +++ b/internal/pipeline/run_size.go @@ -0,0 +1,264 @@ +package pipeline + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Run size accounting. pipeline_runs.size_bytes caches the bytes on disk a run +// accounts for: its artifact tree (/, which holds turn +// documents, inputs, outputs and session exports), any Pi session directory the +// server created for its stages, and an estimate of its database rows (the byte +// length of every large text column plus a small per-row overhead). +// +// -1 means "not computed or stale". Triggers mark a run stale whenever its row, +// a visit or visit evidence changes, and bump size_gen so a size computed +// concurrently with a change is never stored as current (see storeRunSize). +// Existing databases start with every run at -1 and are backfilled lazily. + +// runSizeTriggers are the tables whose writes change a run's footprint. Each +// gets an insert and an update trigger that invalidates the owning run. +var runSizeTriggers = []string{ + "run_steps", "step_notifications", "step_file_changes", "step_control", "step_artifacts", + "step_verdicts", "step_session_exports", "step_usage", "run_session_owners", + "session_export_repairs", "run_notifications", "retry_checkpoints", +} + +// initRunSizes adds the cache columns and invalidation triggers. Every step is +// idempotent so a repeated Open and an upgraded database both open cleanly. +func (s *Store) initRunSizes() error { + if err := s.addColumnIfMissing("pipeline_runs", "size_bytes", "INTEGER NOT NULL DEFAULT -1"); err != nil { + return err + } + if err := s.addColumnIfMissing("pipeline_runs", "size_gen", "INTEGER NOT NULL DEFAULT 0"); err != nil { + return err + } + statements := []string{ + // The nested UPDATE touches only size columns, which are not in the OF + // list, so this trigger cannot re-fire itself. + `CREATE TRIGGER IF NOT EXISTS run_size_stale_run AFTER UPDATE OF status,outcome,output_json,input_json,diagnostic,finished_at,title ON pipeline_runs +BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.id; END`, + } + for _, table := range runSizeTriggers { + for _, event := range []string{"INSERT", "UPDATE"} { + statements = append(statements, fmt.Sprintf(`CREATE TRIGGER IF NOT EXISTS run_size_stale_%s_%s AFTER %s ON %s +BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.run_id; END`, table, strings.ToLower(event), event, table)) + } + } + for _, statement := range statements { + if _, err := s.db.Exec(statement); err != nil { + return err + } + } + return nil +} + +// runSizeRowOverhead approximates SQLite's per-row cost (record header, rowid, +// index entries) beyond the measured column bytes. +const runSizeRowOverhead = 64 + +// runDBBytesQuery estimates one run's database footprint: the byte length of +// every variable-size column in every table scoped to the run, plus a fixed +// per-row overhead. Each subquery uses the table's run_id primary-key prefix or +// index, so the cost is bounded by the run's own rows. +var runDBBytesQuery = func() string { + type part struct { + table, where string + columns []string + } + parts := []part{ + {"pipeline_runs", "id=?1", []string{"request_key", "workspace", "title", "title_source", "title_model", "model", "stage_models", "write_paths", "input_json", "output_json", "outcome", "current_stage", "diagnostic", "output_format"}}, + {"run_steps", "run_id=?1", []string{"stage_key", "outcome", "target", "input_json", "output_json", "diagnostic", "provider", "model", "session_id", "transcript"}}, + {"step_notifications", "run_id=?1", []string{"receipts_json"}}, + {"step_file_changes", "run_id=?1", []string{"changes_json"}}, + {"step_control", "run_id=?1", []string{"evidence_json"}}, + {"step_artifacts", "run_id=?1", []string{"artifacts_json"}}, + {"step_verdicts", "run_id=?1", []string{"verdict_json"}}, + {"step_session_exports", "run_id=?1", []string{"export_json"}}, + {"step_usage", "run_id=?1", []string{"usage_json"}}, + {"run_session_owners", "run_id=?1", []string{"backend", "reference"}}, + {"session_export_repairs", "run_id=?1", []string{"export_json"}}, + {"session_export_jobs", "run_id=?1", []string{"session_id", "backend", "directory", "reference", "token", "diagnostic"}}, + {"run_notifications", "run_id=?1", []string{"event", "channel", "title", "body", "click", "diagnostic", "message_id"}}, + {"retry_checkpoints", "run_id=?1", []string{"mode", "handoff_json", "source_hash"}}, + {"run_schedule_origin", "run_id=?1", []string{"name"}}, + } + terms := make([]string, 0, len(parts)) + for _, p := range parts { + lengths := make([]string, 0, len(p.columns)+1) + for _, column := range p.columns { + lengths = append(lengths, "COALESCE(length(CAST("+column+" AS BLOB)),0)") + } + lengths = append(lengths, fmt.Sprint(runSizeRowOverhead)) + terms = append(terms, "COALESCE((SELECT SUM("+strings.Join(lengths, "+")+") FROM "+p.table+" WHERE "+p.where+"),0)") + } + return "SELECT " + strings.Join(terms, "+") +}() + +// sizeQuerier is the read surface shared by *sql.DB and *sql.Tx, so a size is +// computed identically by the cache refresh and inside the delete transaction. +type sizeQuerier interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// piSessionsBase is the directory the Pi stage client creates its per-visit +// session directories under (see internal/stage/pi.go). It is a variable so +// tests can point it at a temporary directory. +var piSessionsBase = func() (string, error) { + cache, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(cache, "pipelines", "pi-sessions"), nil +} + +// ownedPiSessionDir reports whether reference is a server-created Pi session +// directory: an absolute, clean path directly under piSessionsBase named +// stage-*, that is a real directory rather than a symlink. Anything else - a +// user path, a workspace, a symlink - is never sized or removed. +func ownedPiSessionDir(reference string) bool { + base, err := piSessionsBase() + if err != nil || base == "" || reference == "" || !filepath.IsAbs(reference) || filepath.Clean(reference) != reference { + return false + } + if filepath.Dir(reference) != base || !strings.HasPrefix(filepath.Base(reference), "stage-") { + return false + } + info, err := os.Lstat(reference) + return err == nil && info.IsDir() +} + +// runPiSessionDirs lists the owned Pi session directories recorded for a run. +func runPiSessionDirs(ctx context.Context, q sizeQuerier, id int64) ([]string, error) { + rows, err := q.QueryContext(ctx, `SELECT DISTINCT reference FROM run_session_owners WHERE run_id=? AND backend='pi' AND reference<>''`, id) + if err != nil { + return nil, err + } + defer rows.Close() + var dirs []string + for rows.Next() { + var reference string + if err := rows.Scan(&reference); err != nil { + return nil, err + } + if ownedPiSessionDir(reference) { + dirs = append(dirs, reference) + } + } + return dirs, rows.Err() +} + +// errRunNotFound marks a size or delete request for a run id with no row. +var errRunNotFound = fmt.Errorf("run not found") + +// computeRunSize measures one run now: artifact tree, owned Pi session +// directories and the database estimate. Unreadable files are skipped rather +// than failing the measurement; the size is a best-effort lower bound then. +func computeRunSize(ctx context.Context, q sizeQuerier, artifactsRoot string, id int64) (int64, error) { + var exists bool + if err := q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pipeline_runs WHERE id=?)`, id).Scan(&exists); err != nil { + return 0, err + } + if !exists { + return 0, errRunNotFound + } + var dbBytes int64 + if err := q.QueryRowContext(ctx, runDBBytesQuery, id).Scan(&dbBytes); err != nil { + return 0, err + } + _, artifactBytes, _ := retentionArtifactsSize(artifactsRoot, id) + total := dbBytes + artifactBytes + dirs, err := runPiSessionDirs(ctx, q, id) + if err != nil { + return 0, err + } + for _, dir := range dirs { + _, bytes, _ := retentionWalkSize(dir) + total += bytes + } + return total, nil +} + +// RunSize is one run whose cached size was (re)computed. +type RunSize struct { + ID int64 + Workspace string + SizeBytes int64 +} + +// RefreshRunSize recomputes and caches one run's size and returns it. The value +// is stored only if the run did not change while it was measured; it is +// returned either way, since it is the freshest measurement available. +func (s *Store) RefreshRunSize(ctx context.Context, artifactsRoot string, id int64) (int64, error) { + var gen int64 + if err := s.db.QueryRowContext(ctx, `SELECT size_gen FROM pipeline_runs WHERE id=?`, id).Scan(&gen); err != nil { + if err == sql.ErrNoRows { + return 0, errRunNotFound + } + return 0, err + } + size, err := computeRunSize(ctx, s.db, artifactsRoot, id) + if err != nil { + return 0, err + } + if _, err := s.db.ExecContext(ctx, `UPDATE pipeline_runs SET size_bytes=? WHERE id=? AND size_gen=?`, size, id, gen); err != nil { + return 0, err + } + return size, nil +} + +// RefreshRunSizes computes and caches the size of up to limit runs whose cached +// size is missing or stale, newest first, and returns the runs it measured. A +// limit of zero or less means no limit. It is the lazy backfill: callers run it +// in the background or with a small limit on a request path. +func (s *Store) RefreshRunSizes(ctx context.Context, artifactsRoot string, limit int) ([]RunSize, error) { + if limit <= 0 { + limit = -1 + } + rows, err := s.db.QueryContext(ctx, `SELECT id,workspace FROM pipeline_runs WHERE size_bytes<0 ORDER BY id DESC LIMIT ?`, limit) + if err != nil { + return nil, err + } + var stale []RunSize + for rows.Next() { + var item RunSize + if err := rows.Scan(&item.ID, &item.Workspace); err != nil { + rows.Close() + return nil, err + } + stale = append(stale, item) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + measured := make([]RunSize, 0, len(stale)) + for _, item := range stale { + if err := ctx.Err(); err != nil { + return measured, err + } + size, err := s.RefreshRunSize(ctx, artifactsRoot, item.ID) + if err == errRunNotFound { + continue + } + if err != nil { + return measured, err + } + item.SizeBytes = size + measured = append(measured, item) + } + return measured, nil +} + +// StaleRunSizes counts runs whose cached size is missing or stale. +func (s *Store) StaleRunSizes(ctx context.Context) (int, error) { + var count int + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pipeline_runs WHERE size_bytes<0`).Scan(&count) + return count, err +} diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index c2894b6b..45fb973c 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -26,12 +26,13 @@ CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_j CASE WHEN r.status='waiting' THEN COALESCE((SELECT sequence FROM session_export_gates WHERE run_id=r.id),0) ELSE 0 END, EXISTS(SELECT 1 FROM run_schedule_origin WHERE run_id=r.id), COALESCE((SELECT schedule_id FROM run_schedule_origin WHERE run_id=r.id),0), -COALESCE((SELECT name FROM run_schedule_origin WHERE run_id=r.id),'') +COALESCE((SELECT name FROM run_schedule_origin WHERE run_id=r.id),''), +MAX(r.size_bytes,0) ` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version` func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error { var output, stageModels string - if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName); err != nil { + if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName, &r.SizeBytes); err != nil { return err } if stageModels != "" && stageModels != "{}" { diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 7f5a844d..3a2af589 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -81,6 +81,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU db.Close() return nil, err } + if err := s.initRunSizes(); err != nil { + db.Close() + return nil, err + } return s, nil } diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 57401d18..d5f8ba85 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -238,6 +238,9 @@ type Message struct { // displays. Both are optional so older peers ignore them. RunEvent *RunEventFrame `json:"run_event,omitempty"` RunIDs []int64 `json:"run_ids,omitempty"` + // RunsDeleted is the runs_deleted reply to delete_runs (whose request names + // the runs in RunIDs). The runs_removed push carries only RunIDs. + RunsDeleted *RunsDeletedResult `json:"runs_deleted,omitempty"` // NotificationSettings is the set_notification_settings partial update and // the notification_settings reply (every key). See notification_settings.go. NotificationSettings map[string]any `json:"notification_settings,omitempty"` diff --git a/internal/protocol/runevents.go b/internal/protocol/runevents.go index 3cf602f4..55324a5f 100644 --- a/internal/protocol/runevents.go +++ b/internal/protocol/runevents.go @@ -57,6 +57,10 @@ type RunEventFrame struct { // Artifacts is the number of files recorded for the referenced step, so a // client can surface artifact capture without reading the step. Artifacts int `json:"artifacts,omitempty"` + // SizeBytes is the run's measured on-disk size (see RunSummary.SizeBytes). + // It is sent on subscribe snapshots and terminal frames; absence means + // unchanged. + SizeBytes int64 `json:"size_bytes,omitempty"` // Step is present when a single step changed (started, finished or waiting). Step *RunStepDelta `json:"step,omitempty"` } diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index dd48554a..2e02910e 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -16,8 +16,30 @@ const ( GetRunEvidence = "get_run_evidence" RunEvidence = "run_evidence" RepairSessionExport = "repair_session_export" + // DeleteRuns removes terminal runs and everything the server stores for + // them; the reply is RunsDeleted and other subscribers receive RunsRemoved. + DeleteRuns = "delete_runs" + RunsDeleted = "runs_deleted" + RunsRemoved = "runs_removed" ) +// RunsDeletedResult is the runs_deleted reply to delete_runs. Deleted lists the +// run ids removed, Refused every requested id that was kept with the reason +// (an active run, an unknown id, ...), and FreedBytes the summed size_bytes of +// the deleted runs as measured immediately before deletion. All three are +// always present. +type RunsDeletedResult struct { + Deleted []int64 `json:"deleted"` + Refused []RunDeleteRefusal `json:"refused"` + FreedBytes int64 `json:"freed_bytes"` +} + +// RunDeleteRefusal is one requested run id delete_runs did not remove. +type RunDeleteRefusal struct { + RunID int64 `json:"run_id"` + Reason string `json:"reason"` +} + // Waiting statuses. A human stage pauses its run without a live goroutine, so // the run and its step persist these statuses and the decision path resumes // them. Store, server, CLI and TUI read the one pair below, never a literal. @@ -170,6 +192,11 @@ type RunSummary struct { Model string `json:"model,omitempty"` StageModels map[string]string `json:"stage_models,omitempty"` OutputFormat string `json:"output_format,omitempty"` + // SizeBytes is the bytes on disk attributable to the run: its artifact tree + // (turn documents, inputs, outputs, session exports), server-created Pi + // session directories and an estimate of its database rows. It is cached + // server-side; 0 means not measured yet (a background backfill fills it). + SizeBytes int64 `json:"size_bytes"` } // RunGraphVisit contains only routing and lifecycle facts, never private step diff --git a/internal/server/retry_test.go b/internal/server/retry_test.go index e7aebf70..a367f933 100644 --- a/internal/server/retry_test.go +++ b/internal/server/retry_test.go @@ -127,6 +127,10 @@ esac t.Fatal("retry reused run/session state", retry, err) } unchanged, err := store.GetRun(ctx, original.ID) + if err == nil { + // size_bytes is a cache the background sizer fills in, not history. + unchanged.SizeBytes = original.SizeBytes + } if err != nil || !reflect.DeepEqual(original, unchanged) { t.Fatal("retry modified old run", err) } @@ -234,6 +238,10 @@ func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) { t.Fatal("retry lost idempotency", again, err) } unchanged, err := store.GetRun(ctx, original.ID) + if err == nil { + // size_bytes is a cache the background sizer fills in, not history. + unchanged.SizeBytes = original.SizeBytes + } if err != nil || !reflect.DeepEqual(original, unchanged) { t.Fatal("retry modified failed history", unchanged, err) } diff --git a/internal/server/run_delete.go b/internal/server/run_delete.go new file mode 100644 index 00000000..791d1170 --- /dev/null +++ b/internal/server/run_delete.go @@ -0,0 +1,132 @@ +package server + +import ( + "context" + "log" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// listRunsSizeBudget bounds how many stale run sizes one list_runs request +// measures synchronously; the rest are left to the background backfill so a +// first list over hundreds of unmeasured runs stays fast. +const listRunsSizeBudget = 25 + +// runSizeInterval is how often the background sizer re-checks for stale sizes +// when nothing kicks it. +const runSizeInterval = time.Minute + +// runSizeBatch is how many stale runs one backfill step measures. +const runSizeBatch = 50 + +// kickSizes wakes the background sizer without blocking. +func (r *runner) kickSizes() { + if r.sizeKick == nil { + return + } + select { + case r.sizeKick <- struct{}{}: + default: + } +} + +// sizeLoop is the background run-size backfill. It starts with a pass (so an +// upgraded database is measured without blocking startup), then runs on every +// kick or interval until the runner's context or store closes. It never pushes +// frames: sizes reach clients on list_runs, subscribe snapshots and terminal +// frames. +func (r *runner) sizeLoop() { + defer r.schedulerWG.Done() + ticker := time.NewTicker(runSizeInterval) + defer ticker.Stop() + for { + for { + measured, err := r.store.RefreshRunSizes(r.ctx, r.artifactsRoot, runSizeBatch) + if err != nil { + if r.ctx.Err() == nil { + log.Printf("refresh run sizes: %v", err) + } + break + } + // A short batch means the backlog is drained; runs that changed while + // measured stay stale until the next kick or tick, so an active run + // never spins this loop. + if len(measured) < runSizeBatch { + break + } + select { + case <-r.ctx.Done(): + return + case <-r.store.Done(): + return + default: + } + } + select { + case <-r.ctx.Done(): + return + case <-r.store.Done(): + return + case <-ticker.C: + case <-r.sizeKick: + } + } +} + +// refreshListSizes measures a bounded number of stale sizes before list_runs +// reads them and kicks the background sizer for any remainder. +func (r *runner) refreshListSizes(ctx context.Context) { + if _, err := r.store.RefreshRunSizes(ctx, r.artifactsRoot, listRunsSizeBudget); err != nil { + log.Printf("refresh run sizes: %v", err) + } + if stale, err := r.store.StaleRunSizes(ctx); err == nil && stale > 0 { + r.kickSizes() + } +} + +// finishedSize measures a finishing run's size so its terminal frame carries +// it ("computed when a run finishes"). It is one directory walk and one +// aggregate query; 0 on any error. +func (r *runner) finishedSize(id int64) int64 { + ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second) + defer cancel() + size, err := r.store.RefreshRunSize(ctx, r.artifactsRoot, id) + if err != nil { + return 0 + } + return size +} + +// deleteRuns removes terminal runs and everything the server stores for them, +// then tells every subscribed client which runs are gone. It holds r.mu so a run +// cannot become active between the executing check and the delete. +func (r *runner) deleteRuns(ids []int64) (*protocol.RunsDeletedResult, error) { + r.mu.Lock() + result, err := r.store.DeleteRuns(r.ctx, ids, r.artifactsRoot, func(id int64) string { + if _, ok := r.active[id]; ok { + return "run is executing" + } + return "" + }) + r.mu.Unlock() + if err != nil { + return nil, err + } + for _, id := range result.Deleted { + r.identMu.Lock() + delete(r.idents, id) + r.identMu.Unlock() + r.attentionMu.Lock() + delete(r.attentionSent, id) + r.attentionMu.Unlock() + r.notifyMu.Lock() + delete(r.notifyModes, id) + r.notifyMu.Unlock() + } + if len(result.Deleted) > 0 { + log.Printf("deleted runs %v (refused %d, freed %d bytes)", result.Deleted, len(result.Refused), result.FreedBytes) + r.events.removeRuns(result.Deleted, result.Workspaces) + } + return &result.RunsDeletedResult, nil +} diff --git a/internal/server/run_delete_test.go b/internal/server/run_delete_test.go new file mode 100644 index 00000000..60b7d90e --- /dev/null +++ b/internal/server/run_delete_test.go @@ -0,0 +1,126 @@ +package server + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// TestDeleteRunsOverWebSocket covers the wire contract: list_runs carries a +// measured size_bytes, delete_runs deletes a finished run (rows and artifact +// files), refuses an active run and an unknown id, and a second subscribed +// connection receives runs_removed for the deleted run. +func TestDeleteRunsOverWebSocket(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + store, handler, ids := eventStore(t, ctx, "one") + var block atomic.Bool + release := make(chan struct{}) + defer close(release) + handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) { + if block.Load() { + select { + case <-release: + case <-ctx.Done(): + return stage.Result{}, ctx.Err() + } + } + return stage.Result{Outcome: "success", Output: stage.Values{}}, nil + } + workspace := eventWorkspace(t) + finished, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "del-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "done"}}) + if err != nil { + t.Fatal(err) + } + handler.runner.wg.Wait() + artifacts := filepath.Join(handler.runner.artifactsRoot, fmt.Sprint(finished.ID)) + if err := os.MkdirAll(filepath.Join(artifacts, "extra"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(artifacts, "extra", "big.bin"), make([]byte, 50_000), 0o600); err != nil { + t.Fatal(err) + } + // Files written outside a recorded change do not invalidate the cache, so + // re-measure explicitly to include them. + if _, err := store.RefreshRunSize(ctx, handler.runner.artifactsRoot, finished.ID); err != nil { + t.Fatal(err) + } + block.Store(true) + // A different workspace: one run per workspace at a time. + active, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "del-2", Workspace: eventWorkspace(t), NotifyMode: "off", Input: map[string]any{"prompt": "busy"}}) + if err != nil { + t.Fatal(err) + } + + watcher := dialRunEvents(t, ctx, handler) + watcher.send(protocol.Message{Type: protocol.Subscribe, ID: "sub"}) + snapshotSize := int64(0) + for { + message := watcher.read() + if message.Type == protocol.Subscribe { + break + } + if message.Type == protocol.RunEvent && message.RunEvent.RunID == finished.ID { + snapshotSize = message.RunEvent.SizeBytes + } + } + if snapshotSize < 50_000 { + t.Fatalf("snapshot size_bytes = %d, want the measured artifacts", snapshotSize) + } + + operator := dialRunEvents(t, ctx, handler) + operator.send(protocol.Message{Type: protocol.ListRuns, ID: "list"}) + list := operator.read() + if list.Type != protocol.Runs { + t.Fatalf("list reply = %+v", list) + } + for _, run := range list.Runs { + if run.ID == finished.ID && run.SizeBytes != snapshotSize { + t.Fatalf("list size_bytes = %d, want %d", run.SizeBytes, snapshotSize) + } + } + + operator.send(protocol.Message{Type: protocol.DeleteRuns, ID: "del", RunIDs: []int64{finished.ID, active.ID, 4242}}) + reply := operator.read() + if reply.Type != protocol.RunsDeleted || reply.ID != "del" || reply.RunsDeleted == nil { + t.Fatalf("delete reply = %+v", reply) + } + result := reply.RunsDeleted + if fmt.Sprint(result.Deleted) != fmt.Sprint([]int64{finished.ID}) || len(result.Refused) != 2 || result.FreedBytes < 50_000 { + t.Fatalf("delete result = %+v", result) + } + reasons := map[int64]string{} + for _, refusal := range result.Refused { + reasons[refusal.RunID] = refusal.Reason + } + if reasons[active.ID] == "" || reasons[4242] != "run not found" { + t.Fatalf("refusals = %+v", result.Refused) + } + + for { + message := watcher.read() + if message.Type != protocol.RunsRemoved { + continue + } + if fmt.Sprint(message.RunIDs) != fmt.Sprint([]int64{finished.ID}) { + t.Fatalf("runs_removed = %v", message.RunIDs) + } + break + } + if _, err := os.Stat(artifacts); !os.IsNotExist(err) { + t.Fatalf("artifacts remain: %v", err) + } + if _, err := store.GetRun(ctx, finished.ID); err == nil { + t.Fatal("deleted run still readable") + } + if _, err := store.GetRun(ctx, active.ID); err != nil { + t.Fatalf("active run lost: %v", err) + } +} diff --git a/internal/server/runevents.go b/internal/server/runevents.go index 5d38d6ab..38b91a76 100644 --- a/internal/server/runevents.go +++ b/internal/server/runevents.go @@ -263,6 +263,39 @@ func (h *runEventHub) deliver(frame protocol.RunEventFrame) { } } +// removeRuns tells every subscription watching any of the deleted runs that +// they are gone (a runs_removed push carrying the matching run_ids) and drops +// the runs' coalescers so no pending delta resurrects a deleted row. +// workspaces maps each run to its workspace for subscription scoping. +func (h *runEventHub) removeRuns(ids []int64, workspaces map[int64]string) { + if h == nil || len(ids) == 0 { + return + } + h.mu.Lock() + for _, id := range ids { + if c := h.runs[id]; c != nil { + h.stopTimerLocked(c) + delete(h.runs, id) + } + } + targets := make([]*runSubscription, 0, len(h.subs)) + for sub := range h.subs { + targets = append(targets, sub) + } + h.mu.Unlock() + for _, sub := range targets { + var matched []int64 + for _, id := range ids { + if sub.matches(workspaces[id], id) { + matched = append(matched, id) + } + } + if len(matched) > 0 { + sub.sendMessage(protocol.Message{Type: protocol.RunsRemoved, RunIDs: matched}) + } + } +} + func (h *runEventHub) anySubscriberLocked(workspace string) bool { for sub := range h.subs { if sub.workspace == "" || sub.workspace == workspace { @@ -476,6 +509,9 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) { if in.Artifacts > into.Artifacts { into.Artifacts = in.Artifacts } + if in.SizeBytes > 0 { + into.SizeBytes = in.SizeBytes + } if in.Step != nil { into.Step = in.Step } @@ -535,6 +571,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner CancelRequested: summary.CancelRequested, CleanupUnconfirmed: summary.CleanupUnconfirmed, ExportPendingSequence: summary.ExportPendingSequence, + SizeBytes: summary.SizeBytes, } if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) { // The connection write failed; the subscription is closed and the diff --git a/internal/server/runner.go b/internal/server/runner.go index eaade908..77880952 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -78,6 +78,9 @@ type runner struct { // admission. It is separate from the stage's own notify policy. notifyMu sync.Mutex notifyModes map[int64]string + // sizeKick wakes the background run-size backfill (sizeLoop); nil when the + // loop is not running (a runner built directly in a test). + sizeKick chan struct{} } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { @@ -1349,6 +1352,12 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag status, outcome, diagnostic = "failed", "error", "deliverable contract: "+err.Error() } } + // Measure the run's footprint while it is still recorded as active: its + // files are final, and measuring after the terminal status is written would + // widen the window in which a client sees the run finished but still owned. + // The status write re-marks the cache stale, so the background sizer then + // records the exact value including the final output row. + size := r.finishedSize(id) if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil { log.Printf("finish run %d: %v", id, err) } @@ -1373,7 +1382,9 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag CurrentStage: currentStage, CompletedSteps: completed, FinishedAt: detail.FinishedAt, + SizeBytes: size, }, true) + r.kickSizes() if status != protocol.RunStatusCanceled { r.notifyTerminalStep(id, status) } diff --git a/internal/server/websocket.go b/internal/server/websocket.go index c4cbd2eb..ed10cee5 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -97,6 +97,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if runErr == nil { runs.schedulerWG.Add(1) go runs.scheduleLoop() + // The run-size backfill runs in the background so an upgraded + // database never delays startup. + runs.sizeKick = make(chan struct{}, 1) + runs.schedulerWG.Add(1) + go runs.sizeLoop() } } // chat_sessions lists the operator's own recorded sessions. Prefer an @@ -324,9 +329,14 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin response.Error = err.Error() } } - } else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence || request.Type == protocol.RepairSessionExport) { + } else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence || request.Type == protocol.RepairSessionExport || request.Type == protocol.DeleteRuns) { var err error switch request.Type { + case protocol.DeleteRuns: + // Like cancel and retry, deleting run history is an execution + // operation and is not gated by AllowAuthoring. + response.Type = protocol.RunsDeleted + response.RunsDeleted, err = runs.deleteRuns(request.RunIDs) case protocol.RepairSessionExport: response.Type = protocol.Run response.RunDetail, err = runs.repairSessionExport(ctx, request.RunID, request.ExportSequence) @@ -347,6 +357,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin response.RunDetail, err = runs.setTitle(request.RunID, request.Title) case protocol.ListRuns: response.Type = protocol.Runs + runs.refreshListSizes(connCtx) response.Runs, err = runs.listRuns() if err == nil { response.Usage, err = stores[0].UsageStats(ctx)