Add run size_bytes and delete_runs to free disk space

Every run summary carries size_bytes (artifact tree, server-created Pi
session directories and an estimate of the run's database rows), cached in
pipeline_runs.size_bytes, invalidated by triggers, measured when a run
finishes and backfilled in the background. delete_runs removes terminal
runs with every scoped row and server-owned file, refuses active ones with
a reason, and pushes runs_removed to subscribers. pipelines runs rm is the
CLI form; retention apply now shares the complete row deletion.
This commit is contained in:
2026-10-31 22:11:14 +00:00
parent 62b56e66d0
commit 31e89e872b
20 changed files with 1421 additions and 12 deletions
+23 -1
View File
@@ -319,7 +319,29 @@ and matched field. `list_runs` also returns all-history `usage`
aggregates. `attention_acks` reads and `ack_attention` adds to the operator's
shared in-app acknowledgements (dismissed Needs-you entries and chat read
watermarks; see [ntfy.md](ntfy.md#clearing-notifications-across-devices)); other
connections receive an `attention_acks_changed` push. See
connections receive an `attention_acks_changed` push.
Every run summary (`list_runs` rows, `get_run`, subscribe snapshots and terminal
`run_event` frames) carries `size_bytes`: bytes on disk for the run (artifact tree
with turn documents and session exports, server-created Pi session directories,
plus an estimate of its database rows). It is cached in `pipeline_runs.size_bytes`,
invalidated by triggers when the run changes, measured on completion, and
backfilled in the background; 0 means not measured yet. `delete_runs` removes
finished runs to free space:
```json
{"type":"delete_runs","id":"1","run_ids":[12,13,99]}
{"type":"runs_deleted","id":"1","runs_deleted":{"deleted":[12],"refused":[{"run_id":13,"reason":"run is running; cancel it or wait for it to finish"},{"run_id":99,"reason":"run not found"}],"freed_bytes":48213}}
{"type":"runs_removed","run_ids":[12]}
```
Only terminal runs (succeeded, failed, canceled, interrupted) are deleted; active
runs, unconfirmed cleanup, a running session export, a run an unfinished retry
depends on, and unknown ids are refused individually. Deletion removes every
row scoped to the run in one transaction plus `<artifacts>/<run>` and owned Pi
session directories; user workspaces are never touched. Subscribed connections
whose scope covers the runs receive the `runs_removed` push. `pipelines runs rm
<id...>` is the CLI form. See
`internal/protocol/message.go` for JSON envelopes. Mutating RPCs
have real effects; they are not previews. TUI-only animation, scrolling, and
transient client logs are presentation, not separate server capabilities.
+4
View File
@@ -158,6 +158,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait after the run is admitted (0 waits until terminal); separate from per-request --timeout")
case "runs":
f.StringVar(&o.name, "pipeline", "", "Filter by exact pipeline name or ID")
f.BoolVar(&o.yes, "yes", false, "Confirm `runs rm` without an interactive prompt")
case "search":
f.StringVar(&o.kind, "type", "", "Narrow to one kind: "+strings.Join(protocol.SearchKinds, ", "))
f.IntVar(&o.limit, "limit", 0, "Maximum results (default 50, max 200)")
@@ -751,6 +752,9 @@ func (a App) Run(ctx context.Context, args []string) error {
}
return nil
case "list", "templates", "runs", "ping", "connection":
if command == "runs" && len(pos) > 0 && pos[0] == "rm" {
return a.removeRuns(o, pos[1:], call)
}
if len(pos) > 0 {
return fmt.Errorf("%s takes no positional arguments", command)
}
+2 -1
View File
@@ -19,7 +19,7 @@ var usage = map[string]string{
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
"runs": "runs [--pipeline NAME] [--json]",
"runs": "runs [--pipeline NAME] [--json] | runs rm <run-id> [run-id ...] [--yes] [--json]",
"inspect": "inspect <run-id> [--json]",
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
"cancel": "cancel <run-id> [--json]",
@@ -52,6 +52,7 @@ var notes = map[string]string{
"submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
"retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. --stage selects a recorded stage (use --visit for repeated stages). --input-file accepts a bounded JSON object, merged with replacement at the selected stage; '-' reads stdin. Original history and pinned identity are retained; --version selects an explicit published version of the same pipeline only. --wait-timeout leaves an active retry for later read-only inspection or reattachment with the original --request-key, rather than starting a duplicate retry.",
"runs": "Lists runs newest first; --json includes each run's size_bytes (bytes on disk: artifacts, session exports, server-created Pi session directories and an estimate of its database rows; 0 until first measured). `runs rm <id...>` permanently deletes finished runs (succeeded, failed, canceled, interrupted) with every database row and server-owned file for them, after confirmation (--yes for scripts). Active runs, runs an unfinished retry depends on and unknown ids are refused individually with a reason, and the command then exits 1. User workspaces are never touched. Database pages are reused by SQLite; `retention apply` additionally VACUUMs to shrink the file.",
"title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.",
"wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).",
"decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.",
+70
View File
@@ -0,0 +1,70 @@
package cli
import (
"fmt"
"strconv"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// removeRuns implements `pipelines runs rm <id...>`: one delete_runs request
// for every named run. The server refuses active and unknown runs individually;
// any refusal makes the command exit non-zero after reporting the result.
func (a App) removeRuns(o *options, args []string, call callFunc) error {
if len(args) == 0 {
return fmt.Errorf("runs rm requires at least one run ID; use pipelines help runs")
}
ids := make([]int64, 0, len(args))
for _, value := range args {
id, err := strconv.ParseInt(value, 10, 64)
if err != nil || id <= 0 {
return fmt.Errorf("run ID must be positive: %q", value)
}
ids = append(ids, id)
}
if err := a.confirmAction(fmt.Sprintf("Permanently delete %d run(s) with their artifacts and history?", len(ids)), o.yes); err != nil {
return err
}
r, err := call(protocol.Message{Type: protocol.DeleteRuns, RunIDs: ids})
if err != nil {
return err
}
if r.RunsDeleted == nil {
return fmt.Errorf("server did not return a deletion result; update pipelines-server")
}
result := r.RunsDeleted
if o.json {
if err := a.output(result); err != nil {
return err
}
} else {
for _, id := range result.Deleted {
fmt.Fprintf(a.Out, "deleted\t%d\n", id)
}
for _, refusal := range result.Refused {
fmt.Fprintf(a.Out, "refused\t%d\t%s\n", refusal.RunID, refusal.Reason)
}
fmt.Fprintf(a.Out, "Freed %s\n", formatSize(result.FreedBytes))
}
if len(result.Refused) > 0 {
return &ExitError{Code: 1}
}
return nil
}
// formatSize renders a byte count with a binary unit for human output.
func formatSize(bytes int64) string {
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
value, suffix := float64(bytes), "B"
for _, next := range []string{"KiB", "MiB", "GiB", "TiB"} {
if value < unit {
break
}
value /= unit
suffix = next
}
return fmt.Sprintf("%.1f %s", value, suffix)
}
+60
View File
@@ -0,0 +1,60 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/wstest"
)
func TestCLIRunsRmSendsDeleteRuns(t *testing.T) {
var got protocol.Message
endpoint := wstest.Server(t, func(request protocol.Message) protocol.Message {
got = request
return protocol.Message{Type: protocol.RunsDeleted, RunsDeleted: &protocol.RunsDeletedResult{
Deleted: []int64{3},
Refused: []protocol.RunDeleteRefusal{{RunID: 4, Reason: "run is running; cancel it or wait for it to finish"}},
FreedBytes: 2048,
}}
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var out, errOut bytes.Buffer
app := App{Out: &out, Err: &errOut, In: strings.NewReader("")}
err := app.Run(ctx, []string{"--url", endpoint, "runs", "rm", "3", "4", "--yes", "--json"})
var exit *ExitError
if !errors.As(err, &exit) || exit.Code != 1 {
t.Fatalf("err = %v, want exit 1 for a refusal", err)
}
if got.Type != protocol.DeleteRuns || fmt.Sprint(got.RunIDs) != "[3 4]" {
t.Fatalf("wire request = %+v", got)
}
var decoded protocol.RunsDeletedResult
if err := json.Unmarshal(out.Bytes(), &decoded); err != nil {
t.Fatalf("stdout is not JSON: %v\n%s", err, out.String())
}
if fmt.Sprint(decoded.Deleted) != "[3]" || decoded.FreedBytes != 2048 || len(decoded.Refused) != 1 {
t.Fatalf("decoded = %+v", decoded)
}
// Without --yes a declined prompt sends nothing.
got = protocol.Message{}
declined := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}, In: strings.NewReader("n\n")}
if err := declined.Run(ctx, []string{"--url", endpoint, "runs", "rm", "3"}); err == nil {
t.Fatal("declined deletion succeeded")
}
if got.Type != "" {
t.Fatalf("declined deletion sent %+v", got)
}
if err := declined.Run(ctx, []string{"--url", endpoint, "runs", "rm", "x", "--yes"}); err == nil {
t.Fatal("non-numeric id accepted")
}
}
+31 -7
View File
@@ -614,11 +614,21 @@ func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy,
return nil
}
// deleteRetentionRows removes every table row scoped to the selected runs, in
// foreign-key order, batched at 500 ids per statement in the caller's single
// transaction: step-scoped evidence first, then run_steps, then retry edges,
// then the runs.
// deleteRetentionRows removes every table row scoped to the selected runs. It
// is the same removal delete_runs performs (deleteRunRows).
func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
return deleteRunRows(ctx, tx, ids)
}
// deleteRunRows removes every table row scoped to the given runs, in
// foreign-key order, batched at 500 ids per statement in the caller's single
// transaction: step-scoped evidence first, then run_steps, then retry
// provenance, then the runs. Tables with ON DELETE CASCADE are deleted
// explicitly too, so the removal does not depend on PRAGMA foreign_keys. A table
// missing from an older database is skipped. Schedule event history
// (saved_schedule_events, schedule_events) is audit history of the schedule,
// not of the run, and is kept.
func deleteRunRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
for _, table := range []struct{ name, column string }{
{"step_notifications", "run_id"},
{"run_notifications", "run_id"},
@@ -628,10 +638,15 @@ func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
{"step_verdicts", "run_id"},
{"step_session_exports", "run_id"},
{"step_usage", "run_id"},
{"session_export_repairs", "run_id"},
{"session_export_jobs", "run_id"},
{"session_export_gates", "run_id"},
{"run_session_owners", "run_id"},
{"attention_dismissals", "run_id"},
{"run_schedule_origin", "run_id"},
{"run_steps", "run_id"},
} {
if err := deleteRetentionBatch(ctx, tx, table.name, table.column, ids); err != nil {
if err := deleteRetentionBatch(ctx, tx, table.name, table.column, ids); err != nil && !missingTable(err) {
return err
}
}
@@ -640,13 +655,22 @@ func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
for start := 0; start < len(ids); start += retentionDeleteBatch {
batch := ids[start:min(start+retentionDeleteBatch, len(ids))]
placeholders, args := retentionBindings(batch, 2)
if _, err := tx.ExecContext(ctx, "DELETE FROM run_retries WHERE run_id IN ("+placeholders+") OR original_run_id IN ("+placeholders+")", args...); err != nil {
return err
for _, statement := range []string{
"DELETE FROM run_retries WHERE run_id IN (" + placeholders + ") OR original_run_id IN (" + placeholders + ")",
"DELETE FROM retry_checkpoints WHERE run_id IN (" + placeholders + ") OR source_run_id IN (" + placeholders + ")",
} {
if _, err := tx.ExecContext(ctx, statement, args...); err != nil && !missingTable(err) {
return err
}
}
}
return deleteRetentionBatch(ctx, tx, "pipeline_runs", "id", ids)
}
func missingTable(err error) bool {
return err != nil && strings.Contains(err.Error(), "no such table")
}
// retentionDeleteBatch bounds each IN clause so a large removal does not build
// an unbounded statement.
const retentionDeleteBatch = 500
+292
View File
@@ -0,0 +1,292 @@
package pipeline
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// MaxDeleteRuns bounds one delete_runs request.
const MaxDeleteRuns = 1000
// terminalRunStatuses are the only run statuses delete_runs removes. Anything
// else (queued, running, waiting, or a status this build does not know) is
// refused.
var terminalRunStatuses = map[string]bool{"succeeded": true, "failed": true, "canceled": true, "interrupted": true}
// DeleteRunsResult is what DeleteRuns removed. Workspaces maps each deleted run
// to its workspace so the server can scope the runs_removed push.
type DeleteRunsResult struct {
protocol.RunsDeletedResult
Workspaces map[int64]string
}
// DeleteRuns removes each requested terminal run and everything the server
// stores for it: every database row scoped to it (see deleteRunRows), its
// artifact tree <artifactsRoot>/<id>, and Pi session directories the server
// created for its stages. User workspaces and repositories are never touched.
//
// Eligibility is checked inside the write transaction. busy, when non-nil, lets
// the caller veto a run it knows to be executing; its non-empty reason refuses
// the run. Unknown ids, invalid ids and duplicates are refused individually; an
// I/O or database failure aborts the whole request with nothing removed (the
// artifact trees are staged by rename and restored on failure).
func (s *Store) DeleteRuns(ctx context.Context, ids []int64, artifactsRoot string, busy func(int64) string) (*DeleteRunsResult, error) {
if len(ids) == 0 {
return nil, fmt.Errorf("run_ids is required")
}
if len(ids) > MaxDeleteRuns {
return nil, fmt.Errorf("at most %d runs can be deleted at once", MaxDeleteRuns)
}
result := &DeleteRunsResult{
RunsDeletedResult: protocol.RunsDeletedResult{Deleted: []int64{}, Refused: []protocol.RunDeleteRefusal{}},
Workspaces: map[int64]string{},
}
refuse := func(id int64, reason string) {
result.Refused = append(result.Refused, protocol.RunDeleteRefusal{RunID: id, Reason: reason})
}
seen := map[int64]bool{}
var candidates []int64
for _, id := range ids {
switch {
case id <= 0:
refuse(id, "invalid run id")
case seen[id]:
// A repeated id is the same request; report it once.
default:
seen[id] = true
candidates = append(candidates, id)
}
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return nil, err
}
committed := false
defer func() {
if !committed {
_ = tx.Rollback()
}
}()
var deletable []int64
sizes := map[int64]int64{}
for _, id := range candidates {
reason, workspace, err := deleteRefusal(ctx, tx, id)
if err != nil {
return nil, err
}
if reason == "" && busy != nil {
reason = busy(id)
}
if reason != "" {
refuse(id, reason)
continue
}
size, err := computeRunSize(ctx, tx, artifactsRoot, id)
if err != nil {
return nil, err
}
sizes[id] = size
result.Workspaces[id] = workspace
deletable = append(deletable, id)
}
if len(deletable) == 0 {
return result, nil
}
piDirs, err := deletablePiDirs(ctx, tx, deletable)
if err != nil {
return nil, err
}
stage, err := stageRunArtifacts(artifactsRoot, deletable)
if err != nil {
return nil, err
}
if err := deleteRunRows(ctx, tx, deletable); err != nil {
return nil, errors.Join(fmt.Errorf("delete run history: %w", err), stage.restore())
}
if err := tx.Commit(); err != nil {
return nil, errors.Join(fmt.Errorf("commit run deletion: %w", err), stage.restore())
}
committed = true
// The rows are gone; file cleanup failures cannot resurrect the runs, so
// they are reported only in the freed byte count, never as an error.
for _, id := range deletable {
result.Deleted = append(result.Deleted, id)
result.FreedBytes += sizes[id]
}
if stage.dir != "" {
if err := os.RemoveAll(stage.dir); err != nil {
_, remaining, _ := retentionWalkSize(stage.dir)
result.FreedBytes = max(result.FreedBytes-remaining, 0)
}
}
for _, dir := range piDirs {
if err := os.RemoveAll(dir); err != nil {
_, remaining, _ := retentionWalkSize(dir)
result.FreedBytes = max(result.FreedBytes-remaining, 0)
}
}
return result, nil
}
// deleteRefusal returns why a run must not be deleted ("" when it may be), and
// its workspace. It refuses unknown runs, every non-terminal status, runs with
// unconfirmed cleanup, active or waiting visits, a running session export, and
// runs an unfinished retry still depends on.
func deleteRefusal(ctx context.Context, q sizeQuerier, id int64) (string, string, error) {
var status, workspace string
var cleanupUnconfirmed bool
var activeSteps, runningExports, dependents int
err := q.QueryRowContext(ctx, `SELECT r.status,r.workspace,r.cleanup_unconfirmed,
(SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND (s.status IN ('running','waiting') OR s.agent_active<>0)),
(SELECT COUNT(*) FROM session_export_jobs j WHERE j.run_id=r.id AND j.status='running'),
(SELECT COUNT(*) FROM pipeline_runs d WHERE d.id<>r.id AND d.status NOT IN ('succeeded','failed','canceled','interrupted')
AND (d.id IN (SELECT run_id FROM run_retries WHERE original_run_id=r.id) OR d.id IN (SELECT run_id FROM retry_checkpoints WHERE source_run_id=r.id)))
FROM pipeline_runs r WHERE r.id=?`, id).Scan(&status, &workspace, &cleanupUnconfirmed, &activeSteps, &runningExports, &dependents)
if err == sql.ErrNoRows {
return "run not found", "", nil
}
if err != nil {
return "", "", err
}
switch {
case !terminalRunStatuses[status]:
return fmt.Sprintf("run is %s; cancel it or wait for it to finish", status), workspace, nil
case cleanupUnconfirmed:
return "run cleanup is not confirmed yet", workspace, nil
case activeSteps > 0:
return "a stage of this run is still active or waiting", workspace, nil
case runningExports > 0:
return "a session export for this run is in progress", workspace, nil
case dependents > 0:
return "an unfinished retry depends on this run", workspace, nil
}
return "", workspace, nil
}
// deletablePiDirs lists the owned Pi session directories of the runs being
// deleted that no surviving run also references.
func deletablePiDirs(ctx context.Context, q sizeQuerier, ids []int64) ([]string, error) {
deleting := map[int64]bool{}
for _, id := range ids {
deleting[id] = true
}
var dirs []string
for _, id := range ids {
found, err := runPiSessionDirs(ctx, q, id)
if err != nil {
return nil, err
}
for _, dir := range found {
rows, err := q.QueryContext(ctx, `SELECT DISTINCT run_id FROM run_session_owners WHERE reference=?`, dir)
if err != nil {
return nil, err
}
shared := false
for rows.Next() {
var owner int64
if err := rows.Scan(&owner); err != nil {
rows.Close()
return nil, err
}
if !deleting[owner] {
shared = true
}
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
if !shared {
dirs = append(dirs, dir)
}
}
}
return dirs, nil
}
// stagedArtifacts is a set of run artifact trees renamed into one staging
// directory under the artifacts root, so the database removal and the file
// removal commit or roll back together.
type stagedArtifacts struct {
dir string
staged []string
}
// stageRunArtifacts renames <root>/<id> for each run into a fresh staging
// directory. Only the exact numeric entry is staged; a symlink is moved (the
// link, never its target) and a regular file there aborts with an error.
func stageRunArtifacts(root string, ids []int64) (*stagedArtifacts, error) {
stage := &stagedArtifacts{}
if root == "" {
return stage, nil
}
info, err := os.Stat(root)
if os.IsNotExist(err) {
return stage, nil
}
if err != nil {
return nil, fmt.Errorf("inspect artifacts root %s: %w", root, err)
}
if !info.IsDir() {
return stage, nil
}
for _, id := range ids {
original := filepath.Join(root, strconv.FormatInt(id, 10))
entry, err := os.Lstat(original)
if os.IsNotExist(err) {
continue
}
if err != nil {
return nil, errors.Join(fmt.Errorf("inspect artifacts for run %d: %w", id, err), stage.restore())
}
if !entry.IsDir() && entry.Mode()&os.ModeSymlink == 0 {
return nil, errors.Join(fmt.Errorf("refusing to remove run %d: %s is a regular file, not a run artifact directory", id, original), stage.restore())
}
if stage.dir == "" {
stage.dir = filepath.Join(root, fmt.Sprintf(".delete-staging-%d-%d", time.Now().UnixNano(), os.Getpid()))
if err := os.Mkdir(stage.dir, 0o700); err != nil {
stage.dir = ""
return nil, fmt.Errorf("create delete staging directory: %w", err)
}
}
if err := os.Rename(original, filepath.Join(stage.dir, filepath.Base(original))); err != nil {
return nil, errors.Join(fmt.Errorf("stage artifacts for run %d: %w", id, err), stage.restore())
}
stage.staged = append(stage.staged, original)
}
return stage, nil
}
// restore moves every staged tree back and removes the staging directory.
func (s *stagedArtifacts) restore() error {
var problems []string
for _, original := range s.staged {
if err := os.Rename(filepath.Join(s.dir, filepath.Base(original)), original); err != nil {
problems = append(problems, fmt.Sprintf("restore %s: %v", original, err))
}
}
s.staged = nil
if s.dir != "" {
if err := os.Remove(s.dir); err != nil && !os.IsNotExist(err) {
problems = append(problems, fmt.Sprintf("remove staging directory %s: %v", s.dir, err))
}
}
if len(problems) > 0 {
return fmt.Errorf("restore artifacts: %s", strings.Join(problems, "; "))
}
return nil
}
+308
View File
@@ -0,0 +1,308 @@
package pipeline
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
)
func (f *retentionFixture) sizeColumn(t *testing.T, id int64) int64 {
t.Helper()
var size int64
if err := f.store.db.QueryRow("SELECT size_bytes FROM pipeline_runs WHERE id=?", id).Scan(&size); err != nil {
t.Fatal(err)
}
return size
}
func (f *retentionFixture) listedSize(t *testing.T, id int64) int64 {
t.Helper()
runs, err := f.store.ListRuns(context.Background())
if err != nil {
t.Fatal(err)
}
for _, run := range runs {
if run.ID == id {
return run.SizeBytes
}
}
t.Fatalf("run %d not listed", id)
return 0
}
// usePiSessionsBase points Pi session ownership at a temporary directory so a
// test never reads or removes the real user cache.
func usePiSessionsBase(t *testing.T) string {
t.Helper()
base := filepath.Join(t.TempDir(), "pi-sessions")
if err := os.MkdirAll(base, 0o700); err != nil {
t.Fatal(err)
}
previous := piSessionsBase
piSessionsBase = func() (string, error) { return base, nil }
t.Cleanup(func() { piSessionsBase = previous })
return base
}
func TestRunSizeComputedCachedAndInvalidated(t *testing.T) {
ctx := context.Background()
base := usePiSessionsBase(t)
f := newRetentionFixture(t)
id := f.run(t, "sized", 0)
f.step(t, id, 1, "completed", false)
f.artifact(t, id, 1, "report.md", 10_000)
piDir := filepath.Join(base, "stage-1")
if err := os.MkdirAll(piDir, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(piDir, "rpc.jsonl"), make([]byte, 3_000), 0o600); err != nil {
t.Fatal(err)
}
if _, err := f.store.db.Exec("INSERT INTO run_session_owners(run_id,sequence,backend,reference) VALUES(?,1,'pi',?)", id, piDir); err != nil {
t.Fatal(err)
}
if got := f.sizeColumn(t, id); got != -1 {
t.Fatalf("new run size = %d, want -1 (unmeasured)", got)
}
if got := f.listedSize(t, id); got != 0 {
t.Fatalf("unmeasured listed size = %d, want 0", got)
}
measured, err := f.store.RefreshRunSizes(ctx, f.root, 0)
if err != nil {
t.Fatal(err)
}
if len(measured) != 1 || measured[0].ID != id {
t.Fatalf("measured = %+v", measured)
}
size := measured[0].SizeBytes
if size < 13_000 || size > 20_000 {
t.Fatalf("size = %d, want artifacts + pi dir + modest DB estimate", size)
}
if got := f.listedSize(t, id); got != size {
t.Fatalf("listed size = %d, want cached %d", got, size)
}
again, err := f.store.RefreshRunSizes(ctx, f.root, 0)
if err != nil || len(again) != 0 {
t.Fatalf("cached run re-measured: %+v %v", again, err)
}
// A visit change invalidates the cache; the refreshed size includes it.
if _, err := f.store.db.Exec("UPDATE run_steps SET transcript=? WHERE run_id=?", string(make([]byte, 5_000)), id); err != nil {
t.Fatal(err)
}
if got := f.sizeColumn(t, id); got != -1 {
t.Fatalf("size after step update = %d, want -1", got)
}
grown, err := f.store.RefreshRunSize(ctx, f.root, id)
if err != nil {
t.Fatal(err)
}
if grown < size+5_000 {
t.Fatalf("size after transcript = %d, want >= %d", grown, size+5_000)
}
// A run status change invalidates it too.
f.setStatus(t, id, "failed")
if got := f.sizeColumn(t, id); got != -1 {
t.Fatalf("size after status change = %d, want -1", got)
}
}
func TestRunSizeNotStoredWhenRunChangedDuringMeasurement(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
id := f.run(t, "racy", 0)
var gen int64
if err := f.store.db.QueryRow("SELECT size_gen FROM pipeline_runs WHERE id=?", id).Scan(&gen); err != nil {
t.Fatal(err)
}
f.step(t, id, 1, "completed", false) // a change after the generation was read
if _, err := f.store.db.ExecContext(ctx, "UPDATE pipeline_runs SET size_bytes=? WHERE id=? AND size_gen=?", 1, id, gen); err != nil {
t.Fatal(err)
}
if got := f.sizeColumn(t, id); got != -1 {
t.Fatalf("stale measurement stored: %d", got)
}
}
// runTables lists every table delete_runs must clear, with its run column.
var runTables = map[string]string{
"pipeline_runs": "id", "run_steps": "run_id", "step_notifications": "run_id", "step_file_changes": "run_id",
"step_control": "run_id", "step_artifacts": "run_id", "step_verdicts": "run_id", "step_session_exports": "run_id",
"step_usage": "run_id", "run_session_owners": "run_id", "session_export_repairs": "run_id",
"session_export_jobs": "run_id", "session_export_gates": "run_id", "run_notifications": "run_id",
"attention_dismissals": "run_id", "run_schedule_origin": "run_id", "run_retries": "run_id",
"retry_checkpoints": "run_id",
}
func (f *retentionFixture) seedEverything(t *testing.T, id int64, piDir string) {
t.Helper()
f.step(t, id, 1, "completed", false)
statements := []string{
"INSERT INTO step_notifications(run_id,sequence,receipts_json) VALUES(?,1,'[]')",
"INSERT INTO step_file_changes(run_id,sequence,changes_json) VALUES(?,1,'[]')",
"INSERT INTO step_control(run_id,sequence,evidence_json) VALUES(?,1,'{}')",
"INSERT INTO step_artifacts(run_id,sequence,artifacts_json) VALUES(?,1,'{}')",
"INSERT INTO step_verdicts(run_id,sequence,verdict_json) VALUES(?,1,'{}')",
"INSERT INTO step_session_exports(run_id,sequence,export_json) VALUES(?,1,'{}')",
"INSERT INTO step_usage(run_id,sequence,usage_json) VALUES(?,1,'{}')",
"INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,1,'{}')",
"INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at,status) VALUES(?,1,'s','pi','/tmp','r',0,'complete')",
"INSERT INTO session_export_gates(run_id,sequence) VALUES(?,1)",
"INSERT INTO run_notifications(run_id,sequence,event,channel,title,body,click) VALUES(?,1,'finished','system','t','b','c')",
"INSERT INTO attention_dismissals(key,run_id,dismissed_at) VALUES('run:'||?1,?1,0)",
"INSERT INTO run_schedule_origin(run_id,schedule_id,name) VALUES(?,1,'nightly')",
}
for _, statement := range statements {
if _, err := f.store.db.Exec(statement, id); err != nil {
t.Fatalf("%s: %v", statement, err)
}
}
if _, err := f.store.db.Exec("INSERT INTO run_session_owners(run_id,sequence,backend,reference) VALUES(?,1,'pi',?)", id, piDir); err != nil {
t.Fatal(err)
}
f.artifact(t, id, 1, "out.md", 4_096)
if err := os.MkdirAll(piDir, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(piDir, "rpc.jsonl"), []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
}
func TestDeleteRunsRemovesRowsAndFiles(t *testing.T) {
ctx := context.Background()
base := usePiSessionsBase(t)
f := newRetentionFixture(t)
doomed := f.run(t, "doomed", 0)
kept := f.run(t, "kept", 0)
active := f.run(t, "active", 0)
f.setStatus(t, active, "running")
f.seedEverything(t, doomed, filepath.Join(base, "stage-doomed"))
f.seedEverything(t, kept, filepath.Join(base, "stage-kept"))
// kept is a finished retry of doomed: deleting the original drops the
// provenance edge and checkpoint but keeps the retry itself.
f.retry(t, kept, doomed)
if _, err := f.store.db.Exec("INSERT INTO retry_checkpoints(run_id,source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash) VALUES(?,?,'resume',1,'{}',1,'h')", kept, doomed); err != nil {
t.Fatal(err)
}
// A workspace-like directory outside the data dir must survive.
workspace := filepath.Join(t.TempDir(), "repo")
if err := os.MkdirAll(workspace, 0o700); err != nil {
t.Fatal(err)
}
if _, err := f.store.db.Exec("UPDATE pipeline_runs SET workspace=? WHERE id=?", workspace, doomed); err != nil {
t.Fatal(err)
}
expected, err := f.store.RefreshRunSize(ctx, f.root, doomed)
if err != nil {
t.Fatal(err)
}
result, err := f.store.DeleteRuns(ctx, []int64{doomed, doomed, active, 99999, -3}, f.root, nil)
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(result.Deleted) != fmt.Sprint([]int64{doomed}) {
t.Fatalf("deleted = %v", result.Deleted)
}
reasons := map[int64]string{}
for _, refusal := range result.Refused {
reasons[refusal.RunID] = refusal.Reason
}
if len(result.Refused) != 3 || reasons[active] == "" || reasons[99999] != "run not found" || reasons[-3] != "invalid run id" {
t.Fatalf("refused = %+v", result.Refused)
}
if result.FreedBytes != expected {
t.Fatalf("freed = %d, want measured size %d", result.FreedBytes, expected)
}
if result.Workspaces[doomed] != workspace {
t.Fatalf("workspace = %q", result.Workspaces[doomed])
}
for table, column := range runTables {
var count int
if err := f.store.db.QueryRow("SELECT COUNT(*) FROM "+table+" WHERE "+column+"=?", doomed).Scan(&count); err != nil {
t.Fatalf("%s: %v", table, err)
}
if count != 0 {
t.Fatalf("%s still holds %d rows for deleted run", table, count)
}
}
for _, check := range []string{"SELECT COUNT(*) FROM run_retries WHERE original_run_id=?", "SELECT COUNT(*) FROM retry_checkpoints WHERE source_run_id=?"} {
var count int
if err := f.store.db.QueryRow(check, doomed).Scan(&count); err != nil || count != 0 {
t.Fatalf("%s = %d %v", check, count, err)
}
}
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(doomed))); !os.IsNotExist(err) {
t.Fatalf("artifacts of deleted run remain: %v", err)
}
if _, err := os.Stat(filepath.Join(base, "stage-doomed")); !os.IsNotExist(err) {
t.Fatalf("pi session dir of deleted run remains: %v", err)
}
if entries, _ := os.ReadDir(f.root); len(entries) != 1 {
t.Fatalf("artifacts root entries = %v, want only the kept run (no staging leftovers)", entries)
}
for _, path := range []string{workspace, filepath.Join(f.root, fmt.Sprint(kept)), filepath.Join(base, "stage-kept")} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("%s must survive: %v", path, err)
}
}
for table, column := range runTables {
if table == "run_retries" || table == "retry_checkpoints" {
continue
}
var count int
if err := f.store.db.QueryRow("SELECT COUNT(*) FROM "+table+" WHERE "+column+"=?", kept).Scan(&count); err != nil || count == 0 {
t.Fatalf("%s lost the kept run's rows: %d %v", table, count, err)
}
}
violations, err := f.store.db.Query("PRAGMA foreign_key_check")
if err != nil {
t.Fatal(err)
}
if violations.Next() {
violations.Close()
t.Fatal("foreign key violation after delete")
}
violations.Close()
}
func TestDeleteRunsRefusesActiveAndDependedOnRuns(t *testing.T) {
ctx := context.Background()
f := newRetentionFixture(t)
queued := f.run(t, "queued", 0)
f.setStatus(t, queued, "queued")
waiting := f.run(t, "waiting", 0)
f.setStatus(t, waiting, "waiting")
agent := f.run(t, "agent", 0)
f.step(t, agent, 1, "failed", true)
original := f.run(t, "original", 0)
retry := f.run(t, "retry", 0)
f.setStatus(t, retry, "running")
f.retry(t, retry, original)
busy := f.run(t, "busy", 0)
f.artifact(t, busy, 1, "keep.md", 10)
result, err := f.store.DeleteRuns(ctx, []int64{queued, waiting, agent, original, busy}, f.root, func(id int64) string {
if id == busy {
return "run is executing"
}
return ""
})
if err != nil {
t.Fatal(err)
}
if len(result.Deleted) != 0 || len(result.Refused) != 5 || result.FreedBytes != 0 {
t.Fatalf("result = %+v", result.RunsDeletedResult)
}
if _, err := os.Stat(filepath.Join(f.root, fmt.Sprint(busy))); err != nil {
t.Fatalf("refused run lost artifacts: %v", err)
}
if got := f.count(t, "pipeline_runs"); got != 6 {
t.Fatalf("runs = %d, want 6", got)
}
if _, err := f.store.DeleteRuns(ctx, nil, f.root, nil); err == nil {
t.Fatal("empty run_ids accepted")
}
}
+264
View File
@@ -0,0 +1,264 @@
package pipeline
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
)
// Run size accounting. pipeline_runs.size_bytes caches the bytes on disk a run
// accounts for: its artifact tree (<artifacts>/<run>, which holds turn
// documents, inputs, outputs and session exports), any Pi session directory the
// server created for its stages, and an estimate of its database rows (the byte
// length of every large text column plus a small per-row overhead).
//
// -1 means "not computed or stale". Triggers mark a run stale whenever its row,
// a visit or visit evidence changes, and bump size_gen so a size computed
// concurrently with a change is never stored as current (see storeRunSize).
// Existing databases start with every run at -1 and are backfilled lazily.
// runSizeTriggers are the tables whose writes change a run's footprint. Each
// gets an insert and an update trigger that invalidates the owning run.
var runSizeTriggers = []string{
"run_steps", "step_notifications", "step_file_changes", "step_control", "step_artifacts",
"step_verdicts", "step_session_exports", "step_usage", "run_session_owners",
"session_export_repairs", "run_notifications", "retry_checkpoints",
}
// initRunSizes adds the cache columns and invalidation triggers. Every step is
// idempotent so a repeated Open and an upgraded database both open cleanly.
func (s *Store) initRunSizes() error {
if err := s.addColumnIfMissing("pipeline_runs", "size_bytes", "INTEGER NOT NULL DEFAULT -1"); err != nil {
return err
}
if err := s.addColumnIfMissing("pipeline_runs", "size_gen", "INTEGER NOT NULL DEFAULT 0"); err != nil {
return err
}
statements := []string{
// The nested UPDATE touches only size columns, which are not in the OF
// list, so this trigger cannot re-fire itself.
`CREATE TRIGGER IF NOT EXISTS run_size_stale_run AFTER UPDATE OF status,outcome,output_json,input_json,diagnostic,finished_at,title ON pipeline_runs
BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.id; END`,
}
for _, table := range runSizeTriggers {
for _, event := range []string{"INSERT", "UPDATE"} {
statements = append(statements, fmt.Sprintf(`CREATE TRIGGER IF NOT EXISTS run_size_stale_%s_%s AFTER %s ON %s
BEGIN UPDATE pipeline_runs SET size_bytes=-1,size_gen=size_gen+1 WHERE id=NEW.run_id; END`, table, strings.ToLower(event), event, table))
}
}
for _, statement := range statements {
if _, err := s.db.Exec(statement); err != nil {
return err
}
}
return nil
}
// runSizeRowOverhead approximates SQLite's per-row cost (record header, rowid,
// index entries) beyond the measured column bytes.
const runSizeRowOverhead = 64
// runDBBytesQuery estimates one run's database footprint: the byte length of
// every variable-size column in every table scoped to the run, plus a fixed
// per-row overhead. Each subquery uses the table's run_id primary-key prefix or
// index, so the cost is bounded by the run's own rows.
var runDBBytesQuery = func() string {
type part struct {
table, where string
columns []string
}
parts := []part{
{"pipeline_runs", "id=?1", []string{"request_key", "workspace", "title", "title_source", "title_model", "model", "stage_models", "write_paths", "input_json", "output_json", "outcome", "current_stage", "diagnostic", "output_format"}},
{"run_steps", "run_id=?1", []string{"stage_key", "outcome", "target", "input_json", "output_json", "diagnostic", "provider", "model", "session_id", "transcript"}},
{"step_notifications", "run_id=?1", []string{"receipts_json"}},
{"step_file_changes", "run_id=?1", []string{"changes_json"}},
{"step_control", "run_id=?1", []string{"evidence_json"}},
{"step_artifacts", "run_id=?1", []string{"artifacts_json"}},
{"step_verdicts", "run_id=?1", []string{"verdict_json"}},
{"step_session_exports", "run_id=?1", []string{"export_json"}},
{"step_usage", "run_id=?1", []string{"usage_json"}},
{"run_session_owners", "run_id=?1", []string{"backend", "reference"}},
{"session_export_repairs", "run_id=?1", []string{"export_json"}},
{"session_export_jobs", "run_id=?1", []string{"session_id", "backend", "directory", "reference", "token", "diagnostic"}},
{"run_notifications", "run_id=?1", []string{"event", "channel", "title", "body", "click", "diagnostic", "message_id"}},
{"retry_checkpoints", "run_id=?1", []string{"mode", "handoff_json", "source_hash"}},
{"run_schedule_origin", "run_id=?1", []string{"name"}},
}
terms := make([]string, 0, len(parts))
for _, p := range parts {
lengths := make([]string, 0, len(p.columns)+1)
for _, column := range p.columns {
lengths = append(lengths, "COALESCE(length(CAST("+column+" AS BLOB)),0)")
}
lengths = append(lengths, fmt.Sprint(runSizeRowOverhead))
terms = append(terms, "COALESCE((SELECT SUM("+strings.Join(lengths, "+")+") FROM "+p.table+" WHERE "+p.where+"),0)")
}
return "SELECT " + strings.Join(terms, "+")
}()
// sizeQuerier is the read surface shared by *sql.DB and *sql.Tx, so a size is
// computed identically by the cache refresh and inside the delete transaction.
type sizeQuerier interface {
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}
// piSessionsBase is the directory the Pi stage client creates its per-visit
// session directories under (see internal/stage/pi.go). It is a variable so
// tests can point it at a temporary directory.
var piSessionsBase = func() (string, error) {
cache, err := os.UserCacheDir()
if err != nil {
return "", err
}
return filepath.Join(cache, "pipelines", "pi-sessions"), nil
}
// ownedPiSessionDir reports whether reference is a server-created Pi session
// directory: an absolute, clean path directly under piSessionsBase named
// stage-*, that is a real directory rather than a symlink. Anything else - a
// user path, a workspace, a symlink - is never sized or removed.
func ownedPiSessionDir(reference string) bool {
base, err := piSessionsBase()
if err != nil || base == "" || reference == "" || !filepath.IsAbs(reference) || filepath.Clean(reference) != reference {
return false
}
if filepath.Dir(reference) != base || !strings.HasPrefix(filepath.Base(reference), "stage-") {
return false
}
info, err := os.Lstat(reference)
return err == nil && info.IsDir()
}
// runPiSessionDirs lists the owned Pi session directories recorded for a run.
func runPiSessionDirs(ctx context.Context, q sizeQuerier, id int64) ([]string, error) {
rows, err := q.QueryContext(ctx, `SELECT DISTINCT reference FROM run_session_owners WHERE run_id=? AND backend='pi' AND reference<>''`, id)
if err != nil {
return nil, err
}
defer rows.Close()
var dirs []string
for rows.Next() {
var reference string
if err := rows.Scan(&reference); err != nil {
return nil, err
}
if ownedPiSessionDir(reference) {
dirs = append(dirs, reference)
}
}
return dirs, rows.Err()
}
// errRunNotFound marks a size or delete request for a run id with no row.
var errRunNotFound = fmt.Errorf("run not found")
// computeRunSize measures one run now: artifact tree, owned Pi session
// directories and the database estimate. Unreadable files are skipped rather
// than failing the measurement; the size is a best-effort lower bound then.
func computeRunSize(ctx context.Context, q sizeQuerier, artifactsRoot string, id int64) (int64, error) {
var exists bool
if err := q.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pipeline_runs WHERE id=?)`, id).Scan(&exists); err != nil {
return 0, err
}
if !exists {
return 0, errRunNotFound
}
var dbBytes int64
if err := q.QueryRowContext(ctx, runDBBytesQuery, id).Scan(&dbBytes); err != nil {
return 0, err
}
_, artifactBytes, _ := retentionArtifactsSize(artifactsRoot, id)
total := dbBytes + artifactBytes
dirs, err := runPiSessionDirs(ctx, q, id)
if err != nil {
return 0, err
}
for _, dir := range dirs {
_, bytes, _ := retentionWalkSize(dir)
total += bytes
}
return total, nil
}
// RunSize is one run whose cached size was (re)computed.
type RunSize struct {
ID int64
Workspace string
SizeBytes int64
}
// RefreshRunSize recomputes and caches one run's size and returns it. The value
// is stored only if the run did not change while it was measured; it is
// returned either way, since it is the freshest measurement available.
func (s *Store) RefreshRunSize(ctx context.Context, artifactsRoot string, id int64) (int64, error) {
var gen int64
if err := s.db.QueryRowContext(ctx, `SELECT size_gen FROM pipeline_runs WHERE id=?`, id).Scan(&gen); err != nil {
if err == sql.ErrNoRows {
return 0, errRunNotFound
}
return 0, err
}
size, err := computeRunSize(ctx, s.db, artifactsRoot, id)
if err != nil {
return 0, err
}
if _, err := s.db.ExecContext(ctx, `UPDATE pipeline_runs SET size_bytes=? WHERE id=? AND size_gen=?`, size, id, gen); err != nil {
return 0, err
}
return size, nil
}
// RefreshRunSizes computes and caches the size of up to limit runs whose cached
// size is missing or stale, newest first, and returns the runs it measured. A
// limit of zero or less means no limit. It is the lazy backfill: callers run it
// in the background or with a small limit on a request path.
func (s *Store) RefreshRunSizes(ctx context.Context, artifactsRoot string, limit int) ([]RunSize, error) {
if limit <= 0 {
limit = -1
}
rows, err := s.db.QueryContext(ctx, `SELECT id,workspace FROM pipeline_runs WHERE size_bytes<0 ORDER BY id DESC LIMIT ?`, limit)
if err != nil {
return nil, err
}
var stale []RunSize
for rows.Next() {
var item RunSize
if err := rows.Scan(&item.ID, &item.Workspace); err != nil {
rows.Close()
return nil, err
}
stale = append(stale, item)
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
measured := make([]RunSize, 0, len(stale))
for _, item := range stale {
if err := ctx.Err(); err != nil {
return measured, err
}
size, err := s.RefreshRunSize(ctx, artifactsRoot, item.ID)
if err == errRunNotFound {
continue
}
if err != nil {
return measured, err
}
item.SizeBytes = size
measured = append(measured, item)
}
return measured, nil
}
// StaleRunSizes counts runs whose cached size is missing or stale.
func (s *Store) StaleRunSizes(ctx context.Context) (int, error) {
var count int
err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pipeline_runs WHERE size_bytes<0`).Scan(&count)
return count, err
}
+3 -2
View File
@@ -26,12 +26,13 @@ CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_j
CASE WHEN r.status='waiting' THEN COALESCE((SELECT sequence FROM session_export_gates WHERE run_id=r.id),0) ELSE 0 END,
EXISTS(SELECT 1 FROM run_schedule_origin WHERE run_id=r.id),
COALESCE((SELECT schedule_id FROM run_schedule_origin WHERE run_id=r.id),0),
COALESCE((SELECT name FROM run_schedule_origin WHERE run_id=r.id),'')
COALESCE((SELECT name FROM run_schedule_origin WHERE run_id=r.id),''),
MAX(r.size_bytes,0)
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output, stageModels string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName); err != nil {
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence, &r.Scheduled, &r.ScheduleID, &r.ScheduleName, &r.SizeBytes); err != nil {
return err
}
if stageModels != "" && stageModels != "{}" {
+4
View File
@@ -81,6 +81,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
db.Close()
return nil, err
}
if err := s.initRunSizes(); err != nil {
db.Close()
return nil, err
}
return s, nil
}
+3
View File
@@ -238,6 +238,9 @@ type Message struct {
// displays. Both are optional so older peers ignore them.
RunEvent *RunEventFrame `json:"run_event,omitempty"`
RunIDs []int64 `json:"run_ids,omitempty"`
// RunsDeleted is the runs_deleted reply to delete_runs (whose request names
// the runs in RunIDs). The runs_removed push carries only RunIDs.
RunsDeleted *RunsDeletedResult `json:"runs_deleted,omitempty"`
// NotificationSettings is the set_notification_settings partial update and
// the notification_settings reply (every key). See notification_settings.go.
NotificationSettings map[string]any `json:"notification_settings,omitempty"`
+4
View File
@@ -57,6 +57,10 @@ type RunEventFrame struct {
// Artifacts is the number of files recorded for the referenced step, so a
// client can surface artifact capture without reading the step.
Artifacts int `json:"artifacts,omitempty"`
// SizeBytes is the run's measured on-disk size (see RunSummary.SizeBytes).
// It is sent on subscribe snapshots and terminal frames; absence means
// unchanged.
SizeBytes int64 `json:"size_bytes,omitempty"`
// Step is present when a single step changed (started, finished or waiting).
Step *RunStepDelta `json:"step,omitempty"`
}
+27
View File
@@ -16,8 +16,30 @@ const (
GetRunEvidence = "get_run_evidence"
RunEvidence = "run_evidence"
RepairSessionExport = "repair_session_export"
// DeleteRuns removes terminal runs and everything the server stores for
// them; the reply is RunsDeleted and other subscribers receive RunsRemoved.
DeleteRuns = "delete_runs"
RunsDeleted = "runs_deleted"
RunsRemoved = "runs_removed"
)
// RunsDeletedResult is the runs_deleted reply to delete_runs. Deleted lists the
// run ids removed, Refused every requested id that was kept with the reason
// (an active run, an unknown id, ...), and FreedBytes the summed size_bytes of
// the deleted runs as measured immediately before deletion. All three are
// always present.
type RunsDeletedResult struct {
Deleted []int64 `json:"deleted"`
Refused []RunDeleteRefusal `json:"refused"`
FreedBytes int64 `json:"freed_bytes"`
}
// RunDeleteRefusal is one requested run id delete_runs did not remove.
type RunDeleteRefusal struct {
RunID int64 `json:"run_id"`
Reason string `json:"reason"`
}
// Waiting statuses. A human stage pauses its run without a live goroutine, so
// the run and its step persist these statuses and the decision path resumes
// them. Store, server, CLI and TUI read the one pair below, never a literal.
@@ -170,6 +192,11 @@ type RunSummary struct {
Model string `json:"model,omitempty"`
StageModels map[string]string `json:"stage_models,omitempty"`
OutputFormat string `json:"output_format,omitempty"`
// SizeBytes is the bytes on disk attributable to the run: its artifact tree
// (turn documents, inputs, outputs, session exports), server-created Pi
// session directories and an estimate of its database rows. It is cached
// server-side; 0 means not measured yet (a background backfill fills it).
SizeBytes int64 `json:"size_bytes"`
}
// RunGraphVisit contains only routing and lifecycle facts, never private step
+8
View File
@@ -127,6 +127,10 @@ esac
t.Fatal("retry reused run/session state", retry, err)
}
unchanged, err := store.GetRun(ctx, original.ID)
if err == nil {
// size_bytes is a cache the background sizer fills in, not history.
unchanged.SizeBytes = original.SizeBytes
}
if err != nil || !reflect.DeepEqual(original, unchanged) {
t.Fatal("retry modified old run", err)
}
@@ -234,6 +238,10 @@ func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
t.Fatal("retry lost idempotency", again, err)
}
unchanged, err := store.GetRun(ctx, original.ID)
if err == nil {
// size_bytes is a cache the background sizer fills in, not history.
unchanged.SizeBytes = original.SizeBytes
}
if err != nil || !reflect.DeepEqual(original, unchanged) {
t.Fatal("retry modified failed history", unchanged, err)
}
+132
View File
@@ -0,0 +1,132 @@
package server
import (
"context"
"log"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// listRunsSizeBudget bounds how many stale run sizes one list_runs request
// measures synchronously; the rest are left to the background backfill so a
// first list over hundreds of unmeasured runs stays fast.
const listRunsSizeBudget = 25
// runSizeInterval is how often the background sizer re-checks for stale sizes
// when nothing kicks it.
const runSizeInterval = time.Minute
// runSizeBatch is how many stale runs one backfill step measures.
const runSizeBatch = 50
// kickSizes wakes the background sizer without blocking.
func (r *runner) kickSizes() {
if r.sizeKick == nil {
return
}
select {
case r.sizeKick <- struct{}{}:
default:
}
}
// sizeLoop is the background run-size backfill. It starts with a pass (so an
// upgraded database is measured without blocking startup), then runs on every
// kick or interval until the runner's context or store closes. It never pushes
// frames: sizes reach clients on list_runs, subscribe snapshots and terminal
// frames.
func (r *runner) sizeLoop() {
defer r.schedulerWG.Done()
ticker := time.NewTicker(runSizeInterval)
defer ticker.Stop()
for {
for {
measured, err := r.store.RefreshRunSizes(r.ctx, r.artifactsRoot, runSizeBatch)
if err != nil {
if r.ctx.Err() == nil {
log.Printf("refresh run sizes: %v", err)
}
break
}
// A short batch means the backlog is drained; runs that changed while
// measured stay stale until the next kick or tick, so an active run
// never spins this loop.
if len(measured) < runSizeBatch {
break
}
select {
case <-r.ctx.Done():
return
case <-r.store.Done():
return
default:
}
}
select {
case <-r.ctx.Done():
return
case <-r.store.Done():
return
case <-ticker.C:
case <-r.sizeKick:
}
}
}
// refreshListSizes measures a bounded number of stale sizes before list_runs
// reads them and kicks the background sizer for any remainder.
func (r *runner) refreshListSizes(ctx context.Context) {
if _, err := r.store.RefreshRunSizes(ctx, r.artifactsRoot, listRunsSizeBudget); err != nil {
log.Printf("refresh run sizes: %v", err)
}
if stale, err := r.store.StaleRunSizes(ctx); err == nil && stale > 0 {
r.kickSizes()
}
}
// finishedSize measures a finishing run's size so its terminal frame carries
// it ("computed when a run finishes"). It is one directory walk and one
// aggregate query; 0 on any error.
func (r *runner) finishedSize(id int64) int64 {
ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
defer cancel()
size, err := r.store.RefreshRunSize(ctx, r.artifactsRoot, id)
if err != nil {
return 0
}
return size
}
// deleteRuns removes terminal runs and everything the server stores for them,
// then tells every subscribed client which runs are gone. It holds r.mu so a run
// cannot become active between the executing check and the delete.
func (r *runner) deleteRuns(ids []int64) (*protocol.RunsDeletedResult, error) {
r.mu.Lock()
result, err := r.store.DeleteRuns(r.ctx, ids, r.artifactsRoot, func(id int64) string {
if _, ok := r.active[id]; ok {
return "run is executing"
}
return ""
})
r.mu.Unlock()
if err != nil {
return nil, err
}
for _, id := range result.Deleted {
r.identMu.Lock()
delete(r.idents, id)
r.identMu.Unlock()
r.attentionMu.Lock()
delete(r.attentionSent, id)
r.attentionMu.Unlock()
r.notifyMu.Lock()
delete(r.notifyModes, id)
r.notifyMu.Unlock()
}
if len(result.Deleted) > 0 {
log.Printf("deleted runs %v (refused %d, freed %d bytes)", result.Deleted, len(result.Refused), result.FreedBytes)
r.events.removeRuns(result.Deleted, result.Workspaces)
}
return &result.RunsDeletedResult, nil
}
+126
View File
@@ -0,0 +1,126 @@
package server
import (
"context"
"fmt"
"os"
"path/filepath"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// TestDeleteRunsOverWebSocket covers the wire contract: list_runs carries a
// measured size_bytes, delete_runs deletes a finished run (rows and artifact
// files), refuses an active run and an unknown id, and a second subscribed
// connection receives runs_removed for the deleted run.
func TestDeleteRunsOverWebSocket(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
store, handler, ids := eventStore(t, ctx, "one")
var block atomic.Bool
release := make(chan struct{})
defer close(release)
handler.runner.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
if block.Load() {
select {
case <-release:
case <-ctx.Done():
return stage.Result{}, ctx.Err()
}
}
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
workspace := eventWorkspace(t)
finished, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "del-1", Workspace: workspace, NotifyMode: "off", Input: map[string]any{"prompt": "done"}})
if err != nil {
t.Fatal(err)
}
handler.runner.wg.Wait()
artifacts := filepath.Join(handler.runner.artifactsRoot, fmt.Sprint(finished.ID))
if err := os.MkdirAll(filepath.Join(artifacts, "extra"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(artifacts, "extra", "big.bin"), make([]byte, 50_000), 0o600); err != nil {
t.Fatal(err)
}
// Files written outside a recorded change do not invalidate the cache, so
// re-measure explicitly to include them.
if _, err := store.RefreshRunSize(ctx, handler.runner.artifactsRoot, finished.ID); err != nil {
t.Fatal(err)
}
block.Store(true)
// A different workspace: one run per workspace at a time.
active, err := handler.runner.start(protocol.Message{PipelineID: ids[0], Version: 1, RequestKey: "del-2", Workspace: eventWorkspace(t), NotifyMode: "off", Input: map[string]any{"prompt": "busy"}})
if err != nil {
t.Fatal(err)
}
watcher := dialRunEvents(t, ctx, handler)
watcher.send(protocol.Message{Type: protocol.Subscribe, ID: "sub"})
snapshotSize := int64(0)
for {
message := watcher.read()
if message.Type == protocol.Subscribe {
break
}
if message.Type == protocol.RunEvent && message.RunEvent.RunID == finished.ID {
snapshotSize = message.RunEvent.SizeBytes
}
}
if snapshotSize < 50_000 {
t.Fatalf("snapshot size_bytes = %d, want the measured artifacts", snapshotSize)
}
operator := dialRunEvents(t, ctx, handler)
operator.send(protocol.Message{Type: protocol.ListRuns, ID: "list"})
list := operator.read()
if list.Type != protocol.Runs {
t.Fatalf("list reply = %+v", list)
}
for _, run := range list.Runs {
if run.ID == finished.ID && run.SizeBytes != snapshotSize {
t.Fatalf("list size_bytes = %d, want %d", run.SizeBytes, snapshotSize)
}
}
operator.send(protocol.Message{Type: protocol.DeleteRuns, ID: "del", RunIDs: []int64{finished.ID, active.ID, 4242}})
reply := operator.read()
if reply.Type != protocol.RunsDeleted || reply.ID != "del" || reply.RunsDeleted == nil {
t.Fatalf("delete reply = %+v", reply)
}
result := reply.RunsDeleted
if fmt.Sprint(result.Deleted) != fmt.Sprint([]int64{finished.ID}) || len(result.Refused) != 2 || result.FreedBytes < 50_000 {
t.Fatalf("delete result = %+v", result)
}
reasons := map[int64]string{}
for _, refusal := range result.Refused {
reasons[refusal.RunID] = refusal.Reason
}
if reasons[active.ID] == "" || reasons[4242] != "run not found" {
t.Fatalf("refusals = %+v", result.Refused)
}
for {
message := watcher.read()
if message.Type != protocol.RunsRemoved {
continue
}
if fmt.Sprint(message.RunIDs) != fmt.Sprint([]int64{finished.ID}) {
t.Fatalf("runs_removed = %v", message.RunIDs)
}
break
}
if _, err := os.Stat(artifacts); !os.IsNotExist(err) {
t.Fatalf("artifacts remain: %v", err)
}
if _, err := store.GetRun(ctx, finished.ID); err == nil {
t.Fatal("deleted run still readable")
}
if _, err := store.GetRun(ctx, active.ID); err != nil {
t.Fatalf("active run lost: %v", err)
}
}
+37
View File
@@ -263,6 +263,39 @@ func (h *runEventHub) deliver(frame protocol.RunEventFrame) {
}
}
// removeRuns tells every subscription watching any of the deleted runs that
// they are gone (a runs_removed push carrying the matching run_ids) and drops
// the runs' coalescers so no pending delta resurrects a deleted row.
// workspaces maps each run to its workspace for subscription scoping.
func (h *runEventHub) removeRuns(ids []int64, workspaces map[int64]string) {
if h == nil || len(ids) == 0 {
return
}
h.mu.Lock()
for _, id := range ids {
if c := h.runs[id]; c != nil {
h.stopTimerLocked(c)
delete(h.runs, id)
}
}
targets := make([]*runSubscription, 0, len(h.subs))
for sub := range h.subs {
targets = append(targets, sub)
}
h.mu.Unlock()
for _, sub := range targets {
var matched []int64
for _, id := range ids {
if sub.matches(workspaces[id], id) {
matched = append(matched, id)
}
}
if len(matched) > 0 {
sub.sendMessage(protocol.Message{Type: protocol.RunsRemoved, RunIDs: matched})
}
}
}
func (h *runEventHub) anySubscriberLocked(workspace string) bool {
for sub := range h.subs {
if sub.workspace == "" || sub.workspace == workspace {
@@ -476,6 +509,9 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
if in.Artifacts > into.Artifacts {
into.Artifacts = in.Artifacts
}
if in.SizeBytes > 0 {
into.SizeBytes = in.SizeBytes
}
if in.Step != nil {
into.Step = in.Step
}
@@ -535,6 +571,7 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
ExportPendingSequence: summary.ExportPendingSequence,
SizeBytes: summary.SizeBytes,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
+11
View File
@@ -78,6 +78,9 @@ type runner struct {
// admission. It is separate from the stage's own notify policy.
notifyMu sync.Mutex
notifyModes map[int64]string
// sizeKick wakes the background run-size backfill (sizeLoop); nil when the
// loop is not running (a runner built directly in a test).
sizeKick chan struct{}
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
@@ -1349,6 +1352,12 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
status, outcome, diagnostic = "failed", "error", "deliverable contract: "+err.Error()
}
}
// Measure the run's footprint while it is still recorded as active: its
// files are final, and measuring after the terminal status is written would
// widen the window in which a client sees the run finished but still owned.
// The status write re-marks the cache stale, so the background sizer then
// records the exact value including the final output row.
size := r.finishedSize(id)
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
log.Printf("finish run %d: %v", id, err)
}
@@ -1373,7 +1382,9 @@ func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diag
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: detail.FinishedAt,
SizeBytes: size,
}, true)
r.kickSizes()
if status != protocol.RunStatusCanceled {
r.notifyTerminalStep(id, status)
}
+12 -1
View File
@@ -97,6 +97,11 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
if runErr == nil {
runs.schedulerWG.Add(1)
go runs.scheduleLoop()
// The run-size backfill runs in the background so an upgraded
// database never delays startup.
runs.sizeKick = make(chan struct{}, 1)
runs.schedulerWG.Add(1)
go runs.sizeLoop()
}
}
// chat_sessions lists the operator's own recorded sessions. Prefer an
@@ -324,9 +329,14 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
response.Error = err.Error()
}
}
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence || request.Type == protocol.RepairSessionExport) {
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun || request.Type == protocol.DecideRun || request.Type == protocol.SetRunTitle || request.Type == protocol.GetRunEvidence || request.Type == protocol.RepairSessionExport || request.Type == protocol.DeleteRuns) {
var err error
switch request.Type {
case protocol.DeleteRuns:
// Like cancel and retry, deleting run history is an execution
// operation and is not gated by AllowAuthoring.
response.Type = protocol.RunsDeleted
response.RunsDeleted, err = runs.deleteRuns(request.RunIDs)
case protocol.RepairSessionExport:
response.Type = protocol.Run
response.RunDetail, err = runs.repairSessionExport(ctx, request.RunID, request.ExportSequence)
@@ -347,6 +357,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
response.RunDetail, err = runs.setTitle(request.RunID, request.Title)
case protocol.ListRuns:
response.Type = protocol.Runs
runs.refreshListSizes(connCtx)
response.Runs, err = runs.listRuns()
if err == nil {
response.Usage, err = stores[0].UsageStats(ctx)