From 18eec34f56df74a06addd499df9c46257283bf89 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 6 Nov 2026 21:26:44 +0000 Subject: [PATCH 1/3] Measure server data and registered worktree storage safely --- internal/server/storage_inventory.go | 109 +++++++++++ internal/server/storage_inventory_git.go | 222 ++++++++++++++++++++++ internal/server/storage_inventory_test.go | 162 ++++++++++++++++ internal/server/storage_inventory_walk.go | 139 ++++++++++++++ 4 files changed, 632 insertions(+) create mode 100644 internal/server/storage_inventory.go create mode 100644 internal/server/storage_inventory_git.go create mode 100644 internal/server/storage_inventory_test.go create mode 100644 internal/server/storage_inventory_walk.go diff --git a/internal/server/storage_inventory.go b/internal/server/storage_inventory.go new file mode 100644 index 00000000..9b1f3272 --- /dev/null +++ b/internal/server/storage_inventory.go @@ -0,0 +1,109 @@ +package server + +import ( + "context" + "sort" + "time" +) + +type storageInventoryConfig struct { + DatabasePath, DataRoot, ArtifactsRoot, AttachmentsRoot, SessionsRoot string + WorkspaceRoots, ProtectedWorkspaces []string +} + +type storageCategory struct { + ID string `json:"id"` + Label string `json:"label"` + Path string `json:"path,omitempty"` + Bytes int64 `json:"bytes"` + Files int64 `json:"files"` + Partial bool `json:"partial,omitempty"` + Error string `json:"error,omitempty"` + ModifiedAt time.Time `json:"-"` +} + +type storageWorktree struct { + Path string `json:"path"` + Repository string `json:"repository"` + Branch string `json:"branch,omitempty"` + Bytes int64 `json:"bytes"` + Files int64 `json:"files"` + ModifiedAt time.Time `json:"modified_at"` + Eligible bool `json:"eligible"` + Reason string `json:"reason,omitempty"` + locked bool +} + +type storageInventory struct { + TotalBytes int64 + Categories []storageCategory + Worktrees []storageWorktree + Partial bool + Errors []string +} + +// Inventory counts logical regular-file bytes. It never follows symlinks, and +// each inode is counted once across nested roots and hard-linked files. +func inspectStorage(ctx context.Context, config storageInventoryConfig) (storageInventory, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + result := storageInventory{Categories: []storageCategory{}, Worktrees: []storageWorktree{}} + trees, discoveryErrors := discoverStorageWorktrees(ctx, config.WorkspaceRoots, config.ProtectedWorkspaces) + result.Errors = discoveryErrors + result.Partial = len(discoveryErrors) > 0 + walker := storageWalker{ctx: ctx, seen: map[string]bool{}, inodes: map[[2]uint64]bool{}} + add := func(category storageCategory, roots ...string) storageCategory { + for _, root := range roots { + walker.measure(root, &category) + } + result.TotalBytes += category.Bytes + result.Partial = result.Partial || category.Partial + if category.Error != "" { + result.Errors = append(result.Errors, category.Error) + } + result.Categories = append(result.Categories, category) + return category + } + if config.DatabasePath != "" { + add(storageCategory{ID: "database", Label: "Database and journal", Path: config.DatabasePath}, config.DatabasePath, config.DatabasePath+"-wal", config.DatabasePath+"-shm") + } + add(storageCategory{ID: "artifacts", Label: "Run artifacts and exports", Path: config.ArtifactsRoot}, config.ArtifactsRoot) + add(storageCategory{ID: "attachments", Label: "Chat attachments", Path: config.AttachmentsRoot}, config.AttachmentsRoot) + add(storageCategory{ID: "sessions", Label: "Harness session files", Path: config.SessionsRoot}, config.SessionsRoot) + add(storageCategory{ID: "other_data", Label: "Other server data", Path: config.DataRoot}, config.DataRoot) + category := storageCategory{ID: "worktrees", Label: "Linked worktrees"} + // Nested linked trees are attributed before their parents. + sort.Slice(trees, func(i, j int) bool { return len(trees[i].Path) > len(trees[j].Path) }) + for _, tree := range trees { + measured := storageCategory{} + walker.measure(tree.Path, &measured) + tree.Bytes, tree.Files = measured.Bytes, measured.Files + if measured.ModifiedAt.After(tree.ModifiedAt) { + tree.ModifiedAt = measured.ModifiedAt + } + if measured.Partial { + tree.Eligible = false + tree.Reason = "Storage measurement incomplete" + category.Partial = true + category.Error = measured.Error + } + category.Bytes += measured.Bytes + category.Files += measured.Files + result.Worktrees = append(result.Worktrees, tree) + } + result.TotalBytes += category.Bytes + result.Categories = append(result.Categories, category) + result.Partial = result.Partial || category.Partial + if category.Error != "" { + result.Errors = append(result.Errors, category.Error) + } + roots := append([]string{}, config.WorkspaceRoots...) + sort.Strings(roots) + add(storageCategory{ID: "workspaces", Label: "Other project working files"}, roots...) + sort.Slice(result.Worktrees, func(i, j int) bool { return result.Worktrees[i].Path < result.Worktrees[j].Path }) + if ctx.Err() != nil { + result.Partial = true + result.Errors = append(result.Errors, ctx.Err().Error()) + } + return result, nil +} diff --git a/internal/server/storage_inventory_git.go b/internal/server/storage_inventory_git.go new file mode 100644 index 00000000..6460504b --- /dev/null +++ b/internal/server/storage_inventory_git.go @@ -0,0 +1,222 @@ +package server + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +func discoverStorageWorktrees(ctx context.Context, roots, protected []string) ([]storageWorktree, []string) { + seenRepos := map[string]bool{} + seenTrees := map[string]bool{} + trees := []storageWorktree{} + problems := []string{} + for index, root := range roots { + if index >= 256 { + problems = append(problems, "Workspace discovery limit reached") + break + } + root = filepath.Clean(root) + if _, err := os.Stat(root); os.IsNotExist(err) { + continue + } + common, err := storageGit(ctx, root, "rev-parse", "--git-common-dir") + if err != nil { + continue + } // A recorded workspace need not be a Git repository. + common = resolveGitPath(root, common) + common, err = filepath.EvalSymlinks(common) + if err != nil { + problems = append(problems, err.Error()) + continue + } + if seenRepos[common] { + continue + } + seenRepos[common] = true + if len(seenRepos) > 64 { + problems = append(problems, "Repository discovery limit reached") + break + } + listing, err := storageGit(ctx, root, "worktree", "list", "--porcelain", "-z") + if err != nil { + problems = append(problems, err.Error()) + continue + } + repository := common + for _, record := range strings.Split(listing, "\x00\x00") { + fields := strings.Split(record, "\x00") + if len(fields) > 0 && strings.HasPrefix(fields[0], "worktree ") { + candidate := strings.TrimPrefix(fields[0], "worktree ") + if info, err := os.Lstat(filepath.Join(candidate, ".git")); err == nil && info.IsDir() { + repository = candidate + break + } + } + } + for _, record := range strings.Split(listing, "\x00\x00") { + fields := strings.Split(record, "\x00") + if len(fields) == 0 || !strings.HasPrefix(fields[0], "worktree ") { + continue + } + path := strings.TrimPrefix(fields[0], "worktree ") + if seenTrees[path] { + continue + } + seenTrees[path] = true + if len(trees) >= 512 { + problems = append(problems, "Worktree discovery limit reached") + return trees, problems + } + // The primary checkout has a .git directory; linked worktrees have a file. + gitInfo, err := os.Lstat(filepath.Join(path, ".git")) + if err != nil || !gitInfo.Mode().IsRegular() { + continue + } + tree := storageWorktree{Path: path, Repository: repository} + for _, field := range fields[1:] { + if strings.HasPrefix(field, "branch ") { + tree.Branch = strings.TrimPrefix(field, "branch refs/heads/") + } + if field == "locked" || strings.HasPrefix(field, "locked ") { + tree.locked = true + } + } + if info, err := os.Stat(path); err == nil { + tree.ModifiedAt = info.ModTime().UTC() + } + tree.Reason = storageWorktreeProtection(ctx, tree, protected) + tree.Eligible = tree.Reason == "" + trees = append(trees, tree) + } + } + return trees, problems +} + +// Eligibility is deliberately conservative: no forced removal, branch deletion, +// or deletion of a workspace whose work has not reached the repository's main. +func storageWorktreeProtection(ctx context.Context, tree storageWorktree, protected []string) string { + if tree.locked { + return "Git worktree is locked" + } + root, err := filepath.EvalSymlinks(tree.Path) + if err != nil || root != filepath.Clean(tree.Path) { + return "Worktree path is unavailable or indirect" + } + for _, path := range protected { + if path == "" { + continue + } + absolute, err := storageCanonicalPath(path) + if err != nil { + return "Protected workspace path is unresolved" + } + if root == absolute || pathWithin(root, absolute) || pathWithin(absolute, root) { + return "Workspace is in use by a run or open chat" + } + } + info, err := os.Lstat(filepath.Join(root, ".git")) + if err != nil || !info.Mode().IsRegular() { + return "Primary checkout or invalid linked worktree" + } + gitDir, err := storageGit(ctx, root, "rev-parse", "--git-dir") + if err != nil { + return "Worktree metadata unavailable" + } + common, err := storageGit(ctx, root, "rev-parse", "--git-common-dir") + if err != nil { + return "Repository metadata unavailable" + } + gitDir, err = canonicalDirectory(resolveGitPath(root, gitDir)) + if err != nil { + return "Worktree metadata unavailable" + } + common, err = canonicalDirectory(resolveGitPath(root, common)) + if err != nil || !pathWithin(filepath.Join(common, "worktrees"), gitDir) { + return "Unverified linked worktree" + } + if _, err := os.Stat(filepath.Join(gitDir, "locked")); err == nil { + return "Git worktree is locked" + } + + currentBranch, _ := storageGit(ctx, root, "symbolic-ref", "--quiet", "--short", "HEAD") + if currentBranch == "main" || currentBranch == "master" { + return "Main branch checkout" + } + top, err := storageGit(ctx, root, "rev-parse", "--show-toplevel") + if err != nil || top != root { + return "Worktree root is not verified" + } + + status, err := storageGit(ctx, root, "status", "--porcelain", "--untracked-files=all") + if err != nil { + return "Git status unavailable" + } + if status != "" { + return "Uncommitted or untracked work" + } + if _, err := storageGit(ctx, root, "merge-base", "--is-ancestor", "HEAD", "refs/heads/main"); err != nil { + return "Work is not fully merged into main" + } + return "" +} + +func removeStorageWorktree(ctx context.Context, tree storageWorktree, protected []string) error { + if reason := storageWorktreeProtection(ctx, tree, protected); reason != "" { + return fmt.Errorf("worktree protected: %s", reason) + } + _, err := storageGit(ctx, tree.Repository, "worktree", "remove", "--", tree.Path) + return err +} + +// Git subprocesses have both time and output bounds; no user shell is invoked. +func storageGit(ctx context.Context, directory string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...) + var out storageGitBuffer + command.Stdout = &out + command.Stderr = &out + if err := command.Run(); err != nil { + return "", fmt.Errorf("git %s in %s: %w", strings.Join(args, " "), directory, err) + } + return strings.TrimSpace(out.String()), nil +} + +type storageGitBuffer struct{ bytes.Buffer } + +func (b *storageGitBuffer) Write(data []byte) (int, error) { + if b.Len()+len(data) > 1<<20 { + return 0, fmt.Errorf("git output limit reached") + } + return b.Buffer.Write(data) +} + +// Resolve aliases even when the protected directory is a future child path. +func storageCanonicalPath(path string) (string, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return "", err + } + current := absolute + for { + resolved, err := filepath.EvalSymlinks(current) + if err == nil { + relative, err := filepath.Rel(current, absolute) + if err != nil { + return "", err + } + return filepath.Join(resolved, relative), nil + } + parent := filepath.Dir(current) + if parent == current { + return "", err + } + current = parent + } +} diff --git a/internal/server/storage_inventory_test.go b/internal/server/storage_inventory_test.go new file mode 100644 index 00000000..2ecddabb --- /dev/null +++ b/internal/server/storage_inventory_test.go @@ -0,0 +1,162 @@ +package server + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func storageTestFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } +} + +func TestStorageInventoryDeduplicatesAndExcludesSymlinks(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + artifacts := filepath.Join(data, "artifacts") + database := filepath.Join(data, "state.db") + storageTestFile(t, database, "database") + storageTestFile(t, database+"-wal", "wal") + storageTestFile(t, filepath.Join(artifacts, "1", "output"), "artifact") + storageTestFile(t, filepath.Join(data, "notes"), "note") + outside := filepath.Join(root, "outside") + storageTestFile(t, outside, "must not count") + if err := os.Symlink(outside, filepath.Join(data, "symlink")); err != nil { + t.Fatal(err) + } + if err := os.Link(filepath.Join(data, "notes"), filepath.Join(data, "hardlink")); err != nil { + t.Fatal(err) + } + inventory, err := inspectStorage(context.Background(), storageInventoryConfig{DatabasePath: database, DataRoot: data, ArtifactsRoot: artifacts, WorkspaceRoots: []string{data, data}}) + if err != nil { + t.Fatal(err) + } + if inventory.TotalBytes != 23 { + t.Fatalf("bytes %d, want23: %+v", inventory.TotalBytes, inventory.Categories) + } + if inventory.Partial { + t.Fatalf("unexpected partial: %+v", inventory) + } + categories := map[string]int64{} + for _, item := range inventory.Categories { + categories[item.ID] = item.Bytes + } + if categories["database"] != 11 || categories["artifacts"] != 8 || categories["other_data"] != 4 || categories["workspaces"] != 0 { + t.Fatalf("categories %+v", categories) + } +} + +func TestStorageMeasurementReportsCancellationAndIndirectRoots(t *testing.T) { + root := t.TempDir() + storageTestFile(t, filepath.Join(root, "file"), "content") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _, partial, problems := measureStoragePath(ctx, root) + if !partial || len(problems) == 0 { + t.Fatal("canceled measurement claimed complete") + } + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(root, link); err != nil { + t.Fatal(err) + } + _, bytes, partial, problems := measureStoragePath(context.Background(), link) + if bytes != 0 || !partial || len(problems) == 0 { + t.Fatal("symlink root followed or not reported") + } +} + +func storageTestGit(t *testing.T, root string, args ...string) string { + t.Helper() + out, err := storageGit(context.Background(), root, args...) + if err != nil { + t.Fatal(err) + } + return out +} +func storageTestRepo(t *testing.T) (string, string) { + t.Helper() + base := t.TempDir() + main := filepath.Join(base, "main") + if err := os.Mkdir(main, 0700); err != nil { + t.Fatal(err) + } + storageTestGit(t, main, "init", "-b", "main") + storageTestGit(t, main, "config", "user.email", "fixture@example.invalid") + storageTestGit(t, main, "config", "user.name", "Fixture") + storageTestFile(t, filepath.Join(main, "tracked"), "initial") + storageTestGit(t, main, "add", "tracked") + storageTestGit(t, main, "commit", "-m", "initial") + linked := filepath.Join(base, "linked") + storageTestGit(t, main, "worktree", "add", "-b", "item", linked) + return main, linked +} + +func TestStorageWorktreesRequireMergedCleanInactiveAndUnlocked(t *testing.T) { + ctx := context.Background() + main, linked := storageTestRepo(t) + discover := func(protected ...string) storageWorktree { + t.Helper() + trees, problems := discoverStorageWorktrees(ctx, []string{main, linked}, protected) + if len(problems) > 0 || len(trees) != 1 { + t.Fatalf("discovery %+v %+v", trees, problems) + } + return trees[0] + } + tree := discover() + canonicalMain, err := filepath.EvalSymlinks(main) + if err != nil { + t.Fatal(err) + } + if !tree.Eligible || tree.Repository != canonicalMain { + t.Fatalf("clean merged linked tree not eligible: %+v", tree) + } + if discover(linked).Eligible { + t.Fatal("open chat/run workspace eligible") + } + if discover(filepath.Join(linked, "subdirectory")).Eligible { + t.Fatal("nested active workspace eligible") + } + storageTestFile(t, filepath.Join(linked, "draft"), "do not delete") + if got := discover(); got.Eligible || !strings.Contains(got.Reason, "Uncommitted") { + t.Fatalf("dirty tree %+v", got) + } + if err := removeStorageWorktree(ctx, tree, nil); err == nil { + t.Fatal("stale candidate removed newly dirty tree") + } + if err := os.Remove(filepath.Join(linked, "draft")); err != nil { + t.Fatal(err) + } + storageTestGit(t, main, "worktree", "lock", linked) + if discover().Eligible { + t.Fatal("locked tree eligible") + } + storageTestGit(t, main, "worktree", "unlock", linked) + storageTestFile(t, filepath.Join(linked, "tracked"), "unlanded") + storageTestGit(t, linked, "add", "tracked") + storageTestGit(t, linked, "commit", "-m", "unlanded") + if got := discover(); got.Eligible || !strings.Contains(got.Reason, "merged") { + t.Fatalf("unlanded tree %+v", got) + } + storageTestGit(t, main, "merge", "--ff-only", "item") + if !discover().Eligible { + t.Fatal("landed clean tree not eligible") + } + if err := removeStorageWorktree(ctx, discover(), nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(linked); !os.IsNotExist(err) { + t.Fatalf("worktree remains: %v", err) + } + storageTestGit(t, main, "show-ref", "--verify", "refs/heads/item") + if _, err := os.Stat(main); err != nil { + t.Fatal("main checkout removed") + } +} diff --git a/internal/server/storage_inventory_walk.go b/internal/server/storage_inventory_walk.go new file mode 100644 index 00000000..d45667da --- /dev/null +++ b/internal/server/storage_inventory_walk.go @@ -0,0 +1,139 @@ +package server + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "time" +) + +type storageWalker struct { + ctx context.Context + seen map[string]bool + inodes map[[2]uint64]bool + entries int +} + +func (w *storageWalker) measure(root string, category *storageCategory) { + if strings.TrimSpace(root) == "" { + return + } + root, err := filepath.Abs(root) + if err != nil { + storageMeasureError(category, err) + return + } + info, err := os.Lstat(root) + if os.IsNotExist(err) { + return + } + if err != nil { + storageMeasureError(category, err) + return + } + if info.Mode()&os.ModeSymlink != 0 { + storageMeasureError(category, fmt.Errorf("symlink root excluded: %s", root)) + return + } + if err := w.visit(root, category, 0); err != nil { + storageMeasureError(category, err) + } +} + +// Read directories in bounded chunks rather than allocating a whole directory +// listing. Compare opened directory identity to avoid traversing a replacement +// symlink between Lstat and Open. Regular files are stat-only, never opened. +func (w *storageWalker) visit(path string, category *storageCategory, depth int) error { + if w.ctx.Err() != nil { + return w.ctx.Err() + } + w.entries++ + if w.entries > 500000 { + return fmt.Errorf("storage scan entry limit reached") + } + if depth > 128 { + return fmt.Errorf("storage scan depth limit reached") + } + if w.seen[path] { + return nil + } + w.seen[path] = true + info, err := os.Lstat(path) + if err != nil { + storageMeasureError(category, err) + return nil + } + if info.Mode()&os.ModeSymlink != 0 { + return nil + } + if info.ModTime().After(category.ModifiedAt) { + category.ModifiedAt = info.ModTime().UTC() + } + if info.Mode().IsRegular() { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + key := [2]uint64{uint64(stat.Dev), uint64(stat.Ino)} + if w.inodes[key] { + return nil + } + w.inodes[key] = true + } + category.Files++ + category.Bytes += info.Size() + return nil + } + if !info.IsDir() { + return nil + } + directory, err := os.Open(path) + if err != nil { + storageMeasureError(category, err) + return nil + } + defer directory.Close() + opened, err := directory.Stat() + if err != nil || !os.SameFile(info, opened) { + storageMeasureError(category, fmt.Errorf("directory changed during storage scan: %s", path)) + return nil + } + for { + if w.ctx.Err() != nil { + return w.ctx.Err() + } + entries, err := directory.ReadDir(256) + if err != nil && err != io.EOF { + storageMeasureError(category, err) + return nil + } + for _, entry := range entries { + if err := w.visit(filepath.Join(path, entry.Name()), category, depth+1); err != nil { + return err + } + } + if err == io.EOF { + return nil + } + } +} + +func storageMeasureError(category *storageCategory, err error) { + category.Partial = true + if category.Error == "" { + category.Error = err.Error() + } +} + +func measureStoragePath(ctx context.Context, path string) (files, bytes int64, partial bool, problems []string) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + category := storageCategory{} + walker := storageWalker{ctx: ctx, seen: map[string]bool{}, inodes: map[[2]uint64]bool{}} + walker.measure(path, &category) + if category.Error != "" { + problems = append(problems, category.Error) + } + return category.Files, category.Bytes, category.Partial, problems +} From aa19c82156220184d85bcd55e114ad7e9326d7bc Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 6 Nov 2026 21:29:14 +0000 Subject: [PATCH 2/3] Protect containing worktrees from deleting nested checkouts --- internal/server/storage_inventory.go | 1 + internal/server/storage_inventory_git.go | 10 ++- internal/server/storage_inventory_nested.go | 77 ++++++++++++++++++++ internal/server/storage_inventory_test.go | 78 +++++++++++++++++++++ 4 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 internal/server/storage_inventory_nested.go diff --git a/internal/server/storage_inventory.go b/internal/server/storage_inventory.go index 9b1f3272..1cf1dcce 100644 --- a/internal/server/storage_inventory.go +++ b/internal/server/storage_inventory.go @@ -31,6 +31,7 @@ type storageWorktree struct { ModifiedAt time.Time `json:"modified_at"` Eligible bool `json:"eligible"` Reason string `json:"reason,omitempty"` + knownRoots []string locked bool } diff --git a/internal/server/storage_inventory_git.go b/internal/server/storage_inventory_git.go index 6460504b..fd992a55 100644 --- a/internal/server/storage_inventory_git.go +++ b/internal/server/storage_inventory_git.go @@ -71,14 +71,14 @@ func discoverStorageWorktrees(ctx context.Context, roots, protected []string) ([ seenTrees[path] = true if len(trees) >= 512 { problems = append(problems, "Worktree discovery limit reached") - return trees, problems + return protectNestedStorageWorktrees(ctx, trees, roots, seenTrees, problems), problems } // The primary checkout has a .git directory; linked worktrees have a file. gitInfo, err := os.Lstat(filepath.Join(path, ".git")) if err != nil || !gitInfo.Mode().IsRegular() { continue } - tree := storageWorktree{Path: path, Repository: repository} + tree := storageWorktree{Path: path, Repository: repository, knownRoots: roots} for _, field := range fields[1:] { if strings.HasPrefix(field, "branch ") { tree.Branch = strings.TrimPrefix(field, "branch refs/heads/") @@ -95,7 +95,7 @@ func discoverStorageWorktrees(ctx context.Context, roots, protected []string) ([ trees = append(trees, tree) } } - return trees, problems + return protectNestedStorageWorktrees(ctx, trees, roots, seenTrees, problems), problems } // Eligibility is deliberately conservative: no forced removal, branch deletion, @@ -144,6 +144,10 @@ func storageWorktreeProtection(ctx context.Context, tree storageWorktree, protec return "Git worktree is locked" } + if reason := storageNestedProtection(ctx, root, tree.knownRoots); reason != "" { + return reason + } + currentBranch, _ := storageGit(ctx, root, "symbolic-ref", "--quiet", "--short", "HEAD") if currentBranch == "main" || currentBranch == "master" { return "Main branch checkout" diff --git a/internal/server/storage_inventory_nested.go b/internal/server/storage_inventory_nested.go new file mode 100644 index 00000000..1d7c9889 --- /dev/null +++ b/internal/server/storage_inventory_nested.go @@ -0,0 +1,77 @@ +package server + +import ( + "context" + "strings" +) + +// A clean ignored directory may itself be another checkout. Removing its +// parent would erase that independent tree, so retain every containing tree. +// Primary checkouts and recorded workspaces matter as well as linked leaves. +func protectNestedStorageWorktrees(ctx context.Context, trees []storageWorktree, roots []string, registered map[string]bool, problems []string) []storageWorktree { + known := append([]string{}, roots...) + for path := range registered { + known = append(known, path) + } + incomplete := len(problems) > 0 || len(roots) > 256 + canonical := []string{} + for _, path := range known { + if ctx.Err() != nil { + incomplete = true + break + } + resolved, err := storageCanonicalPath(path) + if err != nil { + incomplete = true + continue + } + canonical = append(canonical, resolved) + } + for index := range trees { + if incomplete { + trees[index].Eligible = false + trees[index].Reason = "Workspace discovery incomplete" + continue + } + for _, path := range canonical { + if pathWithin(trees[index].Path, path) { + trees[index].Eligible = false + trees[index].Reason = "Contains another registered worktree or workspace" + break + } + } + } + return trees +} + +// Refresh the same-repository registry at the destructive boundary. A nested +// tree created after preview must not be removed even when Git ignores it. +func storageNestedProtection(ctx context.Context, root string, known []string) string { + listing, err := storageGit(ctx, root, "worktree", "list", "--porcelain", "-z") + if err != nil { + return "Worktree registry unavailable" + } + paths := append([]string{}, known...) + for _, record := range strings.Split(listing, "\x00\x00") { + fields := strings.Split(record, "\x00") + if len(fields) > 0 && strings.HasPrefix(fields[0], "worktree ") { + paths = append(paths, strings.TrimPrefix(fields[0], "worktree ")) + } + } + if len(known) > 256 { + return "Workspace discovery incomplete" + } + for _, path := range paths { + if ctx.Err() != nil { + return "Workspace discovery incomplete" + } + canonical, err := storageCanonicalPath(path) + if err != nil { + return "Workspace path cannot be verified" + } + if pathWithin(root, canonical) { + return "Contains another registered worktree or workspace" + } + } + return "" +} diff --git a/internal/server/storage_inventory_test.go b/internal/server/storage_inventory_test.go index 2ecddabb..031ad52b 100644 --- a/internal/server/storage_inventory_test.go +++ b/internal/server/storage_inventory_test.go @@ -160,3 +160,81 @@ func TestStorageWorktreesRequireMergedCleanInactiveAndUnlocked(t *testing.T) { t.Fatal("main checkout removed") } } + +func storageTestIgnoredParent(t *testing.T) (string, string, storageWorktree) { + t.Helper() + ctx := context.Background() + main, parent := storageTestRepo(t) + storageTestFile(t, filepath.Join(parent, ".gitignore"), "nested/\n") + storageTestGit(t, parent, "add", ".gitignore") + storageTestGit(t, parent, "commit", "-m", "ignore nested workspaces") + storageTestGit(t, main, "merge", "--ff-only", "item") + trees, problems := discoverStorageWorktrees(ctx, []string{main}, nil) + if len(problems) > 0 || len(trees) != 1 || !trees[0].Eligible { + t.Fatalf("parent initially unavailable: %+v %+v", trees, problems) + } + return main, parent, trees[0] +} + +func TestStorageNestedIgnoredRegisteredWorktreeProtectsParent(t *testing.T) { + ctx := context.Background() + main, parent, before := storageTestIgnoredParent(t) + child := filepath.Join(parent, "nested") + storageTestGit(t, main, "worktree", "add", "-b", "nested-child", child) + if status := storageTestGit(t, parent, "status", "--porcelain", "--untracked-files=all"); status != "" { + t.Fatalf("fixture parent dirty: %q", status) + } + trees, problems := discoverStorageWorktrees(ctx, []string{main}, nil) + if len(problems) > 0 || len(trees) != 2 { + t.Fatalf("trees %+v problems %+v", trees, problems) + } + found := false + for _, tree := range trees { + if tree.Path == before.Path { + found = true + if tree.Eligible || !strings.Contains(tree.Reason, "Contains another") { + t.Fatalf("parent not protected: %+v", tree) + } + } + } + if !found { + t.Fatal("parent missing from inventory") + } + // Before has no knowledge of the child. Removal must read Git's live registry. + if err := removeStorageWorktree(ctx, before, nil); err == nil { + t.Fatal("stale parent candidate removed ignored child") + } + if _, err := os.Stat(filepath.Join(child, "tracked")); err != nil { + t.Fatal("child checkout was damaged") + } + storageTestGit(t, main, "show-ref", "--verify", "refs/heads/nested-child") +} + +func TestStorageNestedForeignPrimaryWorkspaceProtectsParent(t *testing.T) { + ctx := context.Background() + main, parent, before := storageTestIgnoredParent(t) + child := filepath.Join(parent, "nested") + if err := os.Mkdir(child, 0700); err != nil { + t.Fatal(err) + } + storageTestGit(t, child, "init", "-b", "main") + storageTestGit(t, child, "config", "user.email", "fixture@example.invalid") + storageTestGit(t, child, "config", "user.name", "Fixture") + storageTestFile(t, filepath.Join(child, "independent"), "preserve foreign primary") + storageTestGit(t, child, "add", "independent") + storageTestGit(t, child, "commit", "-m", "foreign primary") + trees, problems := discoverStorageWorktrees(ctx, []string{main, child}, nil) + if len(problems) > 0 || len(trees) != 1 || trees[0].Eligible { + t.Fatalf("foreign nested workspace not protected: %+v %+v", trees, problems) + } + // A manager passes the fresh full inventory object to destructive revalidation. + if err := removeStorageWorktree(ctx, trees[0], nil); err == nil { + t.Fatal("parent removed foreign recorded workspace") + } + if _, err := os.Stat(filepath.Join(child, "independent")); err != nil { + t.Fatal("foreign primary damaged") + } + if trees[0].Path != before.Path { + t.Fatal("unexpected parent") + } +} From 6abc309009fff4f5412fecbd999a942c9746a60a Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 6 Nov 2026 21:34:55 +0000 Subject: [PATCH 3/3] Expose storage usage and safe persisted age or size retention --- docs/cli.md | 57 +++++ docs/notes/storage-overview-retention.md | 26 ++ internal/pipeline/run_delete.go | 16 +- internal/pipeline/run_delete_cleanup.go | 61 +++++ internal/pipeline/run_delete_cleanup_test.go | 40 +++ internal/pipeline/settings.go | 3 +- internal/pipeline/storage_policy.go | 103 ++++++++ internal/pipeline/storage_policy_test.go | 44 ++++ internal/protocol/message.go | 4 + internal/protocol/runs.go | 7 +- internal/protocol/storage.go | 79 ++++++ internal/server/run_delete.go | 18 +- internal/server/runner.go | 6 + internal/server/storage.go | 202 ++++++++++++++++ internal/server/storage_cleanup.go | 239 ++++++++++++++++++ internal/server/storage_partial_test.go | 53 ++++ internal/server/storage_sources.go | 93 +++++++ internal/server/storage_test.go | 242 +++++++++++++++++++ internal/server/websocket.go | 28 ++- 19 files changed, 1309 insertions(+), 12 deletions(-) create mode 100644 docs/notes/storage-overview-retention.md create mode 100644 internal/pipeline/run_delete_cleanup.go create mode 100644 internal/pipeline/run_delete_cleanup_test.go create mode 100644 internal/pipeline/storage_policy.go create mode 100644 internal/pipeline/storage_policy_test.go create mode 100644 internal/protocol/storage.go create mode 100644 internal/server/storage.go create mode 100644 internal/server/storage_cleanup.go create mode 100644 internal/server/storage_partial_test.go create mode 100644 internal/server/storage_sources.go create mode 100644 internal/server/storage_test.go diff --git a/docs/cli.md b/docs/cli.md index 983334dd..e5dc40cc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1183,3 +1183,60 @@ continues to its next stage or terminal, and its workspace is released. The server applies this at startup and every 30 seconds, so a run parked before an upgrade recovers after the next restart. A native session larger than the 8 MiB export bound (typically inline image payloads read by a tool) is always partial. + +### Storage overview and retention (application RPC) + +The authenticated application uses `get_storage` to read a `storage` reply. +`storage.total_bytes` sums logical regular-file lengths, deduplicated by inode; +it is not allocated filesystem space or the size of the whole disk. Categories +cover database/journals, run artifacts/exports, chat attachments, server-owned +Pi session files, other data under the database directory, linked worktrees, +and known project workspaces. Symlink targets are excluded. Unknown external +harness histories and repositories without a recorded workspace are outside +this inventory. `measurement` is `logical_file_bytes`; `partial`, category +errors and warnings explicitly mark incomplete scans. Repository identities +are remembered so removing run history cannot make sibling worktrees disappear +from the next measurement. + +`set_storage_policy` requires `storage_policy`: + +```json +{"enabled":false,"max_age_days":30,"max_bytes":10737418240,"include_worktrees":false} +``` + +Zero disables the corresponding limit. Days must be 0–36500 and bytes 0–2^50. +Automatic cleanup requires at least one limit. It is disabled by default, +persisted server-side, and runs hourly while enabled; neither startup nor saving +a policy performs cleanup. Age expiry and size pressure are independent triggers: +expired eligible data is selected, then oldest eligible data until the size target +is met or only protected data remains. Run age is completion time (creation only +when no completion time exists); worktree age is the latest measured modification. +Database pages are not vacuumed on the live service, so a size target is best effort. + +`preview_storage_cleanup` optionally accepts that policy and returns +`storage_cleanup_preview` with `storage_cleanup`: policy, total/reclaimable bytes, +target status, candidate/protected counts and items. Each item identifies its kind, +id, path, measured bytes, reason and protected flag. A preview performs no cleanup. +The result contains an opaque `preview_token` valid for ten minutes. +`apply_storage_cleanup` requires the same `storage_policy` and top-level +`storage_cleanup_token`; it returns `storage_cleanup_applied` with removed count, +measured reduction and any refusal/errors. Tokens are one-use; expired or changed +policies require another preview. Apply never adds candidates that appeared after +the confirmed preview, and revalidates every candidate. + +Cleanup removes eligible completed run records and their server-owned files via +the existing transactional deletion path, never definitions or chat history. +Queued/running/waiting, unconfirmed-cleanup, active/exporting or retry-dependent +runs remain protected. Worktrees are opt-in and must be verified linked trees, +clean, unlocked and fully merged into local main. Primary/main checkouts, any +recorded operator-chat directory, active-run directory, live server-data roots, +unmerged work, and trees containing another registered checkout are retained. +Branches are never deleted and Git removal never uses force. New chat/workspace +admission is serialized with the short final removal gate. + +Storage work runs beside the WebSocket read loop with one bounded worker; busy +requests fail promptly instead of blocking chat prompts or permissions. A pass +has a 20-second context, bounded Git output/time and file traversal, and at most +100 deletion candidates. Partial scans cannot claim that the storage target has +been reached. Cleanup refusals and limits remain visible rather than discarding +protected data to satisfy a quota. diff --git a/docs/notes/storage-overview-retention.md b/docs/notes/storage-overview-retention.md new file mode 100644 index 00000000..5ce5e6f5 --- /dev/null +++ b/docs/notes/storage-overview-retention.md @@ -0,0 +1,26 @@ +# Storage overview and retention + +Added read-only inventory and explicit preview/apply RPCs with a persisted, +disabled-by-default hourly policy. The application can choose rolling days, +logical-file byte target, and eligible linked-worktree inclusion. Limits are OR +triggers; retained active/protected storage can exceed the target honestly. + +Inventory is inode-deduplicated, does not follow symlinks, and reports partial +measurements. Known workspace/repository roots survive history deletion. All +operator-session directories are protected without the chat list's 128-row cap. +Worktree removal verifies clean/merged state and the live registry again, retains +nested/primary/locked/in-use trees, uses no force, and preserves branches. + +Manual cleanup uses a ten-minute one-use preview token bound to the reviewed +policy and candidate identities. Later candidates are not added. Completed-run +removal reuses transactional protections and subscriber broadcasts. Automatic +passes use the same selection/revalidation; there is no startup/save cleanup, +chat-history deletion or live VACUUM. + +The protocol runs asynchronously with one worker so scans do not block the +shared socket's chat traffic. Busy requests fail promptly. Full pass context is +20 seconds; final worktree admission/removal has its own three-second budget. + +No production scans, policy activation, cleanup or service restart were performed +while developing this change. Backend delivery requires the operator's safe +active-work deployment gate; the deployed service is never restarted for tests. diff --git a/internal/pipeline/run_delete.go b/internal/pipeline/run_delete.go index 95b6fd12..507b4093 100644 --- a/internal/pipeline/run_delete.go +++ b/internal/pipeline/run_delete.go @@ -127,15 +127,23 @@ func (s *Store) DeleteRuns(ctx context.Context, ids []int64, artifactsRoot strin result.FreedBytes += sizes[id] } if stage.dir != "" { - if err := os.RemoveAll(stage.dir); err != nil { - _, remaining, _ := retentionWalkSize(stage.dir) + if err := removeRunTree(ctx, stage.dir); err != nil { + remaining := result.FreedBytes + if ctx.Err() == nil { + _, remaining, _ = retentionWalkSize(stage.dir) + } result.FreedBytes = max(result.FreedBytes-remaining, 0) + result.RemainingFileBytes += remaining } } for _, dir := range piDirs { - if err := os.RemoveAll(dir); err != nil { - _, remaining, _ := retentionWalkSize(dir) + if err := removeRunTree(ctx, dir); err != nil { + remaining := result.FreedBytes + if ctx.Err() == nil { + _, remaining, _ = retentionWalkSize(dir) + } result.FreedBytes = max(result.FreedBytes-remaining, 0) + result.RemainingFileBytes += remaining } } return result, nil diff --git a/internal/pipeline/run_delete_cleanup.go b/internal/pipeline/run_delete_cleanup.go new file mode 100644 index 00000000..fd92a38b --- /dev/null +++ b/internal/pipeline/run_delete_cleanup.go @@ -0,0 +1,61 @@ +package pipeline + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" +) + +// removeRunTree preserves the owned-tree cleanup behavior of RemoveAll while +// honoring maintenance cancellation between directory chunks and entries. +// A canceled pass leaves staged data measured/reported, never claimed freed. +func removeRunTree(ctx context.Context, path string) error { + if err := ctx.Err(); err != nil { + return err + } + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() { + return os.Remove(path) + } + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + actual, err := directory.Stat() + if err != nil { + return err + } + if !os.SameFile(info, actual) { + return fmt.Errorf("cleanup directory changed: %s", path) + } + for { + if err := ctx.Err(); err != nil { + return err + } + entries, err := directory.ReadDir(256) + if err != nil && err != io.EOF { + return err + } + for _, entry := range entries { + if err := removeRunTree(ctx, filepath.Join(path, entry.Name())); err != nil { + return err + } + } + if err == io.EOF { + break + } + } + if err := ctx.Err(); err != nil { + return err + } + return os.Remove(path) +} diff --git a/internal/pipeline/run_delete_cleanup_test.go b/internal/pipeline/run_delete_cleanup_test.go new file mode 100644 index 00000000..f62a2fc9 --- /dev/null +++ b/internal/pipeline/run_delete_cleanup_test.go @@ -0,0 +1,40 @@ +package pipeline + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestRunTreeCleanupCancellationAndSymlinks(t *testing.T) { + root := t.TempDir() + owned := filepath.Join(root, "owned") + outside := filepath.Join(root, "outside") + if err := os.MkdirAll(owned, 0700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outside, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(outside, "keep"), []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(owned, "link")); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := removeRunTree(ctx, owned); err == nil { + t.Fatal("canceled cleanup accepted") + } + if _, err := os.Stat(owned); err != nil { + t.Fatal("canceled cleanup removed tree") + } + if err := removeRunTree(context.Background(), owned); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(outside, "keep")); err != nil { + t.Fatal("cleanup followed symlink") + } +} diff --git a/internal/pipeline/settings.go b/internal/pipeline/settings.go index 98009aec..cc7a720e 100644 --- a/internal/pipeline/settings.go +++ b/internal/pipeline/settings.go @@ -15,7 +15,8 @@ const openCodeEndpointSetting = "opencode_endpoint" // schema or ownership table is touched. func (s *Store) initServerSettings() error { _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS server_settings ( -key TEXT PRIMARY KEY, value TEXT NOT NULL);`) +key TEXT PRIMARY KEY, value TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS storage_workspaces(path TEXT PRIMARY KEY);`) return err } diff --git a/internal/pipeline/storage_policy.go b/internal/pipeline/storage_policy.go new file mode 100644 index 00000000..e7d7e93e --- /dev/null +++ b/internal/pipeline/storage_policy.go @@ -0,0 +1,103 @@ +package pipeline + +import ( + "context" + "database/sql" + "encoding/json" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +const storagePolicySetting = "storage_retention_v1" + +func (s *Store) StoragePolicy(ctx context.Context) (protocol.StoragePolicy, error) { + var value string + var policy protocol.StoragePolicy + err := s.db.QueryRowContext(ctx, "SELECT value FROM server_settings WHERE key=?", storagePolicySetting).Scan(&value) + if err == sql.ErrNoRows { + return policy, nil + } + if err != nil { + return policy, err + } + if err = json.Unmarshal([]byte(value), &policy); err != nil { + return policy, err + } + return policy, policy.Validate() +} +func (s *Store) SaveStoragePolicy(ctx context.Context, p protocol.StoragePolicy) error { + if err := p.Validate(); err != nil { + return err + } + value, err := json.Marshal(p) + if err != nil { + return err + } + _, err = s.db.ExecContext(ctx, `INSERT INTO server_settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, storagePolicySetting, string(value)) + return err +} + +// PreviewRunDeletion uses the same eligibility checks as deletion. Its byte +// result is only an eligibility check; storage scans measure reclaimable files. +func (s *Store) PreviewRunDeletion(ctx context.Context, id int64, root string) (int64, string, error) { + reason, _, err := deleteRefusal(ctx, s.db, id) + if err != nil || reason != "" { + return 0, reason, err + } + return 0, "", nil +} + +// DatabasePath identifies the server-owned database for read-only inventory. +func (s *Store) DatabasePath() string { return s.path } + +// StorageOperatorDirectories deliberately has no chat-list limit: an old or +// closed session can still be resumed and must protect its checkout. +func (s *Store) StorageOperatorDirectories(ctx context.Context) ([]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT directory FROM operator_sessions WHERE directory<>''`) + if err != nil { + return nil, err + } + defer rows.Close() + var paths []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + paths = append(paths, path) + } + return paths, rows.Err() +} + +// RememberStorageWorkspaces keeps inventory coverage stable after run history +// is removed. It stores identities only, never creates directories or runs. +func (s *Store) RememberStorageWorkspaces(ctx context.Context, paths []string) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, path := range paths { + if path != "" { + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO storage_workspaces(path) VALUES(?)`, path); err != nil { + return err + } + } + } + return tx.Commit() +} +func (s *Store) StorageWorkspacePaths(ctx context.Context) ([]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT path FROM storage_workspaces ORDER BY path`) + if err != nil { + return nil, err + } + defer rows.Close() + var paths []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + paths = append(paths, path) + } + return paths, rows.Err() +} diff --git a/internal/pipeline/storage_policy_test.go b/internal/pipeline/storage_policy_test.go new file mode 100644 index 00000000..5076832e --- /dev/null +++ b/internal/pipeline/storage_policy_test.go @@ -0,0 +1,44 @@ +package pipeline + +import ( + "context" + "git.a-wan.net/alan/pipelines/internal/protocol" + "path/filepath" + "testing" +) + +func TestStoragePolicyPersistsAndRejectsInvalid(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "store.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + p, err := store.StoragePolicy(ctx) + if err != nil || p.Enabled { + t.Fatalf("default %#v %v", p, err) + } + wanted := protocol.StoragePolicy{Enabled: true, MaxAgeDays: 30, MaxBytes: 1 << 30, IncludeWorktrees: true} + if err := store.SaveStoragePolicy(ctx, wanted); err != nil { + t.Fatal(err) + } + store.Close() + store, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer store.Close() + got, err := store.StoragePolicy(ctx) + if err != nil || got != wanted { + t.Fatalf("persisted %#v %v", got, err) + } + for _, bad := range []protocol.StoragePolicy{{Enabled: true}, {MaxAgeDays: -1}, {MaxAgeDays: 36501}, {MaxBytes: -1}, {MaxBytes: 1 << 51}} { + if err := store.SaveStoragePolicy(ctx, bad); err == nil { + t.Fatalf("accepted %#v", bad) + } + } + got, _ = store.StoragePolicy(ctx) + if got != wanted { + t.Fatal("invalid save changed policy") + } +} diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 8a7d6f74..557cde70 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -127,6 +127,10 @@ type PipelineSummary struct { // are optional so older peers ignore them. See search.go for the per-type payloads // and the ranking/kind/bound vocabulary. type Message struct { + Storage *StorageOverview `json:"storage,omitempty"` + StoragePolicy *StoragePolicy `json:"storage_policy,omitempty"` + StorageCleanup *StorageCleanupResult `json:"storage_cleanup,omitempty"` + StorageCleanupToken string `json:"storage_cleanup_token,omitempty"` Schedule *SavedSchedule `json:"schedule,omitempty"` Schedules []SavedSchedule `json:"schedules,omitempty"` ScheduleID int64 `json:"schedule_id,omitempty"` diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 1879f225..a1726d0b 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -32,9 +32,10 @@ const ( // the deleted runs as measured immediately before deletion. All three are // always present. type RunsDeletedResult struct { - Deleted []int64 `json:"deleted"` - Refused []RunDeleteRefusal `json:"refused"` - FreedBytes int64 `json:"freed_bytes"` + RemainingFileBytes int64 `json:"remaining_file_bytes,omitempty"` + Deleted []int64 `json:"deleted"` + Refused []RunDeleteRefusal `json:"refused"` + FreedBytes int64 `json:"freed_bytes"` } // RunDeleteRefusal is one requested run id delete_runs did not remove. diff --git a/internal/protocol/storage.go b/internal/protocol/storage.go new file mode 100644 index 00000000..ed7b6e97 --- /dev/null +++ b/internal/protocol/storage.go @@ -0,0 +1,79 @@ +package protocol + +import "fmt" + +const ( + GetStorage = "get_storage" + SetStoragePolicy = "set_storage_policy" + PreviewStorageCleanup = "preview_storage_cleanup" + ApplyStorageCleanup = "apply_storage_cleanup" + StorageReply = "storage" + StoragePolicySaved = "storage_policy_saved" + StorageCleanupPreview = "storage_cleanup_preview" + StorageCleanupApplied = "storage_cleanup_applied" +) + +// StoragePolicy is a server-wide opt-in policy. Age expiry and the size target +// are independent triggers; protected work is never removed to meet a target. +type StoragePolicy struct { + Enabled bool `json:"enabled"` + MaxAgeDays int `json:"max_age_days"` + MaxBytes int64 `json:"max_bytes"` + IncludeWorktrees bool `json:"include_worktrees"` +} + +func (p StoragePolicy) Validate() error { + if p.MaxAgeDays < 0 || p.MaxAgeDays > 36500 { + return fmt.Errorf("max_age_days must be between 0 and 36500") + } + if p.MaxBytes < 0 || p.MaxBytes > 1<<50 { + return fmt.Errorf("max_bytes must be between 0 and 1125899906842624") + } + if p.Enabled && p.MaxAgeDays == 0 && p.MaxBytes == 0 { + return fmt.Errorf("automatic retention requires an age or size limit") + } + return nil +} + +type StorageCategory struct { + ID string `json:"id"` + Label string `json:"label"` + Path string `json:"path,omitempty"` + Bytes int64 `json:"bytes"` + Files int64 `json:"files"` + Partial bool `json:"partial,omitempty"` + Error string `json:"error,omitempty"` +} +type StorageOverview struct { + Measurement string `json:"measurement"` + TotalBytes int64 `json:"total_bytes"` + Categories []StorageCategory `json:"categories"` + Policy StoragePolicy `json:"policy"` + MeasuredAt string `json:"measured_at"` + Partial bool `json:"partial"` + Warnings []string `json:"warnings,omitempty"` + AutomaticIntervalSeconds int `json:"automatic_interval_seconds"` +} +type StorageCleanupItem struct { + Kind string `json:"kind"` + ID string `json:"id"` + Path string `json:"path,omitempty"` + Bytes int64 `json:"bytes"` + Reason string `json:"reason,omitempty"` + Protected bool `json:"protected"` +} +type StorageCleanupResult struct { + Policy StoragePolicy `json:"policy"` + TotalBytes int64 `json:"total_bytes"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` + TargetBytes int64 `json:"target_bytes"` + TargetReached bool `json:"target_reached"` + CandidateCount int `json:"candidate_count"` + ProtectedCount int `json:"protected_count"` + Items []StorageCleanupItem `json:"items"` + PreviewToken string `json:"preview_token,omitempty"` + RemovedCount int `json:"removed_count"` + RemovedBytes int64 `json:"removed_bytes"` + Applied bool `json:"applied"` + Errors []string `json:"errors,omitempty"` +} diff --git a/internal/server/run_delete.go b/internal/server/run_delete.go index d2f80076..6f0ef65c 100644 --- a/internal/server/run_delete.go +++ b/internal/server/run_delete.go @@ -2,6 +2,7 @@ package server import ( "context" + "fmt" "log" "time" @@ -103,8 +104,19 @@ func (r *runner) finishedSize(id int64) int64 { // then tells every subscribed client which runs are gone. It holds r.mu so a run // cannot become active between the executing check and the delete. func (r *runner) deleteRuns(ids []int64) (*protocol.RunsDeletedResult, error) { - r.mu.Lock() - result, err := r.store.DeleteRuns(r.ctx, ids, r.artifactsRoot, func(id int64) string { + return r.deleteRunsContext(r.ctx, ids, false) +} + +// Storage maintenance uses its own deadline and never queues behind execution. +func (r *runner) deleteRunsContext(ctx context.Context, ids []int64, try bool) (*protocol.RunsDeletedResult, error) { + if try { + if !r.mu.TryLock() { + return nil, fmt.Errorf("run maintenance is busy; retry shortly") + } + } else { + r.mu.Lock() + } + result, err := r.store.DeleteRuns(ctx, ids, r.artifactsRoot, func(id int64) string { if _, ok := r.active[id]; ok { return "run is executing" } @@ -126,7 +138,7 @@ func (r *runner) deleteRuns(ids []int64) (*protocol.RunsDeletedResult, error) { r.notifyMu.Unlock() // A deleted run's notifications are gone on every device. if r.notes != nil { - c, cancel := context.WithTimeout(r.ctx, 5*time.Second) + c, cancel := context.WithTimeout(ctx, 5*time.Second) r.notes.clearFor(c, pipeline.AppNotificationFilter{RunID: id}) cancel() } diff --git a/internal/server/runner.go b/internal/server/runner.go index 6b8df66c..0d141859 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -313,6 +313,12 @@ func (r *runner) startWithSchedule(request protocol.Message, origin *pipeline.Sc return nil, err } } + // Storage maintenance may have removed a clean inactive tree while an + // admission was generating its title. Refuse that vanished workspace under + // the same lock used by removal instead of launching into a missing tree. + if info, err := os.Stat(workspace); err != nil || !info.IsDir() { + return nil, fmt.Errorf("run workspace is no longer available: %s", workspace) + } id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Schedule: origin, Model: request.Model, StageModels: request.StageModels, NotifyMode: request.NotifyMode, OutputFormat: request.OutputFormat}, runTitle, retryOf...) if err != nil { return nil, err diff --git a/internal/server/storage.go b/internal/server/storage.go new file mode 100644 index 00000000..8ef32e45 --- /dev/null +++ b/internal/server/storage.go @@ -0,0 +1,202 @@ +package server + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log" + "path/filepath" + "sync" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +const storageInterval = time.Hour +const storagePreviewLifetime = 10 * time.Minute + +type storagePreview struct { + result protocol.StorageCleanupResult + expires time.Time +} +type storageManager struct { + scan func(context.Context, storageInventoryConfig) (storageInventory, error) + worker chan struct{} + roots *storageInventoryConfig + workspaceMu sync.RWMutex + mu sync.Mutex + runner *runner + store *pipeline.Store + previews map[string]storagePreview +} + +func newStorageManager(r *runner) *storageManager { + return &storageManager{runner: r, store: r.store, previews: map[string]storagePreview{}, worker: make(chan struct{}, 1)} +} + +// dispatch bounds slow storage work to one goroutine without blocking the +// shared socket read loop. The transport's write callback serializes replies. +func (m *storageManager) dispatch(ctx context.Context, request protocol.Message, write func(protocol.Message) error) { + select { + case m.worker <- struct{}{}: + go func() { defer func() { <-m.worker }(); _ = write(m.handle(ctx, request)) }() + default: + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management is busy; retry shortly"}) + } +} +func isStorageRequest(t string) bool { + return t == protocol.GetStorage || t == protocol.SetStoragePolicy || t == protocol.PreviewStorageCleanup || t == protocol.ApplyStorageCleanup +} +func (m *storageManager) handle(ctx context.Context, request protocol.Message) protocol.Message { + if !m.mu.TryLock() { + return protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management is busy; retry shortly"} + } + defer m.mu.Unlock() + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + response := protocol.Message{ID: request.ID} + var err error + switch request.Type { + case protocol.GetStorage: + response.Type = protocol.StorageReply + response.Storage, err = m.overview(ctx) + case protocol.SetStoragePolicy: + response.Type = protocol.StoragePolicySaved + if request.StoragePolicy == nil { + err = fmt.Errorf("storage_policy is required") + break + } + err = m.store.SaveStoragePolicy(ctx, *request.StoragePolicy) + if err == nil { + response.Storage, err = m.overview(ctx) + } + case protocol.PreviewStorageCleanup: + response.Type = protocol.StorageCleanupPreview + var policy protocol.StoragePolicy + policy, err = m.policy(ctx, request.StoragePolicy) + if err == nil { + response.StorageCleanup, err = m.preview(ctx, policy) + if err == nil { + response.StorageCleanup.PreviewToken = m.remember(*response.StorageCleanup) + } + } + case protocol.ApplyStorageCleanup: + response.Type = protocol.StorageCleanupApplied + var saved storagePreview + saved, err = m.consume(request.StorageCleanupToken, request.StoragePolicy) + if err == nil { + response.StorageCleanup, err = m.apply(ctx, saved.result) + } + } + if err != nil { + response.Type = protocol.Error + response.Error = err.Error() + } + return response +} +func (m *storageManager) policy(ctx context.Context, supplied *protocol.StoragePolicy) (protocol.StoragePolicy, error) { + if supplied != nil { + return *supplied, supplied.Validate() + } + return m.store.StoragePolicy(ctx) +} +func (m *storageManager) remember(result protocol.StorageCleanupResult) string { + for token, p := range m.previews { + if time.Now().After(p.expires) { + delete(m.previews, token) + } + } + // A preview is small and short lived; bound clients repeatedly requesting it. + if len(m.previews) >= 32 { + for token := range m.previews { + delete(m.previews, token) + break + } + } + var key [24]byte + if _, err := rand.Read(key[:]); err != nil { + return "" + } + token := hex.EncodeToString(key[:]) + m.previews[token] = storagePreview{result: result, expires: time.Now().Add(storagePreviewLifetime)} + return token +} +func (m *storageManager) consume(token string, policy *protocol.StoragePolicy) (storagePreview, error) { + p, ok := m.previews[token] + if !ok || time.Now().After(p.expires) { + return p, fmt.Errorf("cleanup preview expired; preview cleanup again") + } + if policy == nil || *policy != p.result.Policy { + return p, fmt.Errorf("retention policy changed; preview cleanup again") + } + delete(m.previews, token) + return p, nil +} +func (m *storageManager) overview(ctx context.Context) (*protocol.StorageOverview, error) { + policy, err := m.store.StoragePolicy(ctx) + if err != nil { + return nil, err + } + inventory, err := m.inventory(ctx) + if err != nil { + return nil, err + } + result := &protocol.StorageOverview{Measurement: "logical_file_bytes", TotalBytes: inventory.TotalBytes, Policy: policy, MeasuredAt: time.Now().UTC().Format(time.RFC3339), Partial: inventory.Partial, Warnings: inventory.Errors, AutomaticIntervalSeconds: int(storageInterval / time.Second), Categories: []protocol.StorageCategory{}} + for _, c := range inventory.Categories { + result.Categories = append(result.Categories, protocol.StorageCategory{ID: c.ID, Label: c.Label, Path: c.Path, Bytes: c.Bytes, Files: c.Files, Partial: c.Partial, Error: c.Error}) + } + return result, nil +} +func storageTerminal(status string) bool { + switch status { + case "succeeded", "failed", "canceled", "interrupted": + return true + } + return false +} +func (m *storageManager) loop(ctx context.Context) { + defer m.runner.schedulerWG.Done() + ticker := time.NewTicker(storageInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-m.store.Done(): + return + case <-ticker.C: + if !m.mu.TryLock() { + continue + } + cycle, cancel := context.WithTimeout(ctx, 20*time.Second) + err := m.automaticPass(cycle) + cancel() + m.mu.Unlock() + if err != nil && ctx.Err() == nil { + log.Printf("storage retention: %v", err) + } + } + } +} + +// canonicalStoragePath supplies the consistent key used by preview identities. +func canonicalStoragePath(path string) string { return filepath.Clean(path) } + +// automaticPass is invoked only by the hourly tick, never at startup or save. +func (m *storageManager) automaticPass(ctx context.Context) error { + policy, err := m.store.StoragePolicy(ctx) + if err != nil || !policy.Enabled { + return err + } + preview, err := m.preview(ctx, policy) + if err != nil || preview.CandidateCount == 0 { + return err + } + result, err := m.apply(ctx, *preview) + if err == nil && len(result.Errors) > 0 { + return fmt.Errorf("retention incomplete: %s", result.Errors[0]) + } + return err +} diff --git a/internal/server/storage_cleanup.go b/internal/server/storage_cleanup.go new file mode 100644 index 00000000..a82eb84f --- /dev/null +++ b/internal/server/storage_cleanup.go @@ -0,0 +1,239 @@ +package server + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strconv" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type storageCandidate struct { + item protocol.StorageCleanupItem + timestamp time.Time +} + +func (m *storageManager) preview(ctx context.Context, policy protocol.StoragePolicy) (*protocol.StorageCleanupResult, error) { + if err := policy.Validate(); err != nil { + return nil, err + } + inventory, err := m.inventory(ctx) + if err != nil { + return nil, err + } + result := &protocol.StorageCleanupResult{Policy: policy, TotalBytes: inventory.TotalBytes, TargetBytes: policy.MaxBytes, Items: []protocol.StorageCleanupItem{}, Errors: append([]string{}, inventory.Errors...)} + runs, err := m.store.ListRuns(ctx) + if err != nil { + return nil, err + } + recordProtected := func(item protocol.StorageCleanupItem) { + result.ProtectedCount++ + if result.ProtectedCount <= 200 { + result.Items = append(result.Items, item) + } + if result.ProtectedCount == 201 { + result.Errors = append(result.Errors, "Only the first 200 protected items are shown; the protected count includes all items.") + } + } + var candidates []storageCandidate + artifactWalker := storageWalker{ctx: ctx, seen: map[string]bool{}, inodes: map[[2]uint64]bool{}} + for _, run := range runs { + if err := ctx.Err(); err != nil { + return nil, err + } + _, reason, err := m.store.PreviewRunDeletion(ctx, run.ID, m.runner.artifactsRoot) + if err != nil { + return nil, err + } + item := protocol.StorageCleanupItem{Kind: "run", ID: strconv.FormatInt(run.ID, 10), Path: filepath.Join(m.runner.artifactsRoot, strconv.FormatInt(run.ID, 10)), Protected: reason != "", Reason: reason} + if reason != "" { + recordProtected(item) + continue + } + measured := storageCategory{} + artifactWalker.measure(item.Path, &measured) + bytes, partial := measured.Bytes, measured.Partial + var problems []string + if measured.Error != "" { + problems = append(problems, measured.Error) + } + if partial { + item.Protected = true + item.Reason = "artifact measurement incomplete" + recordProtected(item) + result.Errors = append(result.Errors, problems...) + continue + } + item.Bytes = bytes + stamp := run.FinishedAt + if stamp == "" { + stamp = run.CreatedAt + } + candidates = append(candidates, storageCandidate{item: item, timestamp: storageTime(stamp)}) + } + if policy.IncludeWorktrees { + for _, tree := range inventory.Worktrees { + item := protocol.StorageCleanupItem{Kind: "worktree", ID: canonicalStoragePath(tree.Path), Path: tree.Path, Bytes: tree.Bytes, Protected: !tree.Eligible, Reason: tree.Reason} + if !tree.Eligible { + recordProtected(item) + continue + } + candidates = append(candidates, storageCandidate{item: item, timestamp: tree.ModifiedAt}) + } + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].timestamp.Before(candidates[j].timestamp) }) + remaining := inventory.TotalBytes + cutoff := time.Now().Add(-time.Duration(policy.MaxAgeDays) * 24 * time.Hour) + for _, candidate := range candidates { + expired := policy.MaxAgeDays > 0 && !candidate.timestamp.IsZero() && candidate.timestamp.Before(cutoff) + over := policy.MaxBytes > 0 && remaining > policy.MaxBytes + if !expired && !over { + continue + } + if result.CandidateCount >= 100 { + result.Errors = append(result.Errors, "Cleanup is limited to 100 items per pass; preview again for remaining items.") + break + } + item := candidate.item + if expired { + item.Reason = "older than retention age" + } else { + item.Reason = "oldest eligible data above storage target" + } + result.Items = append(result.Items, item) + result.CandidateCount++ + result.ReclaimableBytes += item.Bytes + remaining = max(int64(0), remaining-item.Bytes) + } + result.TargetReached = policy.MaxBytes == 0 || remaining <= policy.MaxBytes + if inventory.Partial { + result.Errors = append(result.Errors, "Storage measurement is partial; the total and target result are lower bounds.") + result.TargetReached = false + } + return result, nil +} +func storageTime(value string) time.Time { + for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05"} { + if t, err := time.Parse(layout, value); err == nil { + return t + } + } + return time.Time{} +} +func (m *storageManager) apply(ctx context.Context, preview protocol.StorageCleanupResult) (*protocol.StorageCleanupResult, error) { + result := preview + result.PreviewToken = "" + result.Applied = true + before, err := m.inventory(ctx) + if err != nil { + return nil, err + } + var ids []int64 + for _, item := range preview.Items { + if item.Protected || item.Kind != "run" { + continue + } + id, err := strconv.ParseInt(item.ID, 10, 64) + if err != nil { + return nil, err + } + ids = append(ids, id) + } + if len(ids) > 0 { + deleted, err := m.runner.deleteRunsContext(ctx, ids, true) + if err != nil { + return nil, err + } + result.RemovedCount += len(deleted.Deleted) + removed := map[string]bool{} + for _, id := range deleted.Deleted { + removed[strconv.FormatInt(id, 10)] = true + } + for _, item := range preview.Items { + if item.Kind == "run" && removed[item.ID] { + result.RemovedBytes += item.Bytes + } + } + result.RemovedBytes = max(int64(0), result.RemovedBytes-deleted.RemainingFileBytes) + if deleted.RemainingFileBytes > 0 { + result.Errors = append(result.Errors, "Some staged files remain after run deletion; their bytes are not counted as removed.") + } + for _, refused := range deleted.Refused { + result.Errors = append(result.Errors, fmt.Sprintf("Run %d retained: %s", refused.RunID, refused.Reason)) + } + } + for _, item := range preview.Items { + if item.Protected || item.Kind != "worktree" { + continue + } + if err := ctx.Err(); err != nil { + result.Errors = append(result.Errors, err.Error()) + break + } + // Admission and removal are serialized against the runner. Chat creation + // takes the workspace gate in the websocket dispatcher before recording paths. + if !m.workspaceMu.TryLock() { + result.Errors = append(result.Errors, "Worktree retained: chat workspace admission is busy") + continue + } + if !m.runner.mu.TryLock() { + m.workspaceMu.Unlock() + result.Errors = append(result.Errors, "Worktree retained: run workspace admission is busy") + continue + } + removalCtx, cancelRemoval := context.WithTimeout(ctx, 3*time.Second) + config, err := m.inventoryConfig(removalCtx) + if err == nil { + for _, active := range m.runner.active { + config.ProtectedWorkspaces = append(config.ProtectedWorkspaces, active.workspace) + } + var tree storageWorktree + found := false + for _, known := range before.Worktrees { + if canonicalStoragePath(known.Path) == canonicalStoragePath(item.Path) { + tree = known + found = true + break + } + } + if !found { + err = fmt.Errorf("worktree is no longer inventoried") + } else { + err = removeStorageWorktree(removalCtx, tree, config.ProtectedWorkspaces) + } + } + cancelRemoval() + m.runner.mu.Unlock() + m.workspaceMu.Unlock() + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("Worktree %s retained: %v", item.Path, err)) + } else { + result.RemovedCount++ + result.RemovedBytes += treeBytes(before.Worktrees, item.Path) + } + } + after, err := m.inventory(ctx) + if err != nil { + result.Errors = append(result.Errors, err.Error()) + result.TargetReached = false + return &result, nil + } + result.TotalBytes = after.TotalBytes + // Sum only successfully removed measured file attribution. A partial + // post-scan or concurrent writes must never fabricate a reclaimed total. + result.TargetReached = !after.Partial && (result.Policy.MaxBytes == 0 || after.TotalBytes <= result.Policy.MaxBytes) + result.Errors = append(result.Errors, after.Errors...) + return &result, nil +} + +func treeBytes(trees []storageWorktree, path string) int64 { + for _, tree := range trees { + if canonicalStoragePath(tree.Path) == canonicalStoragePath(path) { + return tree.Bytes + } + } + return 0 +} diff --git a/internal/server/storage_partial_test.go b/internal/server/storage_partial_test.go new file mode 100644 index 00000000..9ce1097f --- /dev/null +++ b/internal/server/storage_partial_test.go @@ -0,0 +1,53 @@ +package server + +import ( + "context" + "git.a-wan.net/alan/pipelines/internal/protocol" + "testing" + "time" +) + +func TestStoragePartialMeasurementSurvivesBudgetExpiry(t *testing.T) { + f := newStorageFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f.m.scan = func(context.Context, storageInventoryConfig) (storageInventory, error) { + cancel() + return storageInventory{TotalBytes: 123, Partial: true, Categories: []storageCategory{{ID: "worktrees", Bytes: 123, Partial: true}}, Worktrees: []storageWorktree{{Path: f.workspace, Repository: f.workspace}}, Errors: []string{"measurement budget expired"}}, nil + } + got, err := f.m.overview(ctx) + if err != nil || got == nil || !got.Partial || got.TotalBytes != 123 { + t.Fatalf("partial %+v %v", got, err) + } +} +func TestStoragePartialPostScanCannotInventRemovedBytes(t *testing.T) { + f := newStorageFixture(t) + old := time.Now().Add(-40 * 24 * time.Hour) + id := f.run(t, "partial-postscan", "succeeded", old, old) + policy := protocol.StoragePolicy{MaxAgeDays: 7} + preview, err := f.m.preview(context.Background(), policy) + if err != nil { + t.Fatal(err) + } + if preview.CandidateCount != 1 { + t.Fatalf("preview %+v", preview) + } + scans := 0 + f.m.scan = func(ctx context.Context, c storageInventoryConfig) (storageInventory, error) { + scans++ + if scans == 1 { + return storageInventory{TotalBytes: 1 << 40}, nil + } + return storageInventory{Partial: true, TotalBytes: 0, Errors: []string{"partial after scan"}}, nil + } + result, err := f.m.apply(context.Background(), *preview) + if err != nil { + t.Fatal(err) + } + if result.RemovedCount != 1 || result.RemovedBytes != 4096 || result.TargetReached { + t.Fatalf("partial reclaim %+v", result) + } + if _, err := f.m.store.GetRun(context.Background(), id); err == nil { + t.Fatal("run was not removed") + } +} diff --git a/internal/server/storage_sources.go b/internal/server/storage_sources.go new file mode 100644 index 00000000..665a7ad4 --- /dev/null +++ b/internal/server/storage_sources.go @@ -0,0 +1,93 @@ +package server + +import ( + "context" + "os" + "path/filepath" +) + +func (m *storageManager) inventory(ctx context.Context) (storageInventory, error) { + config, err := m.inventoryConfig(ctx) + if err != nil { + return storageInventory{}, err + } + scan := m.scan + if scan == nil { + scan = inspectStorage + } + inventory, err := scan(ctx, config) + if err != nil { + return inventory, err + } + // Keep discovered repository identities after deleting the last referencing + // run/worktree; otherwise the next total could silently lose sibling trees. + var observed []string + for _, tree := range inventory.Worktrees { + observed = append(observed, tree.Path, tree.Repository) + } + if len(observed) > 0 && ctx.Err() == nil { + if err := m.store.RememberStorageWorkspaces(ctx, observed); err != nil { + return inventory, err + } + } + return inventory, nil +} +func (m *storageManager) inventoryConfig(ctx context.Context) (storageInventoryConfig, error) { + config := storageInventoryConfig{DataRoot: m.store.Directory(), ArtifactsRoot: m.runner.artifactsRoot} + // The database path is obtained from the store, never supplied by a client. + config.DatabasePath = m.store.DatabasePath() + var err error + if m.roots != nil { + config = *m.roots + config.DatabasePath = m.store.DatabasePath() + config.DataRoot = m.store.Directory() + } else { + cache, cacheErr := os.UserCacheDir() + if cacheErr != nil { + return config, cacheErr + } + config.SessionsRoot = filepath.Join(cache, "pipelines", "pi-sessions") + config.AttachmentsRoot, err = chatAttachmentDir() + if err != nil { + return config, err + } + } + config.ProtectedWorkspaces = append(config.ProtectedWorkspaces, config.DataRoot, config.ArtifactsRoot, config.AttachmentsRoot, config.SessionsRoot) + runs, err := m.store.ListRuns(ctx) + if err != nil { + return config, err + } + for _, run := range runs { + config.WorkspaceRoots = append(config.WorkspaceRoots, run.Workspace) + if !storageTerminal(run.Status) || run.CleanupUnconfirmed { + config.ProtectedWorkspaces = append(config.ProtectedWorkspaces, run.Workspace) + } + } + directories, err := m.store.StorageOperatorDirectories(ctx) + if err != nil { + return config, err + } + config.WorkspaceRoots = append(config.WorkspaceRoots, directories...) + config.ProtectedWorkspaces = append(config.ProtectedWorkspaces, directories...) + if err := m.store.RememberStorageWorkspaces(ctx, config.WorkspaceRoots); err != nil { + return config, err + } + remembered, err := m.store.StorageWorkspacePaths(ctx) + if err != nil { + return config, err + } + config.WorkspaceRoots = append(config.WorkspaceRoots, remembered...) + seen := map[string]bool{} + unique := config.WorkspaceRoots[:0] + for _, path := range config.WorkspaceRoots { + if path != "" { + path = filepath.Clean(path) + if !seen[path] { + unique = append(unique, path) + seen[path] = true + } + } + } + config.WorkspaceRoots = unique + return config, nil +} diff --git a/internal/server/storage_test.go b/internal/server/storage_test.go new file mode 100644 index 00000000..d4994d14 --- /dev/null +++ b/internal/server/storage_test.go @@ -0,0 +1,242 @@ +package server + +import ( + "context" + "database/sql" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type storageFixture struct { + m *storageManager + definition *protocol.PipelineDetail + workspace string +} + +func newStorageFixture(t *testing.T) storageFixture { + t.Helper() + ctx := context.Background() + root := t.TempDir() + store, err := pipeline.Open(filepath.Join(root, "data", "storage.db")) + if err != nil { + if err := os.MkdirAll(filepath.Join(root, "data"), 0700); err != nil { + t.Fatal(err) + } + store, err = pipeline.Open(filepath.Join(root, "data", "storage.db")) + if err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { store.Close() }) + if _, err := store.Submit(ctx, "storage-fixture", pipeline.Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + list, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + d, err := store.Get(ctx, list[0].ID, 1) + if err != nil { + t.Fatal(err) + } + r := &runner{ctx: ctx, store: store, artifactsRoot: filepath.Join(root, "artifacts"), active: map[int64]activeRun{}} + m := newStorageManager(r) + m.roots = &storageInventoryConfig{AttachmentsRoot: filepath.Join(root, "attachments"), SessionsRoot: filepath.Join(root, "sessions")} + workspace := filepath.Join(root, "workspace") + if err := os.MkdirAll(workspace, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workspace, "source.txt"), []byte("source is preserved"), 0600); err != nil { + t.Fatal(err) + } + return storageFixture{m, d, workspace} +} +func (f storageFixture) run(t *testing.T, key, status string, created, finished time.Time) int64 { + t.Helper() + ctx := context.Background() + id, _, err := f.m.store.CreateRun(ctx, f.definition, key, f.workspace, false, nil, map[string]any{"prompt": "fixture"}) + if err != nil { + t.Fatal(err) + } + if storageTerminal(status) { + if err := f.m.store.FinishRun(ctx, id, status, "success", "", nil); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite", f.m.store.DatabasePath()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`UPDATE pipeline_runs SET created_at=?,finished_at=? WHERE id=?`, created.UTC().Format(time.RFC3339), finished.UTC().Format(time.RFC3339), id); err != nil { + t.Fatal(err) + } + path := filepath.Join(f.m.runner.artifactsRoot, strconv.FormatInt(id, 10)) + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "report.txt"), make([]byte, 4096), 0600); err != nil { + t.Fatal(err) + } + return id +} +func TestStorageRetentionPreviewBoundApplyAndCompletionAge(t *testing.T) { + f := newStorageFixture(t) + now := time.Now() + old := now.Add(-40 * 24 * time.Hour) + expired := f.run(t, "old", "succeeded", old, old) + recent := f.run(t, "long-but-recent", "succeeded", old, now) + active := f.run(t, "active", "running", old, old) + policy := protocol.StoragePolicy{MaxAgeDays: 7} + response := f.m.handle(context.Background(), protocol.Message{Type: protocol.PreviewStorageCleanup, StoragePolicy: &policy}) + if response.Type != protocol.StorageCleanupPreview { + t.Fatalf("preview %+v", response) + } + p := response.StorageCleanup + if p.CandidateCount != 1 || p.ProtectedCount != 1 || p.ReclaimableBytes != 4096 || p.PreviewToken == "" { + t.Fatalf("preview %+v", p) + } + // A new eligible run that appears after confirmation is never added to apply. + newerCandidate := f.run(t, "new-candidate", "succeeded", old, old) + applied := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken}) + if applied.Type != protocol.StorageCleanupApplied || applied.StorageCleanup.RemovedCount != 1 { + t.Fatalf("apply %+v", applied) + } + runs, err := f.m.store.ListRuns(context.Background()) + if err != nil { + t.Fatal(err) + } + seen := map[int64]bool{} + for _, r := range runs { + seen[r.ID] = true + } + if seen[expired] || !seen[recent] || !seen[active] || !seen[newerCandidate] { + t.Fatalf("remaining %+v", seen) + } + if _, err := os.Stat(filepath.Join(f.workspace, "source.txt")); err != nil { + t.Fatal("workspace was removed") + } + // Remembered roots retain total inventory coverage after their last run goes. + paths, err := f.m.store.StorageWorkspacePaths(context.Background()) + if err != nil || len(paths) == 0 { + t.Fatalf("roots %v %v", paths, err) + } + again := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken}) + if again.Type != protocol.Error { + t.Fatal("preview token reused") + } +} +func TestStorageTokensPolicyValidationAndSizeTarget(t *testing.T) { + f := newStorageFixture(t) + old := time.Now().Add(-40 * 24 * time.Hour) + f.run(t, "old", "succeeded", old, old) + policy := protocol.StoragePolicy{MaxBytes: 1} + response := f.m.handle(context.Background(), protocol.Message{Type: protocol.PreviewStorageCleanup, StoragePolicy: &policy}) + p := response.StorageCleanup + if p == nil || p.TargetReached || p.CandidateCount != 1 { + t.Fatalf("size preview %+v", response) + } + changed := policy + changed.MaxBytes = 2 + if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &changed, StorageCleanupToken: p.PreviewToken}); got.Type != protocol.Error { + t.Fatal("changed policy accepted") + } + saved := f.m.previews[p.PreviewToken] + saved.expires = time.Now().Add(-time.Second) + f.m.previews[p.PreviewToken] = saved + if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken}); got.Type != protocol.Error { + t.Fatal("expired token accepted") + } + bad := protocol.StoragePolicy{Enabled: true} + if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.SetStoragePolicy, StoragePolicy: &bad}); got.Type != protocol.Error { + t.Fatal("invalid automatic policy accepted") + } + overview := f.m.handle(context.Background(), protocol.Message{Type: protocol.GetStorage}) + if overview.Storage == nil || overview.Storage.Policy.Enabled || overview.Storage.Measurement != "logical_file_bytes" { + t.Fatalf("overview %+v", overview) + } +} +func TestStorageBusyDispatchDoesNotWaitForMaintenance(t *testing.T) { + f := newStorageFixture(t) + f.m.mu.Lock() + replies := make(chan protocol.Message, 1) + f.m.dispatch(context.Background(), protocol.Message{Type: protocol.GetStorage, ID: "busy"}, func(m protocol.Message) error { replies <- m; return nil }) + select { + case reply := <-replies: + if reply.Type != protocol.Error { + t.Fatalf("reply %+v", reply) + } + case <-time.After(time.Second): + t.Fatal("busy storage request blocked") + } + f.m.mu.Unlock() +} +func TestStorageWebsocketProtocolAndNoStartupCleanup(t *testing.T) { + // Isolate attachment and harness cache roots; this test never inventories the + // operator's real home, chats or repositories. + t.Setenv("HOME", t.TempDir()) + t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", t.TempDir()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newStorageFixture(t) + old := time.Now().Add(-40 * 24 * time.Hour) + id := f.run(t, "old", "succeeded", old, old) + policy := protocol.StoragePolicy{Enabled: true, MaxAgeDays: 7} + if err := f.m.store.SaveStoragePolicy(ctx, policy); err != nil { + t.Fatal(err) + } + handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: f.m.runner.artifactsRoot}, f.m.store) + defer func() { cancel(); WaitRuns(handler) }() + srv := httptest.NewServer(handler) + defer srv.Close() + conn := dialAckClient(t, ctx, srv.URL) + reply := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.GetStorage, ID: "storage"}) + if reply.Type != protocol.StorageReply || reply.ID != "storage" || reply.Storage == nil || !reply.Storage.Policy.Enabled { + t.Fatalf("wire %+v", reply) + } + if _, err := f.m.store.GetRun(ctx, id); err != nil { + t.Fatal("startup pruned run", err) + } + invalid := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.SetStoragePolicy, ID: "invalid"}) + if invalid.Type != protocol.Error { + t.Fatalf("invalid %+v", invalid) + } + ping := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.Ping, ID: "ping"}) + if ping.Type != protocol.Pong { + t.Fatal(fmt.Sprint(ping)) + } +} + +func TestStorageAutomaticPolicyOptInAndActiveProtection(t *testing.T) { + f := newStorageFixture(t) + ctx := context.Background() + old := time.Now().Add(-40 * 24 * time.Hour) + expired := f.run(t, "automatic-old", "succeeded", old, old) + active := f.run(t, "automatic-active", "running", old, old) + if err := f.m.automaticPass(ctx); err != nil { + t.Fatal(err) + } + if _, err := f.m.store.GetRun(ctx, expired); err != nil { + t.Fatal("disabled automatic policy deleted history") + } + if err := f.m.store.SaveStoragePolicy(ctx, protocol.StoragePolicy{Enabled: true, MaxAgeDays: 7}); err != nil { + t.Fatal(err) + } + if err := f.m.automaticPass(ctx); err != nil { + t.Fatal(err) + } + if _, err := f.m.store.GetRun(ctx, expired); err == nil { + t.Fatal("enabled policy did not remove expired run") + } + if _, err := f.m.store.GetRun(ctx, active); err != nil { + t.Fatal("active run removed", err) + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index abff045d..71e032c2 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -107,6 +107,12 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin go runs.sizeLoop() } } + var storage *storageManager + if runs != nil { + storage = newStorageManager(runs) + runs.schedulerWG.Add(1) + go storage.loop(ctx) + } // chat_sessions lists the operator's own recorded sessions. Prefer an // injected registry; otherwise derive it from the store. With neither, the // handler fails closed to an empty list. @@ -244,7 +250,27 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin }(request) continue } - if handleChat(connCtx, request, options.Chat, hub, registry, clientID, write) { + // Prevent a new operator workspace admission racing a worktree + // removal. Ordinary reads/prompts do not wait for storage scans. + admission := storage != nil && (request.Type == protocol.ChatCreate || request.Type == protocol.ChatAdopt) + if admission { + storage.workspaceMu.RLock() + } + handledChat := handleChat(connCtx, request, options.Chat, hub, registry, clientID, write) + if admission { + storage.workspaceMu.RUnlock() + } + if handledChat { + continue + } + if isStorageRequest(request.Type) { + if storage == nil { + _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management unavailable: no execution store"}) + } else { + // Disk/Git inventory must never stall chat permissions/prompts + // arriving on this same persistent connection. + storage.dispatch(connCtx, request, write) + } continue } if handleDirectoryListing(request, write) {