diff --git a/internal/server/storage_inventory.go b/internal/server/storage_inventory.go new file mode 100644 index 00000000..9b1f3272 --- /dev/null +++ b/internal/server/storage_inventory.go @@ -0,0 +1,109 @@ +package server + +import ( + "context" + "sort" + "time" +) + +type storageInventoryConfig struct { + DatabasePath, DataRoot, ArtifactsRoot, AttachmentsRoot, SessionsRoot string + WorkspaceRoots, ProtectedWorkspaces []string +} + +type storageCategory struct { + ID string `json:"id"` + Label string `json:"label"` + Path string `json:"path,omitempty"` + Bytes int64 `json:"bytes"` + Files int64 `json:"files"` + Partial bool `json:"partial,omitempty"` + Error string `json:"error,omitempty"` + ModifiedAt time.Time `json:"-"` +} + +type storageWorktree struct { + Path string `json:"path"` + Repository string `json:"repository"` + Branch string `json:"branch,omitempty"` + Bytes int64 `json:"bytes"` + Files int64 `json:"files"` + ModifiedAt time.Time `json:"modified_at"` + Eligible bool `json:"eligible"` + Reason string `json:"reason,omitempty"` + locked bool +} + +type storageInventory struct { + TotalBytes int64 + Categories []storageCategory + Worktrees []storageWorktree + Partial bool + Errors []string +} + +// Inventory counts logical regular-file bytes. It never follows symlinks, and +// each inode is counted once across nested roots and hard-linked files. +func inspectStorage(ctx context.Context, config storageInventoryConfig) (storageInventory, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + result := storageInventory{Categories: []storageCategory{}, Worktrees: []storageWorktree{}} + trees, discoveryErrors := discoverStorageWorktrees(ctx, config.WorkspaceRoots, config.ProtectedWorkspaces) + result.Errors = discoveryErrors + result.Partial = len(discoveryErrors) > 0 + walker := storageWalker{ctx: ctx, seen: map[string]bool{}, inodes: map[[2]uint64]bool{}} + add := func(category storageCategory, roots ...string) storageCategory { + for _, root := range roots { + walker.measure(root, &category) + } + result.TotalBytes += category.Bytes + result.Partial = result.Partial || category.Partial + if category.Error != "" { + result.Errors = append(result.Errors, category.Error) + } + result.Categories = append(result.Categories, category) + return category + } + if config.DatabasePath != "" { + add(storageCategory{ID: "database", Label: "Database and journal", Path: config.DatabasePath}, config.DatabasePath, config.DatabasePath+"-wal", config.DatabasePath+"-shm") + } + add(storageCategory{ID: "artifacts", Label: "Run artifacts and exports", Path: config.ArtifactsRoot}, config.ArtifactsRoot) + add(storageCategory{ID: "attachments", Label: "Chat attachments", Path: config.AttachmentsRoot}, config.AttachmentsRoot) + add(storageCategory{ID: "sessions", Label: "Harness session files", Path: config.SessionsRoot}, config.SessionsRoot) + add(storageCategory{ID: "other_data", Label: "Other server data", Path: config.DataRoot}, config.DataRoot) + category := storageCategory{ID: "worktrees", Label: "Linked worktrees"} + // Nested linked trees are attributed before their parents. + sort.Slice(trees, func(i, j int) bool { return len(trees[i].Path) > len(trees[j].Path) }) + for _, tree := range trees { + measured := storageCategory{} + walker.measure(tree.Path, &measured) + tree.Bytes, tree.Files = measured.Bytes, measured.Files + if measured.ModifiedAt.After(tree.ModifiedAt) { + tree.ModifiedAt = measured.ModifiedAt + } + if measured.Partial { + tree.Eligible = false + tree.Reason = "Storage measurement incomplete" + category.Partial = true + category.Error = measured.Error + } + category.Bytes += measured.Bytes + category.Files += measured.Files + result.Worktrees = append(result.Worktrees, tree) + } + result.TotalBytes += category.Bytes + result.Categories = append(result.Categories, category) + result.Partial = result.Partial || category.Partial + if category.Error != "" { + result.Errors = append(result.Errors, category.Error) + } + roots := append([]string{}, config.WorkspaceRoots...) + sort.Strings(roots) + add(storageCategory{ID: "workspaces", Label: "Other project working files"}, roots...) + sort.Slice(result.Worktrees, func(i, j int) bool { return result.Worktrees[i].Path < result.Worktrees[j].Path }) + if ctx.Err() != nil { + result.Partial = true + result.Errors = append(result.Errors, ctx.Err().Error()) + } + return result, nil +} diff --git a/internal/server/storage_inventory_git.go b/internal/server/storage_inventory_git.go new file mode 100644 index 00000000..6460504b --- /dev/null +++ b/internal/server/storage_inventory_git.go @@ -0,0 +1,222 @@ +package server + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +func discoverStorageWorktrees(ctx context.Context, roots, protected []string) ([]storageWorktree, []string) { + seenRepos := map[string]bool{} + seenTrees := map[string]bool{} + trees := []storageWorktree{} + problems := []string{} + for index, root := range roots { + if index >= 256 { + problems = append(problems, "Workspace discovery limit reached") + break + } + root = filepath.Clean(root) + if _, err := os.Stat(root); os.IsNotExist(err) { + continue + } + common, err := storageGit(ctx, root, "rev-parse", "--git-common-dir") + if err != nil { + continue + } // A recorded workspace need not be a Git repository. + common = resolveGitPath(root, common) + common, err = filepath.EvalSymlinks(common) + if err != nil { + problems = append(problems, err.Error()) + continue + } + if seenRepos[common] { + continue + } + seenRepos[common] = true + if len(seenRepos) > 64 { + problems = append(problems, "Repository discovery limit reached") + break + } + listing, err := storageGit(ctx, root, "worktree", "list", "--porcelain", "-z") + if err != nil { + problems = append(problems, err.Error()) + continue + } + repository := common + for _, record := range strings.Split(listing, "\x00\x00") { + fields := strings.Split(record, "\x00") + if len(fields) > 0 && strings.HasPrefix(fields[0], "worktree ") { + candidate := strings.TrimPrefix(fields[0], "worktree ") + if info, err := os.Lstat(filepath.Join(candidate, ".git")); err == nil && info.IsDir() { + repository = candidate + break + } + } + } + for _, record := range strings.Split(listing, "\x00\x00") { + fields := strings.Split(record, "\x00") + if len(fields) == 0 || !strings.HasPrefix(fields[0], "worktree ") { + continue + } + path := strings.TrimPrefix(fields[0], "worktree ") + if seenTrees[path] { + continue + } + seenTrees[path] = true + if len(trees) >= 512 { + problems = append(problems, "Worktree discovery limit reached") + return trees, problems + } + // The primary checkout has a .git directory; linked worktrees have a file. + gitInfo, err := os.Lstat(filepath.Join(path, ".git")) + if err != nil || !gitInfo.Mode().IsRegular() { + continue + } + tree := storageWorktree{Path: path, Repository: repository} + for _, field := range fields[1:] { + if strings.HasPrefix(field, "branch ") { + tree.Branch = strings.TrimPrefix(field, "branch refs/heads/") + } + if field == "locked" || strings.HasPrefix(field, "locked ") { + tree.locked = true + } + } + if info, err := os.Stat(path); err == nil { + tree.ModifiedAt = info.ModTime().UTC() + } + tree.Reason = storageWorktreeProtection(ctx, tree, protected) + tree.Eligible = tree.Reason == "" + trees = append(trees, tree) + } + } + return trees, problems +} + +// Eligibility is deliberately conservative: no forced removal, branch deletion, +// or deletion of a workspace whose work has not reached the repository's main. +func storageWorktreeProtection(ctx context.Context, tree storageWorktree, protected []string) string { + if tree.locked { + return "Git worktree is locked" + } + root, err := filepath.EvalSymlinks(tree.Path) + if err != nil || root != filepath.Clean(tree.Path) { + return "Worktree path is unavailable or indirect" + } + for _, path := range protected { + if path == "" { + continue + } + absolute, err := storageCanonicalPath(path) + if err != nil { + return "Protected workspace path is unresolved" + } + if root == absolute || pathWithin(root, absolute) || pathWithin(absolute, root) { + return "Workspace is in use by a run or open chat" + } + } + info, err := os.Lstat(filepath.Join(root, ".git")) + if err != nil || !info.Mode().IsRegular() { + return "Primary checkout or invalid linked worktree" + } + gitDir, err := storageGit(ctx, root, "rev-parse", "--git-dir") + if err != nil { + return "Worktree metadata unavailable" + } + common, err := storageGit(ctx, root, "rev-parse", "--git-common-dir") + if err != nil { + return "Repository metadata unavailable" + } + gitDir, err = canonicalDirectory(resolveGitPath(root, gitDir)) + if err != nil { + return "Worktree metadata unavailable" + } + common, err = canonicalDirectory(resolveGitPath(root, common)) + if err != nil || !pathWithin(filepath.Join(common, "worktrees"), gitDir) { + return "Unverified linked worktree" + } + if _, err := os.Stat(filepath.Join(gitDir, "locked")); err == nil { + return "Git worktree is locked" + } + + currentBranch, _ := storageGit(ctx, root, "symbolic-ref", "--quiet", "--short", "HEAD") + if currentBranch == "main" || currentBranch == "master" { + return "Main branch checkout" + } + top, err := storageGit(ctx, root, "rev-parse", "--show-toplevel") + if err != nil || top != root { + return "Worktree root is not verified" + } + + status, err := storageGit(ctx, root, "status", "--porcelain", "--untracked-files=all") + if err != nil { + return "Git status unavailable" + } + if status != "" { + return "Uncommitted or untracked work" + } + if _, err := storageGit(ctx, root, "merge-base", "--is-ancestor", "HEAD", "refs/heads/main"); err != nil { + return "Work is not fully merged into main" + } + return "" +} + +func removeStorageWorktree(ctx context.Context, tree storageWorktree, protected []string) error { + if reason := storageWorktreeProtection(ctx, tree, protected); reason != "" { + return fmt.Errorf("worktree protected: %s", reason) + } + _, err := storageGit(ctx, tree.Repository, "worktree", "remove", "--", tree.Path) + return err +} + +// Git subprocesses have both time and output bounds; no user shell is invoked. +func storageGit(ctx context.Context, directory string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...) + var out storageGitBuffer + command.Stdout = &out + command.Stderr = &out + if err := command.Run(); err != nil { + return "", fmt.Errorf("git %s in %s: %w", strings.Join(args, " "), directory, err) + } + return strings.TrimSpace(out.String()), nil +} + +type storageGitBuffer struct{ bytes.Buffer } + +func (b *storageGitBuffer) Write(data []byte) (int, error) { + if b.Len()+len(data) > 1<<20 { + return 0, fmt.Errorf("git output limit reached") + } + return b.Buffer.Write(data) +} + +// Resolve aliases even when the protected directory is a future child path. +func storageCanonicalPath(path string) (string, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return "", err + } + current := absolute + for { + resolved, err := filepath.EvalSymlinks(current) + if err == nil { + relative, err := filepath.Rel(current, absolute) + if err != nil { + return "", err + } + return filepath.Join(resolved, relative), nil + } + parent := filepath.Dir(current) + if parent == current { + return "", err + } + current = parent + } +} diff --git a/internal/server/storage_inventory_test.go b/internal/server/storage_inventory_test.go new file mode 100644 index 00000000..2ecddabb --- /dev/null +++ b/internal/server/storage_inventory_test.go @@ -0,0 +1,162 @@ +package server + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func storageTestFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } +} + +func TestStorageInventoryDeduplicatesAndExcludesSymlinks(t *testing.T) { + root := t.TempDir() + data := filepath.Join(root, "data") + artifacts := filepath.Join(data, "artifacts") + database := filepath.Join(data, "state.db") + storageTestFile(t, database, "database") + storageTestFile(t, database+"-wal", "wal") + storageTestFile(t, filepath.Join(artifacts, "1", "output"), "artifact") + storageTestFile(t, filepath.Join(data, "notes"), "note") + outside := filepath.Join(root, "outside") + storageTestFile(t, outside, "must not count") + if err := os.Symlink(outside, filepath.Join(data, "symlink")); err != nil { + t.Fatal(err) + } + if err := os.Link(filepath.Join(data, "notes"), filepath.Join(data, "hardlink")); err != nil { + t.Fatal(err) + } + inventory, err := inspectStorage(context.Background(), storageInventoryConfig{DatabasePath: database, DataRoot: data, ArtifactsRoot: artifacts, WorkspaceRoots: []string{data, data}}) + if err != nil { + t.Fatal(err) + } + if inventory.TotalBytes != 23 { + t.Fatalf("bytes %d, want23: %+v", inventory.TotalBytes, inventory.Categories) + } + if inventory.Partial { + t.Fatalf("unexpected partial: %+v", inventory) + } + categories := map[string]int64{} + for _, item := range inventory.Categories { + categories[item.ID] = item.Bytes + } + if categories["database"] != 11 || categories["artifacts"] != 8 || categories["other_data"] != 4 || categories["workspaces"] != 0 { + t.Fatalf("categories %+v", categories) + } +} + +func TestStorageMeasurementReportsCancellationAndIndirectRoots(t *testing.T) { + root := t.TempDir() + storageTestFile(t, filepath.Join(root, "file"), "content") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _, partial, problems := measureStoragePath(ctx, root) + if !partial || len(problems) == 0 { + t.Fatal("canceled measurement claimed complete") + } + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(root, link); err != nil { + t.Fatal(err) + } + _, bytes, partial, problems := measureStoragePath(context.Background(), link) + if bytes != 0 || !partial || len(problems) == 0 { + t.Fatal("symlink root followed or not reported") + } +} + +func storageTestGit(t *testing.T, root string, args ...string) string { + t.Helper() + out, err := storageGit(context.Background(), root, args...) + if err != nil { + t.Fatal(err) + } + return out +} +func storageTestRepo(t *testing.T) (string, string) { + t.Helper() + base := t.TempDir() + main := filepath.Join(base, "main") + if err := os.Mkdir(main, 0700); err != nil { + t.Fatal(err) + } + storageTestGit(t, main, "init", "-b", "main") + storageTestGit(t, main, "config", "user.email", "fixture@example.invalid") + storageTestGit(t, main, "config", "user.name", "Fixture") + storageTestFile(t, filepath.Join(main, "tracked"), "initial") + storageTestGit(t, main, "add", "tracked") + storageTestGit(t, main, "commit", "-m", "initial") + linked := filepath.Join(base, "linked") + storageTestGit(t, main, "worktree", "add", "-b", "item", linked) + return main, linked +} + +func TestStorageWorktreesRequireMergedCleanInactiveAndUnlocked(t *testing.T) { + ctx := context.Background() + main, linked := storageTestRepo(t) + discover := func(protected ...string) storageWorktree { + t.Helper() + trees, problems := discoverStorageWorktrees(ctx, []string{main, linked}, protected) + if len(problems) > 0 || len(trees) != 1 { + t.Fatalf("discovery %+v %+v", trees, problems) + } + return trees[0] + } + tree := discover() + canonicalMain, err := filepath.EvalSymlinks(main) + if err != nil { + t.Fatal(err) + } + if !tree.Eligible || tree.Repository != canonicalMain { + t.Fatalf("clean merged linked tree not eligible: %+v", tree) + } + if discover(linked).Eligible { + t.Fatal("open chat/run workspace eligible") + } + if discover(filepath.Join(linked, "subdirectory")).Eligible { + t.Fatal("nested active workspace eligible") + } + storageTestFile(t, filepath.Join(linked, "draft"), "do not delete") + if got := discover(); got.Eligible || !strings.Contains(got.Reason, "Uncommitted") { + t.Fatalf("dirty tree %+v", got) + } + if err := removeStorageWorktree(ctx, tree, nil); err == nil { + t.Fatal("stale candidate removed newly dirty tree") + } + if err := os.Remove(filepath.Join(linked, "draft")); err != nil { + t.Fatal(err) + } + storageTestGit(t, main, "worktree", "lock", linked) + if discover().Eligible { + t.Fatal("locked tree eligible") + } + storageTestGit(t, main, "worktree", "unlock", linked) + storageTestFile(t, filepath.Join(linked, "tracked"), "unlanded") + storageTestGit(t, linked, "add", "tracked") + storageTestGit(t, linked, "commit", "-m", "unlanded") + if got := discover(); got.Eligible || !strings.Contains(got.Reason, "merged") { + t.Fatalf("unlanded tree %+v", got) + } + storageTestGit(t, main, "merge", "--ff-only", "item") + if !discover().Eligible { + t.Fatal("landed clean tree not eligible") + } + if err := removeStorageWorktree(ctx, discover(), nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(linked); !os.IsNotExist(err) { + t.Fatalf("worktree remains: %v", err) + } + storageTestGit(t, main, "show-ref", "--verify", "refs/heads/item") + if _, err := os.Stat(main); err != nil { + t.Fatal("main checkout removed") + } +} diff --git a/internal/server/storage_inventory_walk.go b/internal/server/storage_inventory_walk.go new file mode 100644 index 00000000..d45667da --- /dev/null +++ b/internal/server/storage_inventory_walk.go @@ -0,0 +1,139 @@ +package server + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "time" +) + +type storageWalker struct { + ctx context.Context + seen map[string]bool + inodes map[[2]uint64]bool + entries int +} + +func (w *storageWalker) measure(root string, category *storageCategory) { + if strings.TrimSpace(root) == "" { + return + } + root, err := filepath.Abs(root) + if err != nil { + storageMeasureError(category, err) + return + } + info, err := os.Lstat(root) + if os.IsNotExist(err) { + return + } + if err != nil { + storageMeasureError(category, err) + return + } + if info.Mode()&os.ModeSymlink != 0 { + storageMeasureError(category, fmt.Errorf("symlink root excluded: %s", root)) + return + } + if err := w.visit(root, category, 0); err != nil { + storageMeasureError(category, err) + } +} + +// Read directories in bounded chunks rather than allocating a whole directory +// listing. Compare opened directory identity to avoid traversing a replacement +// symlink between Lstat and Open. Regular files are stat-only, never opened. +func (w *storageWalker) visit(path string, category *storageCategory, depth int) error { + if w.ctx.Err() != nil { + return w.ctx.Err() + } + w.entries++ + if w.entries > 500000 { + return fmt.Errorf("storage scan entry limit reached") + } + if depth > 128 { + return fmt.Errorf("storage scan depth limit reached") + } + if w.seen[path] { + return nil + } + w.seen[path] = true + info, err := os.Lstat(path) + if err != nil { + storageMeasureError(category, err) + return nil + } + if info.Mode()&os.ModeSymlink != 0 { + return nil + } + if info.ModTime().After(category.ModifiedAt) { + category.ModifiedAt = info.ModTime().UTC() + } + if info.Mode().IsRegular() { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + key := [2]uint64{uint64(stat.Dev), uint64(stat.Ino)} + if w.inodes[key] { + return nil + } + w.inodes[key] = true + } + category.Files++ + category.Bytes += info.Size() + return nil + } + if !info.IsDir() { + return nil + } + directory, err := os.Open(path) + if err != nil { + storageMeasureError(category, err) + return nil + } + defer directory.Close() + opened, err := directory.Stat() + if err != nil || !os.SameFile(info, opened) { + storageMeasureError(category, fmt.Errorf("directory changed during storage scan: %s", path)) + return nil + } + for { + if w.ctx.Err() != nil { + return w.ctx.Err() + } + entries, err := directory.ReadDir(256) + if err != nil && err != io.EOF { + storageMeasureError(category, err) + return nil + } + for _, entry := range entries { + if err := w.visit(filepath.Join(path, entry.Name()), category, depth+1); err != nil { + return err + } + } + if err == io.EOF { + return nil + } + } +} + +func storageMeasureError(category *storageCategory, err error) { + category.Partial = true + if category.Error == "" { + category.Error = err.Error() + } +} + +func measureStoragePath(ctx context.Context, path string) (files, bytes int64, partial bool, problems []string) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + category := storageCategory{} + walker := storageWalker{ctx: ctx, seen: map[string]bool{}, inodes: map[[2]uint64]bool{}} + walker.measure(path, &category) + if category.Error != "" { + problems = append(problems, category.Error) + } + return category.Files, category.Bytes, category.Partial, problems +}