Measure server data and registered worktree storage safely
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type storageInventoryConfig struct {
|
||||
DatabasePath, DataRoot, ArtifactsRoot, AttachmentsRoot, SessionsRoot string
|
||||
WorkspaceRoots, ProtectedWorkspaces []string
|
||||
}
|
||||
|
||||
type storageCategory struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Files int64 `json:"files"`
|
||||
Partial bool `json:"partial,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ModifiedAt time.Time `json:"-"`
|
||||
}
|
||||
|
||||
type storageWorktree struct {
|
||||
Path string `json:"path"`
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Files int64 `json:"files"`
|
||||
ModifiedAt time.Time `json:"modified_at"`
|
||||
Eligible bool `json:"eligible"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
locked bool
|
||||
}
|
||||
|
||||
type storageInventory struct {
|
||||
TotalBytes int64
|
||||
Categories []storageCategory
|
||||
Worktrees []storageWorktree
|
||||
Partial bool
|
||||
Errors []string
|
||||
}
|
||||
|
||||
// Inventory counts logical regular-file bytes. It never follows symlinks, and
|
||||
// each inode is counted once across nested roots and hard-linked files.
|
||||
func inspectStorage(ctx context.Context, config storageInventoryConfig) (storageInventory, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
result := storageInventory{Categories: []storageCategory{}, Worktrees: []storageWorktree{}}
|
||||
trees, discoveryErrors := discoverStorageWorktrees(ctx, config.WorkspaceRoots, config.ProtectedWorkspaces)
|
||||
result.Errors = discoveryErrors
|
||||
result.Partial = len(discoveryErrors) > 0
|
||||
walker := storageWalker{ctx: ctx, seen: map[string]bool{}, inodes: map[[2]uint64]bool{}}
|
||||
add := func(category storageCategory, roots ...string) storageCategory {
|
||||
for _, root := range roots {
|
||||
walker.measure(root, &category)
|
||||
}
|
||||
result.TotalBytes += category.Bytes
|
||||
result.Partial = result.Partial || category.Partial
|
||||
if category.Error != "" {
|
||||
result.Errors = append(result.Errors, category.Error)
|
||||
}
|
||||
result.Categories = append(result.Categories, category)
|
||||
return category
|
||||
}
|
||||
if config.DatabasePath != "" {
|
||||
add(storageCategory{ID: "database", Label: "Database and journal", Path: config.DatabasePath}, config.DatabasePath, config.DatabasePath+"-wal", config.DatabasePath+"-shm")
|
||||
}
|
||||
add(storageCategory{ID: "artifacts", Label: "Run artifacts and exports", Path: config.ArtifactsRoot}, config.ArtifactsRoot)
|
||||
add(storageCategory{ID: "attachments", Label: "Chat attachments", Path: config.AttachmentsRoot}, config.AttachmentsRoot)
|
||||
add(storageCategory{ID: "sessions", Label: "Harness session files", Path: config.SessionsRoot}, config.SessionsRoot)
|
||||
add(storageCategory{ID: "other_data", Label: "Other server data", Path: config.DataRoot}, config.DataRoot)
|
||||
category := storageCategory{ID: "worktrees", Label: "Linked worktrees"}
|
||||
// Nested linked trees are attributed before their parents.
|
||||
sort.Slice(trees, func(i, j int) bool { return len(trees[i].Path) > len(trees[j].Path) })
|
||||
for _, tree := range trees {
|
||||
measured := storageCategory{}
|
||||
walker.measure(tree.Path, &measured)
|
||||
tree.Bytes, tree.Files = measured.Bytes, measured.Files
|
||||
if measured.ModifiedAt.After(tree.ModifiedAt) {
|
||||
tree.ModifiedAt = measured.ModifiedAt
|
||||
}
|
||||
if measured.Partial {
|
||||
tree.Eligible = false
|
||||
tree.Reason = "Storage measurement incomplete"
|
||||
category.Partial = true
|
||||
category.Error = measured.Error
|
||||
}
|
||||
category.Bytes += measured.Bytes
|
||||
category.Files += measured.Files
|
||||
result.Worktrees = append(result.Worktrees, tree)
|
||||
}
|
||||
result.TotalBytes += category.Bytes
|
||||
result.Categories = append(result.Categories, category)
|
||||
result.Partial = result.Partial || category.Partial
|
||||
if category.Error != "" {
|
||||
result.Errors = append(result.Errors, category.Error)
|
||||
}
|
||||
roots := append([]string{}, config.WorkspaceRoots...)
|
||||
sort.Strings(roots)
|
||||
add(storageCategory{ID: "workspaces", Label: "Other project working files"}, roots...)
|
||||
sort.Slice(result.Worktrees, func(i, j int) bool { return result.Worktrees[i].Path < result.Worktrees[j].Path })
|
||||
if ctx.Err() != nil {
|
||||
result.Partial = true
|
||||
result.Errors = append(result.Errors, ctx.Err().Error())
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func discoverStorageWorktrees(ctx context.Context, roots, protected []string) ([]storageWorktree, []string) {
|
||||
seenRepos := map[string]bool{}
|
||||
seenTrees := map[string]bool{}
|
||||
trees := []storageWorktree{}
|
||||
problems := []string{}
|
||||
for index, root := range roots {
|
||||
if index >= 256 {
|
||||
problems = append(problems, "Workspace discovery limit reached")
|
||||
break
|
||||
}
|
||||
root = filepath.Clean(root)
|
||||
if _, err := os.Stat(root); os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
common, err := storageGit(ctx, root, "rev-parse", "--git-common-dir")
|
||||
if err != nil {
|
||||
continue
|
||||
} // A recorded workspace need not be a Git repository.
|
||||
common = resolveGitPath(root, common)
|
||||
common, err = filepath.EvalSymlinks(common)
|
||||
if err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
continue
|
||||
}
|
||||
if seenRepos[common] {
|
||||
continue
|
||||
}
|
||||
seenRepos[common] = true
|
||||
if len(seenRepos) > 64 {
|
||||
problems = append(problems, "Repository discovery limit reached")
|
||||
break
|
||||
}
|
||||
listing, err := storageGit(ctx, root, "worktree", "list", "--porcelain", "-z")
|
||||
if err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
continue
|
||||
}
|
||||
repository := common
|
||||
for _, record := range strings.Split(listing, "\x00\x00") {
|
||||
fields := strings.Split(record, "\x00")
|
||||
if len(fields) > 0 && strings.HasPrefix(fields[0], "worktree ") {
|
||||
candidate := strings.TrimPrefix(fields[0], "worktree ")
|
||||
if info, err := os.Lstat(filepath.Join(candidate, ".git")); err == nil && info.IsDir() {
|
||||
repository = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, record := range strings.Split(listing, "\x00\x00") {
|
||||
fields := strings.Split(record, "\x00")
|
||||
if len(fields) == 0 || !strings.HasPrefix(fields[0], "worktree ") {
|
||||
continue
|
||||
}
|
||||
path := strings.TrimPrefix(fields[0], "worktree ")
|
||||
if seenTrees[path] {
|
||||
continue
|
||||
}
|
||||
seenTrees[path] = true
|
||||
if len(trees) >= 512 {
|
||||
problems = append(problems, "Worktree discovery limit reached")
|
||||
return trees, problems
|
||||
}
|
||||
// The primary checkout has a .git directory; linked worktrees have a file.
|
||||
gitInfo, err := os.Lstat(filepath.Join(path, ".git"))
|
||||
if err != nil || !gitInfo.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
tree := storageWorktree{Path: path, Repository: repository}
|
||||
for _, field := range fields[1:] {
|
||||
if strings.HasPrefix(field, "branch ") {
|
||||
tree.Branch = strings.TrimPrefix(field, "branch refs/heads/")
|
||||
}
|
||||
if field == "locked" || strings.HasPrefix(field, "locked ") {
|
||||
tree.locked = true
|
||||
}
|
||||
}
|
||||
if info, err := os.Stat(path); err == nil {
|
||||
tree.ModifiedAt = info.ModTime().UTC()
|
||||
}
|
||||
tree.Reason = storageWorktreeProtection(ctx, tree, protected)
|
||||
tree.Eligible = tree.Reason == ""
|
||||
trees = append(trees, tree)
|
||||
}
|
||||
}
|
||||
return trees, problems
|
||||
}
|
||||
|
||||
// Eligibility is deliberately conservative: no forced removal, branch deletion,
|
||||
// or deletion of a workspace whose work has not reached the repository's main.
|
||||
func storageWorktreeProtection(ctx context.Context, tree storageWorktree, protected []string) string {
|
||||
if tree.locked {
|
||||
return "Git worktree is locked"
|
||||
}
|
||||
root, err := filepath.EvalSymlinks(tree.Path)
|
||||
if err != nil || root != filepath.Clean(tree.Path) {
|
||||
return "Worktree path is unavailable or indirect"
|
||||
}
|
||||
for _, path := range protected {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
absolute, err := storageCanonicalPath(path)
|
||||
if err != nil {
|
||||
return "Protected workspace path is unresolved"
|
||||
}
|
||||
if root == absolute || pathWithin(root, absolute) || pathWithin(absolute, root) {
|
||||
return "Workspace is in use by a run or open chat"
|
||||
}
|
||||
}
|
||||
info, err := os.Lstat(filepath.Join(root, ".git"))
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
return "Primary checkout or invalid linked worktree"
|
||||
}
|
||||
gitDir, err := storageGit(ctx, root, "rev-parse", "--git-dir")
|
||||
if err != nil {
|
||||
return "Worktree metadata unavailable"
|
||||
}
|
||||
common, err := storageGit(ctx, root, "rev-parse", "--git-common-dir")
|
||||
if err != nil {
|
||||
return "Repository metadata unavailable"
|
||||
}
|
||||
gitDir, err = canonicalDirectory(resolveGitPath(root, gitDir))
|
||||
if err != nil {
|
||||
return "Worktree metadata unavailable"
|
||||
}
|
||||
common, err = canonicalDirectory(resolveGitPath(root, common))
|
||||
if err != nil || !pathWithin(filepath.Join(common, "worktrees"), gitDir) {
|
||||
return "Unverified linked worktree"
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(gitDir, "locked")); err == nil {
|
||||
return "Git worktree is locked"
|
||||
}
|
||||
|
||||
currentBranch, _ := storageGit(ctx, root, "symbolic-ref", "--quiet", "--short", "HEAD")
|
||||
if currentBranch == "main" || currentBranch == "master" {
|
||||
return "Main branch checkout"
|
||||
}
|
||||
top, err := storageGit(ctx, root, "rev-parse", "--show-toplevel")
|
||||
if err != nil || top != root {
|
||||
return "Worktree root is not verified"
|
||||
}
|
||||
|
||||
status, err := storageGit(ctx, root, "status", "--porcelain", "--untracked-files=all")
|
||||
if err != nil {
|
||||
return "Git status unavailable"
|
||||
}
|
||||
if status != "" {
|
||||
return "Uncommitted or untracked work"
|
||||
}
|
||||
if _, err := storageGit(ctx, root, "merge-base", "--is-ancestor", "HEAD", "refs/heads/main"); err != nil {
|
||||
return "Work is not fully merged into main"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func removeStorageWorktree(ctx context.Context, tree storageWorktree, protected []string) error {
|
||||
if reason := storageWorktreeProtection(ctx, tree, protected); reason != "" {
|
||||
return fmt.Errorf("worktree protected: %s", reason)
|
||||
}
|
||||
_, err := storageGit(ctx, tree.Repository, "worktree", "remove", "--", tree.Path)
|
||||
return err
|
||||
}
|
||||
|
||||
// Git subprocesses have both time and output bounds; no user shell is invoked.
|
||||
func storageGit(ctx context.Context, directory string, args ...string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
command := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...)
|
||||
var out storageGitBuffer
|
||||
command.Stdout = &out
|
||||
command.Stderr = &out
|
||||
if err := command.Run(); err != nil {
|
||||
return "", fmt.Errorf("git %s in %s: %w", strings.Join(args, " "), directory, err)
|
||||
}
|
||||
return strings.TrimSpace(out.String()), nil
|
||||
}
|
||||
|
||||
type storageGitBuffer struct{ bytes.Buffer }
|
||||
|
||||
func (b *storageGitBuffer) Write(data []byte) (int, error) {
|
||||
if b.Len()+len(data) > 1<<20 {
|
||||
return 0, fmt.Errorf("git output limit reached")
|
||||
}
|
||||
return b.Buffer.Write(data)
|
||||
}
|
||||
|
||||
// Resolve aliases even when the protected directory is a future child path.
|
||||
func storageCanonicalPath(path string) (string, error) {
|
||||
absolute, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
current := absolute
|
||||
for {
|
||||
resolved, err := filepath.EvalSymlinks(current)
|
||||
if err == nil {
|
||||
relative, err := filepath.Rel(current, absolute)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(resolved, relative), nil
|
||||
}
|
||||
parent := filepath.Dir(current)
|
||||
if parent == current {
|
||||
return "", err
|
||||
}
|
||||
current = parent
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func storageTestFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageInventoryDeduplicatesAndExcludesSymlinks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
data := filepath.Join(root, "data")
|
||||
artifacts := filepath.Join(data, "artifacts")
|
||||
database := filepath.Join(data, "state.db")
|
||||
storageTestFile(t, database, "database")
|
||||
storageTestFile(t, database+"-wal", "wal")
|
||||
storageTestFile(t, filepath.Join(artifacts, "1", "output"), "artifact")
|
||||
storageTestFile(t, filepath.Join(data, "notes"), "note")
|
||||
outside := filepath.Join(root, "outside")
|
||||
storageTestFile(t, outside, "must not count")
|
||||
if err := os.Symlink(outside, filepath.Join(data, "symlink")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Link(filepath.Join(data, "notes"), filepath.Join(data, "hardlink")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inventory, err := inspectStorage(context.Background(), storageInventoryConfig{DatabasePath: database, DataRoot: data, ArtifactsRoot: artifacts, WorkspaceRoots: []string{data, data}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inventory.TotalBytes != 23 {
|
||||
t.Fatalf("bytes %d, want23: %+v", inventory.TotalBytes, inventory.Categories)
|
||||
}
|
||||
if inventory.Partial {
|
||||
t.Fatalf("unexpected partial: %+v", inventory)
|
||||
}
|
||||
categories := map[string]int64{}
|
||||
for _, item := range inventory.Categories {
|
||||
categories[item.ID] = item.Bytes
|
||||
}
|
||||
if categories["database"] != 11 || categories["artifacts"] != 8 || categories["other_data"] != 4 || categories["workspaces"] != 0 {
|
||||
t.Fatalf("categories %+v", categories)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageMeasurementReportsCancellationAndIndirectRoots(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
storageTestFile(t, filepath.Join(root, "file"), "content")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, _, partial, problems := measureStoragePath(ctx, root)
|
||||
if !partial || len(problems) == 0 {
|
||||
t.Fatal("canceled measurement claimed complete")
|
||||
}
|
||||
link := filepath.Join(t.TempDir(), "link")
|
||||
if err := os.Symlink(root, link); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, bytes, partial, problems := measureStoragePath(context.Background(), link)
|
||||
if bytes != 0 || !partial || len(problems) == 0 {
|
||||
t.Fatal("symlink root followed or not reported")
|
||||
}
|
||||
}
|
||||
|
||||
func storageTestGit(t *testing.T, root string, args ...string) string {
|
||||
t.Helper()
|
||||
out, err := storageGit(context.Background(), root, args...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func storageTestRepo(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
base := t.TempDir()
|
||||
main := filepath.Join(base, "main")
|
||||
if err := os.Mkdir(main, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storageTestGit(t, main, "init", "-b", "main")
|
||||
storageTestGit(t, main, "config", "user.email", "fixture@example.invalid")
|
||||
storageTestGit(t, main, "config", "user.name", "Fixture")
|
||||
storageTestFile(t, filepath.Join(main, "tracked"), "initial")
|
||||
storageTestGit(t, main, "add", "tracked")
|
||||
storageTestGit(t, main, "commit", "-m", "initial")
|
||||
linked := filepath.Join(base, "linked")
|
||||
storageTestGit(t, main, "worktree", "add", "-b", "item", linked)
|
||||
return main, linked
|
||||
}
|
||||
|
||||
func TestStorageWorktreesRequireMergedCleanInactiveAndUnlocked(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
main, linked := storageTestRepo(t)
|
||||
discover := func(protected ...string) storageWorktree {
|
||||
t.Helper()
|
||||
trees, problems := discoverStorageWorktrees(ctx, []string{main, linked}, protected)
|
||||
if len(problems) > 0 || len(trees) != 1 {
|
||||
t.Fatalf("discovery %+v %+v", trees, problems)
|
||||
}
|
||||
return trees[0]
|
||||
}
|
||||
tree := discover()
|
||||
canonicalMain, err := filepath.EvalSymlinks(main)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !tree.Eligible || tree.Repository != canonicalMain {
|
||||
t.Fatalf("clean merged linked tree not eligible: %+v", tree)
|
||||
}
|
||||
if discover(linked).Eligible {
|
||||
t.Fatal("open chat/run workspace eligible")
|
||||
}
|
||||
if discover(filepath.Join(linked, "subdirectory")).Eligible {
|
||||
t.Fatal("nested active workspace eligible")
|
||||
}
|
||||
storageTestFile(t, filepath.Join(linked, "draft"), "do not delete")
|
||||
if got := discover(); got.Eligible || !strings.Contains(got.Reason, "Uncommitted") {
|
||||
t.Fatalf("dirty tree %+v", got)
|
||||
}
|
||||
if err := removeStorageWorktree(ctx, tree, nil); err == nil {
|
||||
t.Fatal("stale candidate removed newly dirty tree")
|
||||
}
|
||||
if err := os.Remove(filepath.Join(linked, "draft")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storageTestGit(t, main, "worktree", "lock", linked)
|
||||
if discover().Eligible {
|
||||
t.Fatal("locked tree eligible")
|
||||
}
|
||||
storageTestGit(t, main, "worktree", "unlock", linked)
|
||||
storageTestFile(t, filepath.Join(linked, "tracked"), "unlanded")
|
||||
storageTestGit(t, linked, "add", "tracked")
|
||||
storageTestGit(t, linked, "commit", "-m", "unlanded")
|
||||
if got := discover(); got.Eligible || !strings.Contains(got.Reason, "merged") {
|
||||
t.Fatalf("unlanded tree %+v", got)
|
||||
}
|
||||
storageTestGit(t, main, "merge", "--ff-only", "item")
|
||||
if !discover().Eligible {
|
||||
t.Fatal("landed clean tree not eligible")
|
||||
}
|
||||
if err := removeStorageWorktree(ctx, discover(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(linked); !os.IsNotExist(err) {
|
||||
t.Fatalf("worktree remains: %v", err)
|
||||
}
|
||||
storageTestGit(t, main, "show-ref", "--verify", "refs/heads/item")
|
||||
if _, err := os.Stat(main); err != nil {
|
||||
t.Fatal("main checkout removed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
type storageWalker struct {
|
||||
ctx context.Context
|
||||
seen map[string]bool
|
||||
inodes map[[2]uint64]bool
|
||||
entries int
|
||||
}
|
||||
|
||||
func (w *storageWalker) measure(root string, category *storageCategory) {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return
|
||||
}
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
storageMeasureError(category, err)
|
||||
return
|
||||
}
|
||||
info, err := os.Lstat(root)
|
||||
if os.IsNotExist(err) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
storageMeasureError(category, err)
|
||||
return
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
storageMeasureError(category, fmt.Errorf("symlink root excluded: %s", root))
|
||||
return
|
||||
}
|
||||
if err := w.visit(root, category, 0); err != nil {
|
||||
storageMeasureError(category, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Read directories in bounded chunks rather than allocating a whole directory
|
||||
// listing. Compare opened directory identity to avoid traversing a replacement
|
||||
// symlink between Lstat and Open. Regular files are stat-only, never opened.
|
||||
func (w *storageWalker) visit(path string, category *storageCategory, depth int) error {
|
||||
if w.ctx.Err() != nil {
|
||||
return w.ctx.Err()
|
||||
}
|
||||
w.entries++
|
||||
if w.entries > 500000 {
|
||||
return fmt.Errorf("storage scan entry limit reached")
|
||||
}
|
||||
if depth > 128 {
|
||||
return fmt.Errorf("storage scan depth limit reached")
|
||||
}
|
||||
if w.seen[path] {
|
||||
return nil
|
||||
}
|
||||
w.seen[path] = true
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
storageMeasureError(category, err)
|
||||
return nil
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return nil
|
||||
}
|
||||
if info.ModTime().After(category.ModifiedAt) {
|
||||
category.ModifiedAt = info.ModTime().UTC()
|
||||
}
|
||||
if info.Mode().IsRegular() {
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
key := [2]uint64{uint64(stat.Dev), uint64(stat.Ino)}
|
||||
if w.inodes[key] {
|
||||
return nil
|
||||
}
|
||||
w.inodes[key] = true
|
||||
}
|
||||
category.Files++
|
||||
category.Bytes += info.Size()
|
||||
return nil
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
directory, err := os.Open(path)
|
||||
if err != nil {
|
||||
storageMeasureError(category, err)
|
||||
return nil
|
||||
}
|
||||
defer directory.Close()
|
||||
opened, err := directory.Stat()
|
||||
if err != nil || !os.SameFile(info, opened) {
|
||||
storageMeasureError(category, fmt.Errorf("directory changed during storage scan: %s", path))
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
if w.ctx.Err() != nil {
|
||||
return w.ctx.Err()
|
||||
}
|
||||
entries, err := directory.ReadDir(256)
|
||||
if err != nil && err != io.EOF {
|
||||
storageMeasureError(category, err)
|
||||
return nil
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if err := w.visit(filepath.Join(path, entry.Name()), category, depth+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func storageMeasureError(category *storageCategory, err error) {
|
||||
category.Partial = true
|
||||
if category.Error == "" {
|
||||
category.Error = err.Error()
|
||||
}
|
||||
}
|
||||
|
||||
func measureStoragePath(ctx context.Context, path string) (files, bytes int64, partial bool, problems []string) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
category := storageCategory{}
|
||||
walker := storageWalker{ctx: ctx, seen: map[string]bool{}, inodes: map[[2]uint64]bool{}}
|
||||
walker.measure(path, &category)
|
||||
if category.Error != "" {
|
||||
problems = append(problems, category.Error)
|
||||
}
|
||||
return category.Files, category.Bytes, category.Partial, problems
|
||||
}
|
||||
Reference in New Issue
Block a user