Files
pipelines/internal/server/storage_inventory_walk.go
T

140 lines
3.2 KiB
Go

package server
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"time"
)
type storageWalker struct {
ctx context.Context
seen map[string]bool
inodes map[[2]uint64]bool
entries int
}
func (w *storageWalker) measure(root string, category *storageCategory) {
if strings.TrimSpace(root) == "" {
return
}
root, err := filepath.Abs(root)
if err != nil {
storageMeasureError(category, err)
return
}
info, err := os.Lstat(root)
if os.IsNotExist(err) {
return
}
if err != nil {
storageMeasureError(category, err)
return
}
if info.Mode()&os.ModeSymlink != 0 {
storageMeasureError(category, fmt.Errorf("symlink root excluded: %s", root))
return
}
if err := w.visit(root, category, 0); err != nil {
storageMeasureError(category, err)
}
}
// Read directories in bounded chunks rather than allocating a whole directory
// listing. Compare opened directory identity to avoid traversing a replacement
// symlink between Lstat and Open. Regular files are stat-only, never opened.
func (w *storageWalker) visit(path string, category *storageCategory, depth int) error {
if w.ctx.Err() != nil {
return w.ctx.Err()
}
w.entries++
if w.entries > 500000 {
return fmt.Errorf("storage scan entry limit reached")
}
if depth > 128 {
return fmt.Errorf("storage scan depth limit reached")
}
if w.seen[path] {
return nil
}
w.seen[path] = true
info, err := os.Lstat(path)
if err != nil {
storageMeasureError(category, err)
return nil
}
if info.Mode()&os.ModeSymlink != 0 {
return nil
}
if info.ModTime().After(category.ModifiedAt) {
category.ModifiedAt = info.ModTime().UTC()
}
if info.Mode().IsRegular() {
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
key := [2]uint64{uint64(stat.Dev), uint64(stat.Ino)}
if w.inodes[key] {
return nil
}
w.inodes[key] = true
}
category.Files++
category.Bytes += info.Size()
return nil
}
if !info.IsDir() {
return nil
}
directory, err := os.Open(path)
if err != nil {
storageMeasureError(category, err)
return nil
}
defer directory.Close()
opened, err := directory.Stat()
if err != nil || !os.SameFile(info, opened) {
storageMeasureError(category, fmt.Errorf("directory changed during storage scan: %s", path))
return nil
}
for {
if w.ctx.Err() != nil {
return w.ctx.Err()
}
entries, err := directory.ReadDir(256)
if err != nil && err != io.EOF {
storageMeasureError(category, err)
return nil
}
for _, entry := range entries {
if err := w.visit(filepath.Join(path, entry.Name()), category, depth+1); err != nil {
return err
}
}
if err == io.EOF {
return nil
}
}
}
func storageMeasureError(category *storageCategory, err error) {
category.Partial = true
if category.Error == "" {
category.Error = err.Error()
}
}
func measureStoragePath(ctx context.Context, path string) (files, bytes int64, partial bool, problems []string) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
category := storageCategory{}
walker := storageWalker{ctx: ctx, seen: map[string]bool{}, inodes: map[[2]uint64]bool{}}
walker.measure(path, &category)
if category.Error != "" {
problems = append(problems, category.Error)
}
return category.Files, category.Bytes, category.Partial, problems
}