// Package filediff captures bounded, read-only stage-local workspace changes. package filediff import ( "bytes" "context" "crypto/sha256" "encoding/json" "errors" "fmt" "io" "io/fs" "os" "os/exec" "sort" "strings" "syscall" "unicode/utf8" "git.a-wan.net/alan/pipelines/internal/protocol" "github.com/pmezard/go-difflib/difflib" ) const ( maxFiles = 10000 maxFileBytes = 1 << 20 maxSnapshotBytes = 32 << 20 maxPatchBytes = 128 << 10 maxFilePatchBytes = 64 << 10 maxLines = 5000 ) type file struct { data []byte hash [32]byte mode fs.FileMode binary bool } type Snapshot struct { files map[string]file skipped map[string]bool partial bool available bool diagnostic string scope string } func protected(name string) bool { for _, part := range strings.Split(strings.ToLower(name), "/") { if part == ".git" || part == "secrets" || part == ".ssh" || part == ".aws" || part == ".gnupg" || part == ".kube" || part == "node_modules" { return true } if part == ".env" || strings.HasPrefix(part, ".env.") || strings.HasSuffix(part, ".env") || strings.HasSuffix(part, ".pem") || strings.HasSuffix(part, ".key") || strings.HasSuffix(part, ".db") || strings.HasSuffix(part, ".db-wal") || strings.HasSuffix(part, ".db-shm") || strings.HasSuffix(part, ".sqlite") || strings.HasSuffix(part, ".sqlite3") || part == "auth.json" || part == "credentials.json" { return true } } return false } type boundedOutput struct { bytes.Buffer limit int } func (b *boundedOutput) Write(p []byte) (int, error) { if b.Len()+len(p) > b.limit { return 0, fmt.Errorf("file inventory limit exceeded") } return b.Buffer.Write(p) } // Git only supplies an inventory; never compare against HEAD/index or alter them. // Tracked dirty files and untracked, nonignored files are compared as they exist. func inventory(ctx context.Context, directory string, root *os.Root) ([]string, bool, error) { probe := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "rev-parse", "--is-inside-work-tree") if data, err := probe.Output(); err == nil && strings.TrimSpace(string(data)) == "true" { cmd := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "ls-files", "--cached", "--others", "--exclude-standard", "--deduplicate", "-z", "--", ".") out := &boundedOutput{limit: 2 << 20} cmd.Stdout = out if err := cmd.Run(); err != nil { return nil, true, fmt.Errorf("Git file inventory unavailable") } return strings.Split(strings.TrimSuffix(out.String(), "\x00"), "\x00"), true, nil } if ctx.Err() != nil { return nil, false, ctx.Err() } var names []string err := fs.WalkDir(root.FS(), ".", func(name string, entry fs.DirEntry, err error) error { if ctx.Err() != nil { return ctx.Err() } if err != nil { return err } if name != "." && protected(name) { names = append(names, name) if entry.IsDir() { return fs.SkipDir } return nil } if !entry.IsDir() { names = append(names, name) if len(names) > maxFiles { return fmt.Errorf("file inventory limit exceeded") } } return nil }) return names, false, err } func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snapshot { s := &Snapshot{files: map[string]file{}, skipped: map[string]bool{}} root, err := os.OpenRoot(directory) if err != nil { s.diagnostic = "Workspace snapshot unavailable." return s } defer root.Close() s.available = true names, git, err := inventory(ctx, directory, root) s.scope = "Workspace regular files" if git { s.scope = "Git tracked and nonignored files in the run directory" } if err != nil { s.partial = true s.diagnostic = "Workspace inventory incomplete." } // Keep pre-stage paths even if a stage changes .gitignore/the index. known := map[string]bool{} for _, name := range names { known[name] = true } for _, prior := range previous { if prior != nil { for name := range prior.files { if !known[name] { names = append(names, name) known[name] = true } } } } sort.Strings(names) used := 0 for i, name := range names { if name == "" { continue } if i >= maxFiles || ctx.Err() != nil { s.partial = true s.diagnostic = "Workspace capture reached its time/file limit." break } if !fs.ValidPath(name) || protected(name) { s.skipped[name] = true continue } // Reject symlinks in every component, including aliases to sensitive paths. valid := true parts := strings.Split(name, "/") for j := range parts { info, err := root.Lstat(strings.Join(parts[:j+1], "/")) if errors.Is(err, fs.ErrNotExist) { valid = false break } // tracked deletion if err != nil || info.Mode()&os.ModeSymlink != 0 { s.skipped[name] = true valid = false break } } if !valid { continue } info, err := root.Lstat(name) if err != nil || !info.Mode().IsRegular() || info.Size() > maxFileBytes { s.skipped[name] = true continue } if used+int(info.Size()) > maxSnapshotBytes { s.skipped[name] = true s.partial = true continue } f, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0) if err != nil { s.skipped[name] = true continue } opened, statErr := f.Stat() if statErr != nil || !opened.Mode().IsRegular() { f.Close() s.skipped[name] = true continue } data, readErr := io.ReadAll(io.LimitReader(f, maxFileBytes+1)) f.Close() if readErr != nil || len(data) > maxFileBytes { s.skipped[name] = true continue } used += len(data) binary := bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data) s.files[name] = file{data: data, hash: sha256.Sum256(data), mode: opened.Mode().Perm(), binary: binary} } return s } func Compare(before, after *Snapshot) *protocol.StageFileChanges { result := &protocol.StageFileChanges{Status: "complete", Files: []protocol.FileChange{}} if before == nil || after == nil || !before.available || !after.available { result.Status = "unavailable" result.Diagnostic = "Workspace snapshot unavailable; changes could not be compared." return result } result.Scope = before.scope if before.partial || after.partial { result.Status = "partial" result.Diagnostic = "Workspace capture incomplete; some changes may be missing." } skipped := map[string]bool{} for name := range before.skipped { skipped[name] = true } for name := range after.skipped { skipped[name] = true } result.Skipped = len(skipped) if result.Skipped > 0 { result.Status = "partial" result.Diagnostic = "Excluded protected, symlinked, oversized or unreadable files; this is not a complete workspace diff." } keys := map[string]bool{} for name := range before.files { keys[name] = true } for name := range after.files { keys[name] = true } names := make([]string, 0, len(keys)) for name := range keys { names = append(names, name) } sort.Strings(names) remaining := maxPatchBytes retained := 0 for _, name := range names { if skipped[name] { continue } a, aOK := before.files[name] b, bOK := after.files[name] // Missing files in an incomplete inventory cannot prove deletion/addition. if !aOK && before.partial || !bOK && after.partial { continue } if aOK && bOK && a.hash == b.hash && a.mode == b.mode { continue } kind := "modified" if !aOK { kind = "added" } if !bOK { kind = "deleted" } change := protocol.FileChange{Path: name, Kind: kind, Binary: a.binary || b.binary} if aOK { change.BeforeMode = fmt.Sprintf("%04o", a.mode) } if bOK { change.AfterMode = fmt.Sprintf("%04o", b.mode) } if !change.Binary && a.hash != b.hash { if remaining == 0 || bytes.Count(a.data, []byte("\n"))+bytes.Count(b.data, []byte("\n")) > maxLines { change.Truncated = true } else { from, to := "a/"+name, "b/"+name if !aOK { from = "/dev/null" } if !bOK { to = "/dev/null" } // Quote paths so names cannot forge headers or terminal control sequences. patch, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: diffLines(a.data), B: diffLines(b.data), FromFile: fmt.Sprintf("%q", from), ToFile: fmt.Sprintf("%q", to), Context: 3}) if err != nil { change.Truncated = true } else { limit := min(remaining, maxFilePatchBytes) if len(patch) > limit { patch = patch[:limit] for !utf8.ValidString(patch) && len(patch) > 0 { patch = patch[:len(patch)-1] } change.Truncated = true } change.Patch = patch remaining -= len(patch) } } } if change.Truncated { result.Status = "partial" result.Diagnostic = "Some patches were omitted/truncated by capture limits." } encoded, _ := json.Marshal(change) if retained+len(encoded)+1 > maxPatchBytes { change.Patch = "" change.Truncated = true encoded, _ = json.Marshal(change) } if retained+len(encoded)+1 > maxPatchBytes { result.Omitted++ result.Status = "partial" result.Diagnostic = "Additional file changes omitted by the retained evidence size limit." continue } if change.Truncated { result.Status = "partial" } retained += len(encoded) + 1 result.Files = append(result.Files, change) } return result } func diffLines(data []byte) []string { if len(data) == 0 { return nil } lines := strings.SplitAfter(string(data), "\n") if lines[len(lines)-1] == "" { return lines[:len(lines)-1] } lines[len(lines)-1] += "\n\\ No newline at end of file\n" return lines }