// Package filediff captures bounded, read-only stage-local workspace changes. package filediff import ( "bufio" "bytes" "context" "crypto/sha256" "encoding/json" "errors" "fmt" "io" "io/fs" "os" "os/exec" "sort" "strconv" "strings" "sync/atomic" "syscall" "unicode/utf8" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/workenv" "github.com/pmezard/go-difflib/difflib" ) const ( maxFiles = 10000 maxFileBytes = 1 << 20 maxSnapshotBytes = 32 << 20 maxPatchBytes = 128 << 10 maxFilePatchBytes = 64 << 10 maxLines = 5000 binarySniffBytes = 8 << 10 ) // maxHashBytes bounds one hash-only file; larger files are skipped (and, in a // Git working tree, listed only when their Git status changes). It is a // variable so package tests can exercise that path without huge files. var maxHashBytes int64 = 256 << 20 // snapshotBudget is the retained text bytes per snapshot. Files beyond it (or // beyond the per-file limit) are still compared by content hash and listed // without a patch, so an over-budget workspace never hides which files changed. var snapshotBudget atomic.Int64 func init() { snapshotBudget.Store(maxSnapshotBytes) } // SetSnapshotBudgetForTesting lowers the retained-text budget so tests can // exercise an over-budget workspace without writing tens of megabytes. It // returns a function restoring the previous budget. Production never calls it. func SetSnapshotBudgetForTesting(n int64) (restore func()) { previous := snapshotBudget.Swap(n) return func() { snapshotBudget.Store(previous) } } // No-patch reasons recorded on protocol.FileChange.NoPatchReason. const ( reasonBinary = "binary" // binary content; metadata only reasonFileSize = "file_size" // text over the 1 MiB per-file limit reasonBudget = "budget" // the per-snapshot retained-text budget ran out reasonLines = "lines" // over the combined line limit for one comparison reasonEvidence = "evidence_size" // the per-visit retained patch/evidence limit ran out reasonGit = "git_status" // seen only as a Git status change; not compared reasonBaseline = "baseline" // the stage-start content could not be recovered ) type file struct { data []byte hash [32]byte size int64 mode fs.FileMode binary bool // hashOnly marks a text file compared by content hash only: it exceeded the // per-file or per-snapshot text budget, so no patch can be produced. limit // names which (reasonFileSize or reasonBudget). hashOnly bool limit string // blob is the Git index object id of a text file that was clean (matching // the index and HEAD) when captured. Its content is not retained: Git // already holds it, and a later capture recovers it only if the file // changed. This keeps the text budget for files Git has no baseline for. blob string } // baseline is stage-start content recovered from Git for a changed file whose // start snapshot kept only its blob id; hash is verified against that snapshot. type baseline struct { data []byte hash [32]byte } type Snapshot struct { files map[string]file skipped map[string]bool partial bool available bool diagnostic string scope string // gitStatus maps run-relative paths to their porcelain XY status code when // the run directory is a Git working tree and the status read succeeded // (gitOK). Compare uses it to list changed paths the content snapshot could // not compare (time/file limit, unreadable). gitStatus map[string]string gitOK bool // baselines holds Git-recovered start content for files that changed since // the previous snapshot; missing records why one could not be recovered. baselines map[string]baseline missing map[string]string } // gitCommand runs git for directory with the run's directory environment // (workenv.WithEnvironment) when ctx carries one, as the stage's own git sees. func gitCommand(ctx context.Context, directory string, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, "git", args...) cmd.Env = workenv.CommandEnv(ctx, directory) return cmd } // readGitIndex returns the stage-0 index object id of every tracked path in // the run directory (paths relative to it, as ls-files prints them). func readGitIndex(ctx context.Context, directory string) (map[string]string, bool) { cmd := gitCommand(ctx, directory, "-c", "core.fsmonitor=false", "-C", directory, "ls-files", "--stage", "-z", "--", ".") out := &boundedOutput{limit: 8 << 20} cmd.Stdout = out if err := cmd.Run(); err != nil { return nil, false } blobs := map[string]string{} for _, entry := range strings.Split(out.String(), "\x00") { meta, name, ok := strings.Cut(entry, "\t") fields := strings.Fields(meta) if !ok || len(fields) != 3 || fields[2] != "0" || name == "" { continue } blobs[name] = fields[1] } return blobs, true } // readBlobs reads Git objects by id with one `git cat-file --batch`. Objects // that are missing, not blobs or larger than maxFileBytes are left out. func readBlobs(ctx context.Context, directory string, ids []string) map[string][]byte { result := map[string][]byte{} if len(ids) == 0 { return result } cmd := gitCommand(ctx, directory, "-c", "core.fsmonitor=false", "-C", directory, "cat-file", "--batch") cmd.Stdin = strings.NewReader(strings.Join(ids, "\n") + "\n") pipe, err := cmd.StdoutPipe() if err != nil { return result } if err := cmd.Start(); err != nil { return result } reader := bufio.NewReader(pipe) for range ids { header, err := reader.ReadString('\n') if err != nil { break } fields := strings.Fields(header) if len(fields) != 3 { continue // " missing" carries no content } size, err := strconv.ParseInt(fields[2], 10, 64) if err != nil || size < 0 { break } if fields[1] != "blob" || size > maxFileBytes { if _, err := io.CopyN(io.Discard, reader, size+1); err != nil { break } continue } data := make([]byte, size+1) // content plus the trailing newline if _, err := io.ReadFull(reader, data); err != nil { break } result[fields[0]] = data[:size] } _, _ = io.Copy(io.Discard, pipe) _ = cmd.Wait() return result } // readGitStatus reads `git status --porcelain=v1 -z` for the run directory and // returns its paths relative to that directory. It only reads: optional locks // are disabled so the index is never refreshed or rewritten. func readGitStatus(ctx context.Context, directory string) (map[string]string, bool) { prefixOut, err := gitCommand(ctx, directory, "-c", "core.fsmonitor=false", "-C", directory, "rev-parse", "--show-prefix").Output() if err != nil { return nil, false } prefix := strings.TrimSpace(string(prefixOut)) cmd := gitCommand(ctx, directory, "--no-optional-locks", "-c", "core.fsmonitor=false", "-C", directory, "status", "--porcelain=v1", "-z", "--no-renames", "--untracked-files=all", "--", ".") out := &boundedOutput{limit: 4 << 20} cmd.Stdout = out if err := cmd.Run(); err != nil { return nil, false } status := map[string]string{} for _, entry := range strings.Split(out.String(), "\x00") { if len(entry) < 4 || entry[2] != ' ' { continue } name, ok := strings.CutPrefix(entry[3:], prefix) if !ok || name == "" { continue // outside the run directory } status[name] = entry[:2] } return status, true } // hashFile streams f from its start into a content hash without retaining it. // It returns ok=false when the file exceeds maxHashBytes or cannot be read. func hashFile(f *os.File) (hash [32]byte, binary bool, ok bool) { if _, err := f.Seek(0, io.SeekStart); err != nil { return hash, false, false } h := sha256.New() head := make([]byte, binarySniffBytes) n, err := io.ReadFull(f, head) if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) { return hash, false, false } head = head[:n] h.Write(head) copied, err := io.Copy(h, io.LimitReader(f, maxHashBytes+1-int64(n))) if err != nil || int64(n)+copied > maxHashBytes { return hash, false, false } copy(hash[:], h.Sum(nil)) return hash, bytes.IndexByte(head, 0) >= 0, true } func protected(name string) bool { for _, part := range strings.Split(strings.ToLower(name), "/") { // A committed template (.env.example, config.key.sample) holds // placeholders, not secrets; excluding it would mark every capture of // such a repository incomplete. if template(part) { continue } if part == ".git" || part == "secrets" || part == ".ssh" || part == ".aws" || part == ".gnupg" || part == ".kube" || part == "node_modules" { return true } if part == ".env" || strings.HasPrefix(part, ".env.") || strings.HasSuffix(part, ".env") || strings.HasSuffix(part, ".pem") || strings.HasSuffix(part, ".key") || strings.HasSuffix(part, ".db") || strings.HasSuffix(part, ".db-wal") || strings.HasSuffix(part, ".db-shm") || strings.HasSuffix(part, ".sqlite") || strings.HasSuffix(part, ".sqlite3") || part == "auth.json" || part == "credentials.json" { return true } } return false } func template(part string) bool { for _, suffix := range []string{".example", ".sample", ".template", ".dist"} { if strings.HasSuffix(part, suffix) { return true } } return false } type boundedOutput struct { bytes.Buffer limit int } func (b *boundedOutput) Write(p []byte) (int, error) { if b.Len()+len(p) > b.limit { return 0, fmt.Errorf("file inventory limit exceeded") } return b.Buffer.Write(p) } // inventory lists the files to snapshot. Git is only read, never altered; // the baseline is still stage-start content (see Capture and recoverBaselines). func inventory(ctx context.Context, directory string, root *os.Root) ([]string, bool, error) { probe := gitCommand(ctx, directory, "-c", "core.fsmonitor=false", "-C", directory, "rev-parse", "--is-inside-work-tree") if data, err := probe.Output(); err == nil && strings.TrimSpace(string(data)) == "true" { cmd := gitCommand(ctx, directory, "-c", "core.fsmonitor=false", "-C", directory, "ls-files", "--cached", "--others", "--exclude-standard", "--deduplicate", "-z", "--", ".") out := &boundedOutput{limit: 2 << 20} cmd.Stdout = out if err := cmd.Run(); err != nil { return nil, true, fmt.Errorf("Git file inventory unavailable") } return strings.Split(strings.TrimSuffix(out.String(), "\x00"), "\x00"), true, nil } if ctx.Err() != nil { return nil, false, ctx.Err() } var names []string err := fs.WalkDir(root.FS(), ".", func(name string, entry fs.DirEntry, err error) error { if ctx.Err() != nil { return ctx.Err() } if err != nil { return err } if name != "." && protected(name) { names = append(names, name) if entry.IsDir() { return fs.SkipDir } return nil } if !entry.IsDir() { names = append(names, name) if len(names) > maxFiles { return fmt.Errorf("file inventory limit exceeded") } } return nil }) return names, false, err } func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snapshot { s := &Snapshot{files: map[string]file{}, skipped: map[string]bool{}} root, err := os.OpenRoot(directory) if err != nil { s.diagnostic = "Workspace snapshot unavailable." return s } defer root.Close() s.available = true names, git, err := inventory(ctx, directory, root) s.scope = "Workspace regular files" if git { s.scope = "Git tracked and nonignored files in the run directory" } if err != nil { s.partial = true s.diagnostic = "Workspace inventory incomplete." } var prev *Snapshot for _, prior := range previous { if prior != nil { prev = prior break } } // A stage-start capture keeps only the blob id of a text file that is clean // in Git (no status entry, stage-0 index entry): Git holds its content. Only // dirty and untracked files, which have no Git baseline, spend the budget. var blobs map[string]string if git { s.gitStatus, s.gitOK = readGitStatus(ctx, directory) if s.gitOK && prev == nil { blobs, _ = readGitIndex(ctx, directory) } } // Keep pre-stage paths even if a stage changes .gitignore/the index. known := map[string]bool{} for _, name := range names { known[name] = true } for _, prior := range previous { if prior != nil { for name := range prior.files { if !known[name] { names = append(names, name) known[name] = true } } } } sort.Strings(names) used := 0 for i, name := range names { if name == "" { continue } if i >= maxFiles || ctx.Err() != nil { s.partial = true s.diagnostic = "Workspace capture reached its time/file limit." break } if !fs.ValidPath(name) || protected(name) { s.skipped[name] = true continue } // Reject symlinks in every component, including aliases to sensitive paths. valid := true parts := strings.Split(name, "/") for j := range parts { info, err := root.Lstat(strings.Join(parts[:j+1], "/")) if errors.Is(err, fs.ErrNotExist) { valid = false break } // tracked deletion if err != nil || info.Mode()&os.ModeSymlink != 0 { s.skipped[name] = true valid = false break } } if !valid { continue } info, err := root.Lstat(name) if err != nil || !info.Mode().IsRegular() || info.Size() > maxHashBytes { s.skipped[name] = true continue } f, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0) if err != nil { s.skipped[name] = true continue } opened, statErr := f.Stat() if statErr != nil || !opened.Mode().IsRegular() { f.Close() s.skipped[name] = true continue } if opened.Size() <= maxFileBytes { data, readErr := io.ReadAll(io.LimitReader(f, maxFileBytes+1)) if readErr != nil { f.Close() s.skipped[name] = true continue } if len(data) <= maxFileBytes { f.Close() binary := bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data) entry := file{hash: sha256.Sum256(data), size: int64(len(data)), mode: opened.Mode().Perm(), binary: binary} _, dirty := s.gitStatus[name] p, seen := file{}, false if prev != nil { p, seen = prev.files[name] } // Binary changes need only metadata and a hash, never retained patch bytes. // Counting screenshots against the retained-text budget can otherwise // exhaust it before any source files are reached in sorted inventory order. // Text beyond the budget keeps only its hash, so a change to it is still // listed (without a patch) instead of being skipped unseen. switch { case binary: case seen && p.hash == entry.hash && !p.binary: // Unchanged since the previous snapshot: share its representation // without spending this snapshot's budget on it again. entry.data, entry.blob, entry.hashOnly, entry.limit = p.data, p.blob, p.hashOnly, p.limit case blobs[name] != "" && !dirty: entry.blob = blobs[name] case int64(used+len(data)) > snapshotBudget.Load(): entry.hashOnly, entry.limit = true, reasonBudget default: entry.data = data used += len(data) } s.files[name] = entry continue } // The file grew past the per-file limit while being read: hash it. } hash, binary, ok := hashFile(f) f.Close() if !ok { s.skipped[name] = true continue } s.files[name] = file{hash: hash, size: opened.Size(), mode: opened.Mode().Perm(), binary: binary, hashOnly: !binary, limit: reasonFileSize} } if prev != nil { s.recoverBaselines(ctx, directory, prev, used) } return s } // recoverBaselines reads, in one Git batch, the stage-start content of every // file prev kept only as a clean blob id that has since changed or gone. The // content is used only if its hash matches what prev read from disk, so // clean/smudge filters, line-ending conversion or an index changed under the // capture can never yield a wrong patch; such files are listed without one. func (s *Snapshot) recoverBaselines(ctx context.Context, directory string, prev *Snapshot, used int) { var names []string for name, p := range prev.files { if p.blob == "" || p.data != nil || p.binary || s.skipped[name] { continue } now, ok := s.files[name] if ok && now.hash == p.hash || !ok && s.partial { continue // unchanged, or an incomplete inventory cannot prove deletion } names = append(names, name) } if len(names) == 0 { return } sort.Strings(names) s.baselines, s.missing = map[string]baseline{}, map[string]string{} ids := make([]string, 0, len(names)) wanted := make([]string, 0, len(names)) for _, name := range names { p := prev.files[name] if int64(used)+p.size > snapshotBudget.Load() { s.missing[name] = reasonBudget continue } used += int(p.size) ids = append(ids, p.blob) wanted = append(wanted, name) } contents := readBlobs(ctx, directory, ids) for _, name := range wanted { p := prev.files[name] data, ok := contents[p.blob] if !ok || sha256.Sum256(data) != p.hash { s.missing[name] = reasonBaseline continue } s.baselines[name] = baseline{data: data, hash: p.hash} } } func Compare(before, after *Snapshot) *protocol.StageFileChanges { result := &protocol.StageFileChanges{Status: "complete", Files: []protocol.FileChange{}} if before == nil || after == nil || !before.available || !after.available { result.Status = "unavailable" result.Diagnostic = "Workspace snapshot unavailable; changes could not be compared." return result } result.Scope = before.scope if before.partial || after.partial { result.Status = "partial" result.Diagnostic = "Workspace capture incomplete; some changes may be missing." } skipped := map[string]bool{} for name := range before.skipped { skipped[name] = true } for name := range after.skipped { skipped[name] = true } result.Skipped = len(skipped) if result.Skipped > 0 { result.Status = "partial" result.Diagnostic = "Excluded protected, symlinked, oversized or unreadable files; this is not a complete workspace diff." } keys := map[string]bool{} for name := range before.files { keys[name] = true } for name := range after.files { keys[name] = true } names := make([]string, 0, len(keys)) for name := range keys { names = append(names, name) } sort.Strings(names) remaining := maxPatchBytes retained := 0 add := func(change protocol.FileChange) { if change.Truncated { result.Status = "partial" if !change.NoPatch { result.Diagnostic = "Some patches were omitted/truncated by capture limits." } } encoded, _ := json.Marshal(change) if retained+len(encoded)+1 > maxPatchBytes { if change.Patch != "" { change.Patch = "" change.NoPatchReason = reasonEvidence } change.Truncated = true encoded, _ = json.Marshal(change) } if retained+len(encoded)+1 > maxPatchBytes { result.Omitted++ result.Status = "partial" result.Diagnostic = "Additional file changes omitted by the retained evidence size limit." return } if change.Truncated { result.Status = "partial" } retained += len(encoded) + 1 result.Files = append(result.Files, change) } listed := 0 // decided records every path the content comparison settled (changed or // not), so the Git-status pass below only speaks for the rest. decided := map[string]bool{} for _, name := range names { if skipped[name] { continue } a, aOK := before.files[name] b, bOK := after.files[name] // Missing files in an incomplete inventory cannot prove deletion/addition. if !aOK && before.partial || !bOK && after.partial { continue } decided[name] = true if aOK && bOK && a.hash == b.hash && a.mode == b.mode { continue } kind := "modified" if !aOK { kind = "added" } if !bOK { kind = "deleted" } change := protocol.FileChange{Path: name, Kind: kind, Binary: a.binary || b.binary} if aOK { change.BeforeMode = fmt.Sprintf("%04o", a.mode) } if bOK { change.AfterMode = fmt.Sprintf("%04o", b.mode) } // A clean-at-start file kept only its Git blob id; its start content // comes from the after snapshot's verified Git recovery. aData, unavailable := a.data, "" if aOK && !a.binary && a.blob != "" && a.data == nil { if base, ok := after.baselines[name]; ok && base.hash == a.hash { aData = base.data } else if unavailable = after.missing[name]; unavailable == "" { unavailable = reasonBaseline } } if change.Binary { if a.hash != b.hash { change.NoPatchReason = reasonBinary } } else if a.hash != b.hash && (a.hashOnly || b.hashOnly || unavailable != "") { // Content compared by hash only: list it, without a patch. change.Truncated = true change.NoPatch = true switch { case a.limit == reasonFileSize || b.limit == reasonFileSize: change.NoPatchReason = reasonFileSize case unavailable != "": change.NoPatchReason = unavailable default: change.NoPatchReason = reasonBudget } listed++ } else if a.hash != b.hash { if remaining == 0 { change.Truncated, change.NoPatchReason = true, reasonEvidence } else if bytes.Count(aData, []byte("\n"))+bytes.Count(b.data, []byte("\n")) > maxLines { change.Truncated, change.NoPatchReason = true, reasonLines } else { from, to := "a/"+name, "b/"+name if !aOK { from = "/dev/null" } if !bOK { to = "/dev/null" } // Quote paths so names cannot forge headers or terminal control sequences. patch, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: diffLines(aData), B: diffLines(b.data), FromFile: fmt.Sprintf("%q", from), ToFile: fmt.Sprintf("%q", to), Context: 3}) if err != nil { change.Truncated = true } else { limit := min(remaining, maxFilePatchBytes) if len(patch) > limit { patch = patch[:limit] for !utf8.ValidString(patch) && len(patch) > 0 { patch = patch[:len(patch)-1] } change.Truncated = true } change.Patch = patch remaining -= len(patch) } } } add(change) } // In a Git working tree, paths the content snapshot could not settle // (skipped as unreadable/oversized, or beyond the time/file limit) are still // listed when their Git status changed during the visit. A path dirty with // the same status before and after cannot be proven changed and stays // counted as skipped. Protected paths are never listed. if before.gitOK && after.gitOK { gitNames := map[string]bool{} for name := range before.gitStatus { gitNames[name] = true } for name := range after.gitStatus { gitNames[name] = true } ordered := make([]string, 0, len(gitNames)) for name := range gitNames { if !decided[name] && fs.ValidPath(name) && !protected(name) { ordered = append(ordered, name) } } sort.Strings(ordered) for _, name := range ordered { was, now := before.gitStatus[name], after.gitStatus[name] if was == now { continue } kind := "modified" switch { case now == "" && was == "??", strings.Contains(now, "D"): kind = "deleted" case was == "" && (now == "??" || strings.HasPrefix(now, "A")): kind = "added" } add(protocol.FileChange{Path: name, Kind: kind, Truncated: true, NoPatch: true, NoPatchReason: reasonGit}) listed++ } } if listed > 0 && result.Status == "partial" && result.Omitted == 0 { result.Diagnostic = "Some changed files exceeded capture limits and are listed without patches; this is not a complete workspace diff." } return result } func diffLines(data []byte) []string { if len(data) == 0 { return nil } lines := strings.SplitAfter(string(data), "\n") if lines[len(lines)-1] == "" { return lines[:len(lines)-1] } lines[len(lines)-1] += "\n\\ No newline at end of file\n" return lines }