Files
pipelines/internal/filediff/snapshot.go
T
alan f16d61cdfa Skip fallback needs complete, empty evidence; capture lists over-budget changes
The server skip fallback for stages with a skipped edge now applies only
when every earlier visit recorded a complete file-change capture with no
changed files and no image artifact. Any change (no repo-specific UI path
or extension list), any partial/unavailable/missing capture, a human
decision or a checkpoint-retry prefix runs the model. auto_skip=false opts
a stage out. The fallback verdict names the server fallback.

Capture compares over-budget text by hash and, in Git, lists paths it
could not compare when their git status changes, marked no_patch, so a
large workspace no longer records files: [] for a real edit. Run detail
gains a warnings list for partial/unavailable captures, also printed by
inspect on stderr.
2026-11-04 12:56:40 +00:00

507 lines
16 KiB
Go

// Package filediff captures bounded, read-only stage-local workspace changes.
package filediff
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"sort"
"strings"
"sync/atomic"
"syscall"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
"github.com/pmezard/go-difflib/difflib"
)
const (
maxFiles = 10000
maxFileBytes = 1 << 20
maxSnapshotBytes = 32 << 20
maxPatchBytes = 128 << 10
maxFilePatchBytes = 64 << 10
maxLines = 5000
binarySniffBytes = 8 << 10
)
// maxHashBytes bounds one hash-only file; larger files are skipped (and, in a
// Git working tree, listed only when their Git status changes). It is a
// variable so package tests can exercise that path without huge files.
var maxHashBytes int64 = 256 << 20
// snapshotBudget is the retained text bytes per snapshot. Files beyond it (or
// beyond the per-file limit) are still compared by content hash and listed
// without a patch, so an over-budget workspace never hides which files changed.
var snapshotBudget atomic.Int64
func init() { snapshotBudget.Store(maxSnapshotBytes) }
// SetSnapshotBudgetForTesting lowers the retained-text budget so tests can
// exercise an over-budget workspace without writing tens of megabytes. It
// returns a function restoring the previous budget. Production never calls it.
func SetSnapshotBudgetForTesting(n int64) (restore func()) {
previous := snapshotBudget.Swap(n)
return func() { snapshotBudget.Store(previous) }
}
type file struct {
data []byte
hash [32]byte
mode fs.FileMode
binary bool
// hashOnly marks a text file compared by content hash only: it exceeded the
// per-file or per-snapshot text budget, so no patch can be produced.
hashOnly bool
}
type Snapshot struct {
files map[string]file
skipped map[string]bool
partial bool
available bool
diagnostic string
scope string
// gitStatus maps run-relative paths to their porcelain XY status code when
// the run directory is a Git working tree and the status read succeeded
// (gitOK). Compare uses it to list changed paths the content snapshot could
// not compare (time/file limit, unreadable); it never reads HEAD/index content.
gitStatus map[string]string
gitOK bool
}
// readGitStatus reads `git status --porcelain=v1 -z` for the run directory and
// returns its paths relative to that directory. It only reads: optional locks
// are disabled so the index is never refreshed or rewritten.
func readGitStatus(ctx context.Context, directory string) (map[string]string, bool) {
prefixOut, err := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "rev-parse", "--show-prefix").Output()
if err != nil {
return nil, false
}
prefix := strings.TrimSpace(string(prefixOut))
cmd := exec.CommandContext(ctx, "git", "--no-optional-locks", "-c", "core.fsmonitor=false", "-C", directory, "status", "--porcelain=v1", "-z", "--no-renames", "--untracked-files=all", "--", ".")
out := &boundedOutput{limit: 4 << 20}
cmd.Stdout = out
if err := cmd.Run(); err != nil {
return nil, false
}
status := map[string]string{}
for _, entry := range strings.Split(out.String(), "\x00") {
if len(entry) < 4 || entry[2] != ' ' {
continue
}
name, ok := strings.CutPrefix(entry[3:], prefix)
if !ok || name == "" {
continue // outside the run directory
}
status[name] = entry[:2]
}
return status, true
}
// hashFile streams f from its start into a content hash without retaining it.
// It returns ok=false when the file exceeds maxHashBytes or cannot be read.
func hashFile(f *os.File) (hash [32]byte, binary bool, ok bool) {
if _, err := f.Seek(0, io.SeekStart); err != nil {
return hash, false, false
}
h := sha256.New()
head := make([]byte, binarySniffBytes)
n, err := io.ReadFull(f, head)
if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) {
return hash, false, false
}
head = head[:n]
h.Write(head)
copied, err := io.Copy(h, io.LimitReader(f, maxHashBytes+1-int64(n)))
if err != nil || int64(n)+copied > maxHashBytes {
return hash, false, false
}
copy(hash[:], h.Sum(nil))
return hash, bytes.IndexByte(head, 0) >= 0, true
}
func protected(name string) bool {
for _, part := range strings.Split(strings.ToLower(name), "/") {
if part == ".git" || part == "secrets" || part == ".ssh" || part == ".aws" || part == ".gnupg" || part == ".kube" || part == "node_modules" {
return true
}
if part == ".env" || strings.HasPrefix(part, ".env.") || strings.HasSuffix(part, ".env") || strings.HasSuffix(part, ".pem") || strings.HasSuffix(part, ".key") || strings.HasSuffix(part, ".db") || strings.HasSuffix(part, ".db-wal") || strings.HasSuffix(part, ".db-shm") || strings.HasSuffix(part, ".sqlite") || strings.HasSuffix(part, ".sqlite3") || part == "auth.json" || part == "credentials.json" {
return true
}
}
return false
}
type boundedOutput struct {
bytes.Buffer
limit int
}
func (b *boundedOutput) Write(p []byte) (int, error) {
if b.Len()+len(p) > b.limit {
return 0, fmt.Errorf("file inventory limit exceeded")
}
return b.Buffer.Write(p)
}
// Git only supplies an inventory; never compare against HEAD/index or alter them.
// Tracked dirty files and untracked, nonignored files are compared as they exist.
func inventory(ctx context.Context, directory string, root *os.Root) ([]string, bool, error) {
probe := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "rev-parse", "--is-inside-work-tree")
if data, err := probe.Output(); err == nil && strings.TrimSpace(string(data)) == "true" {
cmd := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "ls-files", "--cached", "--others", "--exclude-standard", "--deduplicate", "-z", "--", ".")
out := &boundedOutput{limit: 2 << 20}
cmd.Stdout = out
if err := cmd.Run(); err != nil {
return nil, true, fmt.Errorf("Git file inventory unavailable")
}
return strings.Split(strings.TrimSuffix(out.String(), "\x00"), "\x00"), true, nil
}
if ctx.Err() != nil {
return nil, false, ctx.Err()
}
var names []string
err := fs.WalkDir(root.FS(), ".", func(name string, entry fs.DirEntry, err error) error {
if ctx.Err() != nil {
return ctx.Err()
}
if err != nil {
return err
}
if name != "." && protected(name) {
names = append(names, name)
if entry.IsDir() {
return fs.SkipDir
}
return nil
}
if !entry.IsDir() {
names = append(names, name)
if len(names) > maxFiles {
return fmt.Errorf("file inventory limit exceeded")
}
}
return nil
})
return names, false, err
}
func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snapshot {
s := &Snapshot{files: map[string]file{}, skipped: map[string]bool{}}
root, err := os.OpenRoot(directory)
if err != nil {
s.diagnostic = "Workspace snapshot unavailable."
return s
}
defer root.Close()
s.available = true
names, git, err := inventory(ctx, directory, root)
s.scope = "Workspace regular files"
if git {
s.scope = "Git tracked and nonignored files in the run directory"
}
if err != nil {
s.partial = true
s.diagnostic = "Workspace inventory incomplete."
}
if git {
s.gitStatus, s.gitOK = readGitStatus(ctx, directory)
}
// Keep pre-stage paths even if a stage changes .gitignore/the index.
known := map[string]bool{}
for _, name := range names {
known[name] = true
}
for _, prior := range previous {
if prior != nil {
for name := range prior.files {
if !known[name] {
names = append(names, name)
known[name] = true
}
}
}
}
sort.Strings(names)
used := 0
for i, name := range names {
if name == "" {
continue
}
if i >= maxFiles || ctx.Err() != nil {
s.partial = true
s.diagnostic = "Workspace capture reached its time/file limit."
break
}
if !fs.ValidPath(name) || protected(name) {
s.skipped[name] = true
continue
}
// Reject symlinks in every component, including aliases to sensitive paths.
valid := true
parts := strings.Split(name, "/")
for j := range parts {
info, err := root.Lstat(strings.Join(parts[:j+1], "/"))
if errors.Is(err, fs.ErrNotExist) {
valid = false
break
} // tracked deletion
if err != nil || info.Mode()&os.ModeSymlink != 0 {
s.skipped[name] = true
valid = false
break
}
}
if !valid {
continue
}
info, err := root.Lstat(name)
if err != nil || !info.Mode().IsRegular() || info.Size() > maxHashBytes {
s.skipped[name] = true
continue
}
f, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0)
if err != nil {
s.skipped[name] = true
continue
}
opened, statErr := f.Stat()
if statErr != nil || !opened.Mode().IsRegular() {
f.Close()
s.skipped[name] = true
continue
}
if opened.Size() <= maxFileBytes {
data, readErr := io.ReadAll(io.LimitReader(f, maxFileBytes+1))
if readErr != nil {
f.Close()
s.skipped[name] = true
continue
}
if len(data) <= maxFileBytes {
f.Close()
binary := bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data)
hash := sha256.Sum256(data)
hashOnly := false
// Binary changes need only metadata and a hash, never retained patch bytes.
// Counting screenshots against the retained-text budget can otherwise
// exhaust it before any source files are reached in sorted inventory order.
// Text beyond the budget keeps only its hash, so a change to it is still
// listed (without a patch) instead of being skipped unseen.
if binary {
data = nil
} else if int64(used+len(data)) > snapshotBudget.Load() {
data = nil
hashOnly = true
} else {
used += len(data)
}
s.files[name] = file{data: data, hash: hash, mode: opened.Mode().Perm(), binary: binary, hashOnly: hashOnly}
continue
}
// The file grew past the per-file limit while being read: hash it.
}
hash, binary, ok := hashFile(f)
f.Close()
if !ok {
s.skipped[name] = true
continue
}
s.files[name] = file{hash: hash, mode: opened.Mode().Perm(), binary: binary, hashOnly: !binary}
}
return s
}
func Compare(before, after *Snapshot) *protocol.StageFileChanges {
result := &protocol.StageFileChanges{Status: "complete", Files: []protocol.FileChange{}}
if before == nil || after == nil || !before.available || !after.available {
result.Status = "unavailable"
result.Diagnostic = "Workspace snapshot unavailable; changes could not be compared."
return result
}
result.Scope = before.scope
if before.partial || after.partial {
result.Status = "partial"
result.Diagnostic = "Workspace capture incomplete; some changes may be missing."
}
skipped := map[string]bool{}
for name := range before.skipped {
skipped[name] = true
}
for name := range after.skipped {
skipped[name] = true
}
result.Skipped = len(skipped)
if result.Skipped > 0 {
result.Status = "partial"
result.Diagnostic = "Excluded protected, symlinked, oversized or unreadable files; this is not a complete workspace diff."
}
keys := map[string]bool{}
for name := range before.files {
keys[name] = true
}
for name := range after.files {
keys[name] = true
}
names := make([]string, 0, len(keys))
for name := range keys {
names = append(names, name)
}
sort.Strings(names)
remaining := maxPatchBytes
retained := 0
add := func(change protocol.FileChange) {
if change.Truncated {
result.Status = "partial"
if !change.NoPatch {
result.Diagnostic = "Some patches were omitted/truncated by capture limits."
}
}
encoded, _ := json.Marshal(change)
if retained+len(encoded)+1 > maxPatchBytes {
change.Patch = ""
change.Truncated = true
encoded, _ = json.Marshal(change)
}
if retained+len(encoded)+1 > maxPatchBytes {
result.Omitted++
result.Status = "partial"
result.Diagnostic = "Additional file changes omitted by the retained evidence size limit."
return
}
if change.Truncated {
result.Status = "partial"
}
retained += len(encoded) + 1
result.Files = append(result.Files, change)
}
listed := 0
// decided records every path the content comparison settled (changed or
// not), so the Git-status pass below only speaks for the rest.
decided := map[string]bool{}
for _, name := range names {
if skipped[name] {
continue
}
a, aOK := before.files[name]
b, bOK := after.files[name]
// Missing files in an incomplete inventory cannot prove deletion/addition.
if !aOK && before.partial || !bOK && after.partial {
continue
}
decided[name] = true
if aOK && bOK && a.hash == b.hash && a.mode == b.mode {
continue
}
kind := "modified"
if !aOK {
kind = "added"
}
if !bOK {
kind = "deleted"
}
change := protocol.FileChange{Path: name, Kind: kind, Binary: a.binary || b.binary}
if aOK {
change.BeforeMode = fmt.Sprintf("%04o", a.mode)
}
if bOK {
change.AfterMode = fmt.Sprintf("%04o", b.mode)
}
if !change.Binary && a.hash != b.hash && (a.hashOnly || b.hashOnly) {
// Over-budget content was compared by hash: list it, without a patch.
change.Truncated = true
change.NoPatch = true
listed++
} else if !change.Binary && a.hash != b.hash {
if remaining == 0 || bytes.Count(a.data, []byte("\n"))+bytes.Count(b.data, []byte("\n")) > maxLines {
change.Truncated = true
} else {
from, to := "a/"+name, "b/"+name
if !aOK {
from = "/dev/null"
}
if !bOK {
to = "/dev/null"
}
// Quote paths so names cannot forge headers or terminal control sequences.
patch, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: diffLines(a.data), B: diffLines(b.data), FromFile: fmt.Sprintf("%q", from), ToFile: fmt.Sprintf("%q", to), Context: 3})
if err != nil {
change.Truncated = true
} else {
limit := min(remaining, maxFilePatchBytes)
if len(patch) > limit {
patch = patch[:limit]
for !utf8.ValidString(patch) && len(patch) > 0 {
patch = patch[:len(patch)-1]
}
change.Truncated = true
}
change.Patch = patch
remaining -= len(patch)
}
}
}
add(change)
}
// In a Git working tree, paths the content snapshot could not settle
// (skipped as unreadable/oversized, or beyond the time/file limit) are still
// listed when their Git status changed during the visit. A path dirty with
// the same status before and after cannot be proven changed and stays
// counted as skipped. Protected paths are never listed.
if before.gitOK && after.gitOK {
gitNames := map[string]bool{}
for name := range before.gitStatus {
gitNames[name] = true
}
for name := range after.gitStatus {
gitNames[name] = true
}
ordered := make([]string, 0, len(gitNames))
for name := range gitNames {
if !decided[name] && fs.ValidPath(name) && !protected(name) {
ordered = append(ordered, name)
}
}
sort.Strings(ordered)
for _, name := range ordered {
was, now := before.gitStatus[name], after.gitStatus[name]
if was == now {
continue
}
kind := "modified"
switch {
case now == "" && was == "??", strings.Contains(now, "D"):
kind = "deleted"
case was == "" && (now == "??" || strings.HasPrefix(now, "A")):
kind = "added"
}
add(protocol.FileChange{Path: name, Kind: kind, Truncated: true, NoPatch: true})
listed++
}
}
if listed > 0 && result.Status == "partial" && result.Omitted == 0 {
result.Diagnostic = "Some changed files exceeded capture limits and are listed without patches; this is not a complete workspace diff."
}
return result
}
func diffLines(data []byte) []string {
if len(data) == 0 {
return nil
}
lines := strings.SplitAfter(string(data), "\n")
if lines[len(lines)-1] == "" {
return lines[:len(lines)-1]
}
lines[len(lines)-1] += "\n\\ No newline at end of file\n"
return lines
}