diff --git a/docs/stage-file-diffs.md b/docs/stage-file-diffs.md index 9f068ac0..96909a7b 100644 --- a/docs/stage-file-diffs.md +++ b/docs/stage-file-diffs.md @@ -18,8 +18,16 @@ includes the same evidence under each step's `file_changes` field. content is the baseline. Changes are retained separately for repeated visits. - In a Git working tree, capture includes tracked and nonignored untracked files within the run directory. Previously captured files remain eligible after an - index/ignore change. Git is used only for inventory, never to stage or revert. + index/ignore change. Git is only read, never used to stage or revert. Outside Git, capture walks regular files in the run directory. +- In Git, the stage-start snapshot keeps text only for files that are dirty or + untracked at start (`git status`); a clean tracked file keeps just its hash and + index blob id. If it changes, its start content is read back from that blob + (`git cat-file --batch`) and used only when its SHA-256 matches what was read + from disk at start. A commit, reset or index change by the stage therefore does + not move the baseline, and the text budget is no longer spent alphabetically on + files that never change. A blob that cannot be verified (filters, line-ending + conversion, `assume-unchanged`) is listed with `no_patch_reason: "baseline"`. - Added/deleted files and permission changes are recorded. Renames appear as deletion/addition. Binary changes retain metadata, not binary contents. - Symlinks, special files, common credential/environment/key/database paths, @@ -39,6 +47,11 @@ includes the same evidence under each step's `file_changes` field. `git status --porcelain` entry changed during the visit; a path dirty with the same status before and after cannot be proven changed and stays in `skipped`. Git is only read (`--no-optional-locks`); protected paths are never listed. +- Each content change without a (complete) patch carries `no_patch_reason`: + `binary`, `file_size` (text over 1 MiB), `budget` (32 MiB snapshot text + budget), `lines` (over 5,000 combined lines), `evidence_size` (per-visit + retained evidence limit), `git_status` (seen only in Git status) or `baseline`. + Binary changes stay `complete`; the others make the visit `partial`. - `pipelines inspect RUN_ID --json` carries a top-level `warnings` list (also printed to stderr) naming every visit whose capture was partial or unavailable. - Failed/canceled stages still retain changes when execution returns. Unconfirmed diff --git a/internal/author/skill/references/operations.md b/internal/author/skill/references/operations.md index a1f1d434..01d0614a 100644 --- a/internal/author/skill/references/operations.md +++ b/internal/author/skill/references/operations.md @@ -197,6 +197,10 @@ The stage's File changes section highlights added/removed lines and hunk headers 32 MiB budget is still compared by hash, and in Git a path the snapshot could not compare is listed when its `git status` entry changes; such entries carry `no_patch: true` (and `truncated`) with no patch, and the visit is `partial`. + `no_patch_reason` says why (`binary`, `file_size`, `budget`, `lines`, + `evidence_size`, `git_status`, `baseline`). In Git only dirty/untracked files + spend the text budget; clean files' start content is read back from their + verified stage-start index blob, so stage commits do not move the baseline. Missing/skipped/truncated captures must be described as partial/unavailable; `inspect` adds a run-level `warnings` entry naming every partial/unavailable visit. - Failed/canceled stages retain changes when execution returns; unconfirmed agent diff --git a/internal/filediff/git_baseline_test.go b/internal/filediff/git_baseline_test.go new file mode 100644 index 00000000..bb6a8f41 --- /dev/null +++ b/internal/filediff/git_baseline_test.go @@ -0,0 +1,246 @@ +package filediff + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func fileByPath(t *testing.T, changes *protocol.StageFileChanges, name string) protocol.FileChange { + t.Helper() + for _, f := range changes.Files { + if f.Path == name { + return f + } + } + t.Fatalf("%s not listed: %+v", name, changes) + return protocol.FileChange{} +} + +// TestGitCleanFilesDoNotSpendBudget is the large-repository regression: clean +// tracked files that exhaust the text budget alphabetically must not cost a +// small, late edited file its patch. Their start content comes from Git. +func TestGitCleanFilesDoNotSpendBudget(t *testing.T) { + restore := SetSnapshotBudgetForTesting(64) + defer restore() + dir, git := gitRepo(t) + if err := os.MkdirAll(filepath.Join(dir, "tests/server"), 0o700); err != nil { + t.Fatal(err) + } + for i := 0; i < 20; i++ { + put(t, dir, fmt.Sprintf("a-%02d.txt", i), strings.Repeat("filler\n", 4)) + } + put(t, dir, "tests/server/test_file_intake_start.py", "def test_one():\n pass\n") + git("add", ".") + git("commit", "-qm", "init") + ctx := context.Background() + before := Capture(ctx, dir) + put(t, dir, "tests/server/test_file_intake_start.py", "def test_one():\n assert True\n") + changes := Compare(before, Capture(ctx, dir, before)) + if changes.Status != "complete" || len(changes.Files) != 1 { + t.Fatalf("capture = %+v, want complete with one file", changes) + } + f := changes.Files[0] + if f.NoPatch || f.Truncated || f.NoPatchReason != "" || !strings.Contains(f.Patch, "- pass") || !strings.Contains(f.Patch, "+ assert True") { + t.Fatalf("late edit lost its patch: %+v", f) + } + for name, entry := range before.files { + if entry.data != nil { + t.Fatalf("clean tracked %s retained content", name) + } + } +} + +// TestGitStageCommitsMidVisit proves the baseline is the content recorded at +// visit start, not the HEAD the stage moved by committing. +func TestGitStageCommitsMidVisit(t *testing.T) { + dir, git := gitRepo(t) + put(t, dir, "contracts.py", "one\ntwo\n") + put(t, dir, "keep.txt", "keep\n") + git("add", ".") + git("commit", "-qm", "init") + ctx := context.Background() + before := Capture(ctx, dir) + put(t, dir, "contracts.py", "one\ntwo\nthree\n") + put(t, dir, "added.py", "new\n") + git("add", ".") + git("commit", "-qm", "stage commit") + put(t, dir, "contracts.py", "one\nTWO\nthree\n") + changes := Compare(before, Capture(ctx, dir, before)) + if changes.Status != "complete" || len(changes.Files) != 2 { + t.Fatalf("capture = %+v", changes) + } + c := fileByPath(t, changes, "contracts.py") + if !strings.Contains(c.Patch, "-two") || !strings.Contains(c.Patch, "+TWO") || !strings.Contains(c.Patch, "+three") { + t.Fatalf("patch not against stage-start content: %q", c.Patch) + } + if a := fileByPath(t, changes, "added.py"); a.Kind != "added" || !strings.Contains(a.Patch, "+new") { + t.Fatalf("committed new file: %+v", a) + } +} + +// TestGitDirtyAtStartEditedFurther proves pre-existing dirty content, not +// HEAD, is the baseline, and that only such files retain start content. +func TestGitDirtyAtStartEditedFurther(t *testing.T) { + dir, git := gitRepo(t) + put(t, dir, "dirty.txt", "committed\n") + put(t, dir, "clean.txt", "clean\n") + git("add", ".") + git("commit", "-qm", "init") + put(t, dir, "dirty.txt", "dirty before stage\n") + put(t, dir, "untracked.txt", "untracked before\n") + ctx := context.Background() + before := Capture(ctx, dir) + if before.files["dirty.txt"].data == nil || before.files["untracked.txt"].data == nil || before.files["clean.txt"].data != nil { + t.Fatalf("retained content: dirty=%q untracked=%q clean=%q", before.files["dirty.txt"].data, before.files["untracked.txt"].data, before.files["clean.txt"].data) + } + put(t, dir, "dirty.txt", "stage edit\n") + put(t, dir, "untracked.txt", "untracked after\n") + changes := Compare(before, Capture(ctx, dir, before)) + if changes.Status != "complete" || len(changes.Files) != 2 { + t.Fatalf("capture = %+v", changes) + } + d := fileByPath(t, changes, "dirty.txt") + if !strings.Contains(d.Patch, "-dirty before stage") || strings.Contains(d.Patch, "committed") { + t.Fatalf("dirty baseline wrong: %q", d.Patch) + } + if u := fileByPath(t, changes, "untracked.txt"); !strings.Contains(u.Patch, "-untracked before") { + t.Fatalf("untracked baseline wrong: %q", u.Patch) + } +} + +// TestGitAddDeleteRename covers added, deleted (worktree and git rm) and +// renamed files whose start content is recovered from Git. +func TestGitAddDeleteRename(t *testing.T) { + dir, git := gitRepo(t) + put(t, dir, "gone.txt", "gone line\n") + put(t, dir, "removed.txt", "removed line\n") + put(t, dir, "old.txt", "moved line\n") + git("add", ".") + git("commit", "-qm", "init") + ctx := context.Background() + before := Capture(ctx, dir) + if err := os.Remove(filepath.Join(dir, "gone.txt")); err != nil { + t.Fatal(err) + } + git("rm", "-q", "removed.txt") + git("mv", "old.txt", "new.txt") + put(t, dir, "fresh.txt", "fresh line\n") + changes := Compare(before, Capture(ctx, dir, before)) + if changes.Status != "complete" || len(changes.Files) != 5 { + t.Fatalf("capture = %+v", changes) + } + for name, want := range map[string]string{"gone.txt": "-gone line", "removed.txt": "-removed line", "old.txt": "-moved line"} { + if f := fileByPath(t, changes, name); f.Kind != "deleted" || !strings.Contains(f.Patch, want) || !strings.Contains(f.Patch, "/dev/null") { + t.Fatalf("deleted %s: %+v", name, f) + } + } + for name, want := range map[string]string{"new.txt": "+moved line", "fresh.txt": "+fresh line"} { + if f := fileByPath(t, changes, name); f.Kind != "added" || !strings.Contains(f.Patch, want) { + t.Fatalf("added %s: %+v", name, f) + } + } +} + +// TestGitOversizedFileHasReason proves only a file over the per-file cap loses +// its patch, with a reason, while a small edit beside it keeps its patch. +func TestGitOversizedFileHasReason(t *testing.T) { + dir, git := gitRepo(t) + put(t, dir, "big.txt", strings.Repeat("x\n", maxFileBytes/2+1)) + put(t, dir, "small.txt", "a\n") + put(t, dir, "image.bin", "\x00one") + git("add", ".") + git("commit", "-qm", "init") + ctx := context.Background() + before := Capture(ctx, dir) + put(t, dir, "big.txt", strings.Repeat("y\n", maxFileBytes/2+1)) + put(t, dir, "small.txt", "b\n") + put(t, dir, "image.bin", "\x00two") + changes := Compare(before, Capture(ctx, dir, before)) + if changes.Status != "partial" || len(changes.Files) != 3 { + t.Fatalf("capture = %+v", changes) + } + if b := fileByPath(t, changes, "big.txt"); !b.NoPatch || b.Patch != "" || b.NoPatchReason != reasonFileSize { + t.Fatalf("oversized: %+v", b) + } + if s := fileByPath(t, changes, "small.txt"); !strings.Contains(s.Patch, "+b") || s.NoPatchReason != "" { + t.Fatalf("small: %+v", s) + } + if i := fileByPath(t, changes, "image.bin"); !i.Binary || i.NoPatch || i.NoPatchReason != reasonBinary { + t.Fatalf("binary: %+v", i) + } +} + +// TestGitUnverifiableBaselineHasNoPatch proves a file Git reports clean whose +// index content differs from disk (assume-unchanged, filters) never yields a +// patch against the wrong baseline. +func TestGitUnverifiableBaselineHasNoPatch(t *testing.T) { + dir, git := gitRepo(t) + put(t, dir, "hidden.txt", "index\n") + git("add", ".") + git("commit", "-qm", "init") + git("update-index", "--assume-unchanged", "hidden.txt") + put(t, dir, "hidden.txt", "disk before\n") + ctx := context.Background() + before := Capture(ctx, dir) + put(t, dir, "hidden.txt", "disk after\n") + changes := Compare(before, Capture(ctx, dir, before)) + if len(changes.Files) != 1 { + t.Fatalf("capture = %+v", changes) + } + if f := changes.Files[0]; !f.NoPatch || f.Patch != "" || f.NoPatchReason != reasonBaseline || changes.Status != "partial" { + t.Fatalf("unverifiable baseline: %+v (%s)", f, changes.Status) + } +} + +// TestGitLargeRepositoryCaptureTime measures a ~6,500-file, ~43 MB committed +// repository (the shape that exhausted the budget) and checks the late edit +// keeps its patch. Run with -v to see the timing; -short skips it. +func TestGitLargeRepositoryCaptureTime(t *testing.T) { + if testing.Short() { + t.Skip("large fixture") + } + dir, git := gitRepo(t) + line := strings.Repeat("abcdefghij", 7) + "\n" // 71 bytes + body := strings.Repeat(line, 93) // ~6.6 KB per file + for d := 0; d < 65; d++ { + sub := filepath.Join(dir, "src", fmt.Sprintf("pkg%02d", d)) + if err := os.MkdirAll(sub, 0o700); err != nil { + t.Fatal(err) + } + for i := 0; i < 100; i++ { + put(t, sub, fmt.Sprintf("m%03d.py", i), body) + } + } + if err := os.MkdirAll(filepath.Join(dir, "tests/server"), 0o700); err != nil { + t.Fatal(err) + } + put(t, dir, "tests/server/test_file_intake_start.py", "def test_one():\n pass\n") + git("add", ".") + git("commit", "-qm", "init") + ctx := context.Background() + start := time.Now() + before := Capture(ctx, dir) + beforeTime := time.Since(start) + put(t, dir, "tests/server/test_file_intake_start.py", "def test_one():\n assert True\n") + put(t, filepath.Join(dir, "src/pkg64"), "m099.py", body+"tail\n") + start = time.Now() + after := Capture(ctx, dir, before) + afterTime := time.Since(start) + changes := Compare(before, after) + t.Logf("6,501 files / ~43 MB: before %v, after %v", beforeTime, afterTime) + if changes.Status != "complete" || len(changes.Files) != 2 { + t.Fatalf("capture = %+v", changes) + } + for _, f := range changes.Files { + if f.Patch == "" || f.NoPatch { + t.Fatalf("no patch for %s: %+v", f.Path, f) + } + } +} diff --git a/internal/filediff/snapshot.go b/internal/filediff/snapshot.go index a496bcc8..836be5fc 100644 --- a/internal/filediff/snapshot.go +++ b/internal/filediff/snapshot.go @@ -2,6 +2,7 @@ package filediff import ( + "bufio" "bytes" "context" "crypto/sha256" @@ -13,6 +14,7 @@ import ( "os" "os/exec" "sort" + "strconv" "strings" "sync/atomic" "syscall" @@ -52,14 +54,40 @@ func SetSnapshotBudgetForTesting(n int64) (restore func()) { return func() { snapshotBudget.Store(previous) } } +// No-patch reasons recorded on protocol.FileChange.NoPatchReason. +const ( + reasonBinary = "binary" // binary content; metadata only + reasonFileSize = "file_size" // text over the 1 MiB per-file limit + reasonBudget = "budget" // the per-snapshot retained-text budget ran out + reasonLines = "lines" // over the combined line limit for one comparison + reasonEvidence = "evidence_size" // the per-visit retained patch/evidence limit ran out + reasonGit = "git_status" // seen only as a Git status change; not compared + reasonBaseline = "baseline" // the stage-start content could not be recovered +) + type file struct { data []byte hash [32]byte + size int64 mode fs.FileMode binary bool // hashOnly marks a text file compared by content hash only: it exceeded the - // per-file or per-snapshot text budget, so no patch can be produced. + // per-file or per-snapshot text budget, so no patch can be produced. limit + // names which (reasonFileSize or reasonBudget). hashOnly bool + limit string + // blob is the Git index object id of a text file that was clean (matching + // the index and HEAD) when captured. Its content is not retained: Git + // already holds it, and a later capture recovers it only if the file + // changed. This keeps the text budget for files Git has no baseline for. + blob string +} + +// baseline is stage-start content recovered from Git for a changed file whose +// start snapshot kept only its blob id; hash is verified against that snapshot. +type baseline struct { + data []byte + hash [32]byte } type Snapshot struct { @@ -72,9 +100,81 @@ type Snapshot struct { // gitStatus maps run-relative paths to their porcelain XY status code when // the run directory is a Git working tree and the status read succeeded // (gitOK). Compare uses it to list changed paths the content snapshot could - // not compare (time/file limit, unreadable); it never reads HEAD/index content. + // not compare (time/file limit, unreadable). gitStatus map[string]string gitOK bool + // baselines holds Git-recovered start content for files that changed since + // the previous snapshot; missing records why one could not be recovered. + baselines map[string]baseline + missing map[string]string +} + +// readGitIndex returns the stage-0 index object id of every tracked path in +// the run directory (paths relative to it, as ls-files prints them). +func readGitIndex(ctx context.Context, directory string) (map[string]string, bool) { + cmd := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "ls-files", "--stage", "-z", "--", ".") + out := &boundedOutput{limit: 8 << 20} + cmd.Stdout = out + if err := cmd.Run(); err != nil { + return nil, false + } + blobs := map[string]string{} + for _, entry := range strings.Split(out.String(), "\x00") { + meta, name, ok := strings.Cut(entry, "\t") + fields := strings.Fields(meta) + if !ok || len(fields) != 3 || fields[2] != "0" || name == "" { + continue + } + blobs[name] = fields[1] + } + return blobs, true +} + +// readBlobs reads Git objects by id with one `git cat-file --batch`. Objects +// that are missing, not blobs or larger than maxFileBytes are left out. +func readBlobs(ctx context.Context, directory string, ids []string) map[string][]byte { + result := map[string][]byte{} + if len(ids) == 0 { + return result + } + cmd := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "cat-file", "--batch") + cmd.Stdin = strings.NewReader(strings.Join(ids, "\n") + "\n") + pipe, err := cmd.StdoutPipe() + if err != nil { + return result + } + if err := cmd.Start(); err != nil { + return result + } + reader := bufio.NewReader(pipe) + for range ids { + header, err := reader.ReadString('\n') + if err != nil { + break + } + fields := strings.Fields(header) + if len(fields) != 3 { + continue // " missing" carries no content + } + size, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil || size < 0 { + break + } + if fields[1] != "blob" || size > maxFileBytes { + if _, err := io.CopyN(io.Discard, reader, size+1); err != nil { + break + } + continue + } + data := make([]byte, size+1) // content plus the trailing newline + if _, err := io.ReadFull(reader, data); err != nil { + break + } + result[fields[0]] = data[:size] + } + _, _ = io.Copy(io.Discard, pipe) + _ = cmd.Wait() + return result } // readGitStatus reads `git status --porcelain=v1 -z` for the run directory and @@ -152,8 +252,8 @@ func (b *boundedOutput) Write(p []byte) (int, error) { return b.Buffer.Write(p) } -// Git only supplies an inventory; never compare against HEAD/index or alter them. -// Tracked dirty files and untracked, nonignored files are compared as they exist. +// inventory lists the files to snapshot. Git is only read, never altered; +// the baseline is still stage-start content (see Capture and recoverBaselines). func inventory(ctx context.Context, directory string, root *os.Root) ([]string, bool, error) { probe := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "rev-parse", "--is-inside-work-tree") if data, err := probe.Output(); err == nil && strings.TrimSpace(string(data)) == "true" { @@ -212,8 +312,22 @@ func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snap s.partial = true s.diagnostic = "Workspace inventory incomplete." } + var prev *Snapshot + for _, prior := range previous { + if prior != nil { + prev = prior + break + } + } + // A stage-start capture keeps only the blob id of a text file that is clean + // in Git (no status entry, stage-0 index entry): Git holds its content. Only + // dirty and untracked files, which have no Git baseline, spend the budget. + var blobs map[string]string if git { s.gitStatus, s.gitOK = readGitStatus(ctx, directory) + if s.gitOK && prev == nil { + blobs, _ = readGitIndex(ctx, directory) + } } // Keep pre-stage paths even if a stage changes .gitignore/the index. known := map[string]bool{} @@ -289,22 +403,32 @@ func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snap if len(data) <= maxFileBytes { f.Close() binary := bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data) - hash := sha256.Sum256(data) - hashOnly := false + entry := file{hash: sha256.Sum256(data), size: int64(len(data)), mode: opened.Mode().Perm(), binary: binary} + _, dirty := s.gitStatus[name] + p, seen := file{}, false + if prev != nil { + p, seen = prev.files[name] + } // Binary changes need only metadata and a hash, never retained patch bytes. // Counting screenshots against the retained-text budget can otherwise // exhaust it before any source files are reached in sorted inventory order. // Text beyond the budget keeps only its hash, so a change to it is still // listed (without a patch) instead of being skipped unseen. - if binary { - data = nil - } else if int64(used+len(data)) > snapshotBudget.Load() { - data = nil - hashOnly = true - } else { + switch { + case binary: + case seen && p.hash == entry.hash && !p.binary: + // Unchanged since the previous snapshot: share its representation + // without spending this snapshot's budget on it again. + entry.data, entry.blob, entry.hashOnly, entry.limit = p.data, p.blob, p.hashOnly, p.limit + case blobs[name] != "" && !dirty: + entry.blob = blobs[name] + case int64(used+len(data)) > snapshotBudget.Load(): + entry.hashOnly, entry.limit = true, reasonBudget + default: + entry.data = data used += len(data) } - s.files[name] = file{data: data, hash: hash, mode: opened.Mode().Perm(), binary: binary, hashOnly: hashOnly} + s.files[name] = entry continue } // The file grew past the per-file limit while being read: hash it. @@ -315,11 +439,60 @@ func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snap s.skipped[name] = true continue } - s.files[name] = file{hash: hash, mode: opened.Mode().Perm(), binary: binary, hashOnly: !binary} + s.files[name] = file{hash: hash, size: opened.Size(), mode: opened.Mode().Perm(), binary: binary, hashOnly: !binary, limit: reasonFileSize} + } + if prev != nil { + s.recoverBaselines(ctx, directory, prev, used) } return s } +// recoverBaselines reads, in one Git batch, the stage-start content of every +// file prev kept only as a clean blob id that has since changed or gone. The +// content is used only if its hash matches what prev read from disk, so +// clean/smudge filters, line-ending conversion or an index changed under the +// capture can never yield a wrong patch; such files are listed without one. +func (s *Snapshot) recoverBaselines(ctx context.Context, directory string, prev *Snapshot, used int) { + var names []string + for name, p := range prev.files { + if p.blob == "" || p.data != nil || p.binary || s.skipped[name] { + continue + } + now, ok := s.files[name] + if ok && now.hash == p.hash || !ok && s.partial { + continue // unchanged, or an incomplete inventory cannot prove deletion + } + names = append(names, name) + } + if len(names) == 0 { + return + } + sort.Strings(names) + s.baselines, s.missing = map[string]baseline{}, map[string]string{} + ids := make([]string, 0, len(names)) + wanted := make([]string, 0, len(names)) + for _, name := range names { + p := prev.files[name] + if int64(used)+p.size > snapshotBudget.Load() { + s.missing[name] = reasonBudget + continue + } + used += int(p.size) + ids = append(ids, p.blob) + wanted = append(wanted, name) + } + contents := readBlobs(ctx, directory, ids) + for _, name := range wanted { + p := prev.files[name] + data, ok := contents[p.blob] + if !ok || sha256.Sum256(data) != p.hash { + s.missing[name] = reasonBaseline + continue + } + s.baselines[name] = baseline{data: data, hash: p.hash} + } +} + func Compare(before, after *Snapshot) *protocol.StageFileChanges { result := &protocol.StageFileChanges{Status: "complete", Files: []protocol.FileChange{}} if before == nil || after == nil || !before.available || !after.available { @@ -367,7 +540,10 @@ func Compare(before, after *Snapshot) *protocol.StageFileChanges { } encoded, _ := json.Marshal(change) if retained+len(encoded)+1 > maxPatchBytes { - change.Patch = "" + if change.Patch != "" { + change.Patch = "" + change.NoPatchReason = reasonEvidence + } change.Truncated = true encoded, _ = json.Marshal(change) } @@ -415,14 +591,38 @@ func Compare(before, after *Snapshot) *protocol.StageFileChanges { if bOK { change.AfterMode = fmt.Sprintf("%04o", b.mode) } - if !change.Binary && a.hash != b.hash && (a.hashOnly || b.hashOnly) { - // Over-budget content was compared by hash: list it, without a patch. + // A clean-at-start file kept only its Git blob id; its start content + // comes from the after snapshot's verified Git recovery. + aData, unavailable := a.data, "" + if aOK && !a.binary && a.blob != "" && a.data == nil { + if base, ok := after.baselines[name]; ok && base.hash == a.hash { + aData = base.data + } else if unavailable = after.missing[name]; unavailable == "" { + unavailable = reasonBaseline + } + } + if change.Binary { + if a.hash != b.hash { + change.NoPatchReason = reasonBinary + } + } else if a.hash != b.hash && (a.hashOnly || b.hashOnly || unavailable != "") { + // Content compared by hash only: list it, without a patch. change.Truncated = true change.NoPatch = true + switch { + case a.limit == reasonFileSize || b.limit == reasonFileSize: + change.NoPatchReason = reasonFileSize + case unavailable != "": + change.NoPatchReason = unavailable + default: + change.NoPatchReason = reasonBudget + } listed++ - } else if !change.Binary && a.hash != b.hash { - if remaining == 0 || bytes.Count(a.data, []byte("\n"))+bytes.Count(b.data, []byte("\n")) > maxLines { - change.Truncated = true + } else if a.hash != b.hash { + if remaining == 0 { + change.Truncated, change.NoPatchReason = true, reasonEvidence + } else if bytes.Count(aData, []byte("\n"))+bytes.Count(b.data, []byte("\n")) > maxLines { + change.Truncated, change.NoPatchReason = true, reasonLines } else { from, to := "a/"+name, "b/"+name if !aOK { @@ -432,7 +632,7 @@ func Compare(before, after *Snapshot) *protocol.StageFileChanges { to = "/dev/null" } // Quote paths so names cannot forge headers or terminal control sequences. - patch, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: diffLines(a.data), B: diffLines(b.data), FromFile: fmt.Sprintf("%q", from), ToFile: fmt.Sprintf("%q", to), Context: 3}) + patch, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: diffLines(aData), B: diffLines(b.data), FromFile: fmt.Sprintf("%q", from), ToFile: fmt.Sprintf("%q", to), Context: 3}) if err != nil { change.Truncated = true } else { @@ -483,7 +683,7 @@ func Compare(before, after *Snapshot) *protocol.StageFileChanges { case was == "" && (now == "??" || strings.HasPrefix(now, "A")): kind = "added" } - add(protocol.FileChange{Path: name, Kind: kind, Truncated: true, NoPatch: true}) + add(protocol.FileChange{Path: name, Kind: kind, Truncated: true, NoPatch: true, NoPatchReason: reasonGit}) listed++ } } diff --git a/internal/filediff/snapshot_test.go b/internal/filediff/snapshot_test.go index cb1b3e83..e434b9c1 100644 --- a/internal/filediff/snapshot_test.go +++ b/internal/filediff/snapshot_test.go @@ -212,14 +212,14 @@ func gitRepo(t *testing.T) (string, func(...string)) { return dir, run } -// TestOverBudgetChangeIsListedWithoutPatch is the run-236 regression: a -// workspace whose text exceeds the snapshot budget must still list the one -// changed file (sorted last, after the budget is exhausted) instead of +// TestOverBudgetChangeIsListedWithoutPatch is the run-236 regression outside +// Git: a workspace whose text exceeds the snapshot budget must still list the +// one changed file (sorted last, after the budget is exhausted) instead of // returning files: [] with every late path skipped. func TestOverBudgetChangeIsListedWithoutPatch(t *testing.T) { restore := SetSnapshotBudgetForTesting(64) defer restore() - dir, git := gitRepo(t) + dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, "web/src"), 0o700); err != nil { t.Fatal(err) } @@ -227,8 +227,6 @@ func TestOverBudgetChangeIsListedWithoutPatch(t *testing.T) { put(t, dir, fmt.Sprintf("a-%02d.txt", i), strings.Repeat("filler\n", 4)) } put(t, dir, "web/src/Inbox.test.tsx", "it('works')\n") - git("add", ".") - git("commit", "-qm", "init") ctx := context.Background() before := Capture(ctx, dir) put(t, dir, "web/src/Inbox.test.tsx", "it('works')\nit('more')\n") @@ -236,7 +234,7 @@ func TestOverBudgetChangeIsListedWithoutPatch(t *testing.T) { if len(changes.Files) != 1 || changes.Files[0].Path != "web/src/Inbox.test.tsx" { t.Fatalf("over-budget change not listed: %+v", changes) } - if f := changes.Files[0]; f.Kind != "modified" || !f.NoPatch || f.Patch != "" { + if f := changes.Files[0]; f.Kind != "modified" || !f.NoPatch || f.Patch != "" || f.NoPatchReason != reasonBudget { t.Fatalf("over-budget change should be listed without patch: %+v", f) } if changes.Status != "partial" || changes.Skipped != 0 { diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 0869e695..65396543 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -469,9 +469,15 @@ type FileChange struct { // NoPatch marks a change listed without any patch: the content exceeded the // capture budget (compared by hash) or the change was seen only in Git // status. Truncated is also set so older readers show "patch omitted". - NoPatch bool `json:"no_patch,omitempty"` - BeforeMode string `json:"before_mode,omitempty"` - AfterMode string `json:"after_mode,omitempty"` + NoPatch bool `json:"no_patch,omitempty"` + // NoPatchReason says why a content change has no (complete) patch: + // "binary", "file_size" (text over 1 MiB), "budget" (snapshot text budget), + // "lines" (over 5,000 combined lines), "evidence_size" (per-visit retained + // evidence limit), "git_status" (seen only in Git status) or "baseline" + // (stage-start content unrecoverable). Empty when a patch was recorded. + NoPatchReason string `json:"no_patch_reason,omitempty"` + BeforeMode string `json:"before_mode,omitempty"` + AfterMode string `json:"after_mode,omitempty"` } type NotificationReceipt struct { diff --git a/internal/server/ui_skip_test.go b/internal/server/ui_skip_test.go index 7b333ad7..e48de9b0 100644 --- a/internal/server/ui_skip_test.go +++ b/internal/server/ui_skip_test.go @@ -210,8 +210,8 @@ func TestUISkipPartialCaptureNeverSkips(t *testing.T) { // TestUISkipOverBudgetGitWorkspaceRunsModel is the run-236 regression: in a Git // workspace whose text exceeds the snapshot budget, a stage edits one .tsx file -// sorted after the budget is exhausted. The capture must list that file -// (without a patch) and the next stage, which declares `skipped`, must run its +// sorted after the budget would be exhausted. The capture must list that file +// (now with its patch: clean files spend no budget) and the next stage, which declares `skipped`, must run its // model rather than be auto-routed with "no recorded file changes". func TestUISkipOverBudgetGitWorkspaceRunsModel(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { @@ -244,13 +244,12 @@ func TestUISkipOverBudgetGitWorkspaceRunsModel(t *testing.T) { git("commit", "-qm", "fixture") detail, visited, verdictCalls := runUISkipIn(t, uiReviewerSource(true), map[string]string{"implement": target}, dir) capture := detail.Steps[0].FileChanges - if capture == nil || len(capture.Files) != 1 || capture.Files[0].Path != target || !capture.Files[0].NoPatch { - t.Fatalf("over-budget capture did not list the edited file: %+v", capture) + // Clean tracked files take their baseline from Git, so the budget no longer + // runs out and the edited file keeps a full patch. + if capture == nil || capture.Status != "complete" || len(capture.Files) != 1 || capture.Files[0].Path != target || capture.Files[0].NoPatch || !strings.Contains(capture.Files[0].Patch, "+implement") { + t.Fatalf("over-budget capture did not patch the edited file: %+v", capture) } assertModelRan(t, detail, visited, verdictCalls) - if len(detail.Warnings) == 0 { - t.Fatal("partial capture produced no run-level warning") - } } // TestUISkipOptOut proves auto_skip=false makes a `skipped` stage run its model