Merge: git baselines give full patches for changed files in large repos
This commit is contained in:
@@ -18,8 +18,16 @@ includes the same evidence under each step's `file_changes` field.
|
||||
content is the baseline. Changes are retained separately for repeated visits.
|
||||
- In a Git working tree, capture includes tracked and nonignored untracked files
|
||||
within the run directory. Previously captured files remain eligible after an
|
||||
index/ignore change. Git is used only for inventory, never to stage or revert.
|
||||
index/ignore change. Git is only read, never used to stage or revert.
|
||||
Outside Git, capture walks regular files in the run directory.
|
||||
- In Git, the stage-start snapshot keeps text only for files that are dirty or
|
||||
untracked at start (`git status`); a clean tracked file keeps just its hash and
|
||||
index blob id. If it changes, its start content is read back from that blob
|
||||
(`git cat-file --batch`) and used only when its SHA-256 matches what was read
|
||||
from disk at start. A commit, reset or index change by the stage therefore does
|
||||
not move the baseline, and the text budget is no longer spent alphabetically on
|
||||
files that never change. A blob that cannot be verified (filters, line-ending
|
||||
conversion, `assume-unchanged`) is listed with `no_patch_reason: "baseline"`.
|
||||
- Added/deleted files and permission changes are recorded. Renames appear as
|
||||
deletion/addition. Binary changes retain metadata, not binary contents.
|
||||
- Symlinks, special files, common credential/environment/key/database paths,
|
||||
@@ -39,6 +47,11 @@ includes the same evidence under each step's `file_changes` field.
|
||||
`git status --porcelain` entry changed during the visit; a path dirty with the
|
||||
same status before and after cannot be proven changed and stays in `skipped`.
|
||||
Git is only read (`--no-optional-locks`); protected paths are never listed.
|
||||
- Each content change without a (complete) patch carries `no_patch_reason`:
|
||||
`binary`, `file_size` (text over 1 MiB), `budget` (32 MiB snapshot text
|
||||
budget), `lines` (over 5,000 combined lines), `evidence_size` (per-visit
|
||||
retained evidence limit), `git_status` (seen only in Git status) or `baseline`.
|
||||
Binary changes stay `complete`; the others make the visit `partial`.
|
||||
- `pipelines inspect RUN_ID --json` carries a top-level `warnings` list (also
|
||||
printed to stderr) naming every visit whose capture was partial or unavailable.
|
||||
- Failed/canceled stages still retain changes when execution returns. Unconfirmed
|
||||
|
||||
@@ -197,6 +197,10 @@ The stage's File changes section highlights added/removed lines and hunk headers
|
||||
32 MiB budget is still compared by hash, and in Git a path the snapshot could
|
||||
not compare is listed when its `git status` entry changes; such entries carry
|
||||
`no_patch: true` (and `truncated`) with no patch, and the visit is `partial`.
|
||||
`no_patch_reason` says why (`binary`, `file_size`, `budget`, `lines`,
|
||||
`evidence_size`, `git_status`, `baseline`). In Git only dirty/untracked files
|
||||
spend the text budget; clean files' start content is read back from their
|
||||
verified stage-start index blob, so stage commits do not move the baseline.
|
||||
Missing/skipped/truncated captures must be described as partial/unavailable;
|
||||
`inspect` adds a run-level `warnings` entry naming every partial/unavailable visit.
|
||||
- Failed/canceled stages retain changes when execution returns; unconfirmed agent
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
package filediff
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func fileByPath(t *testing.T, changes *protocol.StageFileChanges, name string) protocol.FileChange {
|
||||
t.Helper()
|
||||
for _, f := range changes.Files {
|
||||
if f.Path == name {
|
||||
return f
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s not listed: %+v", name, changes)
|
||||
return protocol.FileChange{}
|
||||
}
|
||||
|
||||
// TestGitCleanFilesDoNotSpendBudget is the large-repository regression: clean
|
||||
// tracked files that exhaust the text budget alphabetically must not cost a
|
||||
// small, late edited file its patch. Their start content comes from Git.
|
||||
func TestGitCleanFilesDoNotSpendBudget(t *testing.T) {
|
||||
restore := SetSnapshotBudgetForTesting(64)
|
||||
defer restore()
|
||||
dir, git := gitRepo(t)
|
||||
if err := os.MkdirAll(filepath.Join(dir, "tests/server"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 20; i++ {
|
||||
put(t, dir, fmt.Sprintf("a-%02d.txt", i), strings.Repeat("filler\n", 4))
|
||||
}
|
||||
put(t, dir, "tests/server/test_file_intake_start.py", "def test_one():\n pass\n")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "init")
|
||||
ctx := context.Background()
|
||||
before := Capture(ctx, dir)
|
||||
put(t, dir, "tests/server/test_file_intake_start.py", "def test_one():\n assert True\n")
|
||||
changes := Compare(before, Capture(ctx, dir, before))
|
||||
if changes.Status != "complete" || len(changes.Files) != 1 {
|
||||
t.Fatalf("capture = %+v, want complete with one file", changes)
|
||||
}
|
||||
f := changes.Files[0]
|
||||
if f.NoPatch || f.Truncated || f.NoPatchReason != "" || !strings.Contains(f.Patch, "- pass") || !strings.Contains(f.Patch, "+ assert True") {
|
||||
t.Fatalf("late edit lost its patch: %+v", f)
|
||||
}
|
||||
for name, entry := range before.files {
|
||||
if entry.data != nil {
|
||||
t.Fatalf("clean tracked %s retained content", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitStageCommitsMidVisit proves the baseline is the content recorded at
|
||||
// visit start, not the HEAD the stage moved by committing.
|
||||
func TestGitStageCommitsMidVisit(t *testing.T) {
|
||||
dir, git := gitRepo(t)
|
||||
put(t, dir, "contracts.py", "one\ntwo\n")
|
||||
put(t, dir, "keep.txt", "keep\n")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "init")
|
||||
ctx := context.Background()
|
||||
before := Capture(ctx, dir)
|
||||
put(t, dir, "contracts.py", "one\ntwo\nthree\n")
|
||||
put(t, dir, "added.py", "new\n")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "stage commit")
|
||||
put(t, dir, "contracts.py", "one\nTWO\nthree\n")
|
||||
changes := Compare(before, Capture(ctx, dir, before))
|
||||
if changes.Status != "complete" || len(changes.Files) != 2 {
|
||||
t.Fatalf("capture = %+v", changes)
|
||||
}
|
||||
c := fileByPath(t, changes, "contracts.py")
|
||||
if !strings.Contains(c.Patch, "-two") || !strings.Contains(c.Patch, "+TWO") || !strings.Contains(c.Patch, "+three") {
|
||||
t.Fatalf("patch not against stage-start content: %q", c.Patch)
|
||||
}
|
||||
if a := fileByPath(t, changes, "added.py"); a.Kind != "added" || !strings.Contains(a.Patch, "+new") {
|
||||
t.Fatalf("committed new file: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitDirtyAtStartEditedFurther proves pre-existing dirty content, not
|
||||
// HEAD, is the baseline, and that only such files retain start content.
|
||||
func TestGitDirtyAtStartEditedFurther(t *testing.T) {
|
||||
dir, git := gitRepo(t)
|
||||
put(t, dir, "dirty.txt", "committed\n")
|
||||
put(t, dir, "clean.txt", "clean\n")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "init")
|
||||
put(t, dir, "dirty.txt", "dirty before stage\n")
|
||||
put(t, dir, "untracked.txt", "untracked before\n")
|
||||
ctx := context.Background()
|
||||
before := Capture(ctx, dir)
|
||||
if before.files["dirty.txt"].data == nil || before.files["untracked.txt"].data == nil || before.files["clean.txt"].data != nil {
|
||||
t.Fatalf("retained content: dirty=%q untracked=%q clean=%q", before.files["dirty.txt"].data, before.files["untracked.txt"].data, before.files["clean.txt"].data)
|
||||
}
|
||||
put(t, dir, "dirty.txt", "stage edit\n")
|
||||
put(t, dir, "untracked.txt", "untracked after\n")
|
||||
changes := Compare(before, Capture(ctx, dir, before))
|
||||
if changes.Status != "complete" || len(changes.Files) != 2 {
|
||||
t.Fatalf("capture = %+v", changes)
|
||||
}
|
||||
d := fileByPath(t, changes, "dirty.txt")
|
||||
if !strings.Contains(d.Patch, "-dirty before stage") || strings.Contains(d.Patch, "committed") {
|
||||
t.Fatalf("dirty baseline wrong: %q", d.Patch)
|
||||
}
|
||||
if u := fileByPath(t, changes, "untracked.txt"); !strings.Contains(u.Patch, "-untracked before") {
|
||||
t.Fatalf("untracked baseline wrong: %q", u.Patch)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitAddDeleteRename covers added, deleted (worktree and git rm) and
|
||||
// renamed files whose start content is recovered from Git.
|
||||
func TestGitAddDeleteRename(t *testing.T) {
|
||||
dir, git := gitRepo(t)
|
||||
put(t, dir, "gone.txt", "gone line\n")
|
||||
put(t, dir, "removed.txt", "removed line\n")
|
||||
put(t, dir, "old.txt", "moved line\n")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "init")
|
||||
ctx := context.Background()
|
||||
before := Capture(ctx, dir)
|
||||
if err := os.Remove(filepath.Join(dir, "gone.txt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
git("rm", "-q", "removed.txt")
|
||||
git("mv", "old.txt", "new.txt")
|
||||
put(t, dir, "fresh.txt", "fresh line\n")
|
||||
changes := Compare(before, Capture(ctx, dir, before))
|
||||
if changes.Status != "complete" || len(changes.Files) != 5 {
|
||||
t.Fatalf("capture = %+v", changes)
|
||||
}
|
||||
for name, want := range map[string]string{"gone.txt": "-gone line", "removed.txt": "-removed line", "old.txt": "-moved line"} {
|
||||
if f := fileByPath(t, changes, name); f.Kind != "deleted" || !strings.Contains(f.Patch, want) || !strings.Contains(f.Patch, "/dev/null") {
|
||||
t.Fatalf("deleted %s: %+v", name, f)
|
||||
}
|
||||
}
|
||||
for name, want := range map[string]string{"new.txt": "+moved line", "fresh.txt": "+fresh line"} {
|
||||
if f := fileByPath(t, changes, name); f.Kind != "added" || !strings.Contains(f.Patch, want) {
|
||||
t.Fatalf("added %s: %+v", name, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitOversizedFileHasReason proves only a file over the per-file cap loses
|
||||
// its patch, with a reason, while a small edit beside it keeps its patch.
|
||||
func TestGitOversizedFileHasReason(t *testing.T) {
|
||||
dir, git := gitRepo(t)
|
||||
put(t, dir, "big.txt", strings.Repeat("x\n", maxFileBytes/2+1))
|
||||
put(t, dir, "small.txt", "a\n")
|
||||
put(t, dir, "image.bin", "\x00one")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "init")
|
||||
ctx := context.Background()
|
||||
before := Capture(ctx, dir)
|
||||
put(t, dir, "big.txt", strings.Repeat("y\n", maxFileBytes/2+1))
|
||||
put(t, dir, "small.txt", "b\n")
|
||||
put(t, dir, "image.bin", "\x00two")
|
||||
changes := Compare(before, Capture(ctx, dir, before))
|
||||
if changes.Status != "partial" || len(changes.Files) != 3 {
|
||||
t.Fatalf("capture = %+v", changes)
|
||||
}
|
||||
if b := fileByPath(t, changes, "big.txt"); !b.NoPatch || b.Patch != "" || b.NoPatchReason != reasonFileSize {
|
||||
t.Fatalf("oversized: %+v", b)
|
||||
}
|
||||
if s := fileByPath(t, changes, "small.txt"); !strings.Contains(s.Patch, "+b") || s.NoPatchReason != "" {
|
||||
t.Fatalf("small: %+v", s)
|
||||
}
|
||||
if i := fileByPath(t, changes, "image.bin"); !i.Binary || i.NoPatch || i.NoPatchReason != reasonBinary {
|
||||
t.Fatalf("binary: %+v", i)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitUnverifiableBaselineHasNoPatch proves a file Git reports clean whose
|
||||
// index content differs from disk (assume-unchanged, filters) never yields a
|
||||
// patch against the wrong baseline.
|
||||
func TestGitUnverifiableBaselineHasNoPatch(t *testing.T) {
|
||||
dir, git := gitRepo(t)
|
||||
put(t, dir, "hidden.txt", "index\n")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "init")
|
||||
git("update-index", "--assume-unchanged", "hidden.txt")
|
||||
put(t, dir, "hidden.txt", "disk before\n")
|
||||
ctx := context.Background()
|
||||
before := Capture(ctx, dir)
|
||||
put(t, dir, "hidden.txt", "disk after\n")
|
||||
changes := Compare(before, Capture(ctx, dir, before))
|
||||
if len(changes.Files) != 1 {
|
||||
t.Fatalf("capture = %+v", changes)
|
||||
}
|
||||
if f := changes.Files[0]; !f.NoPatch || f.Patch != "" || f.NoPatchReason != reasonBaseline || changes.Status != "partial" {
|
||||
t.Fatalf("unverifiable baseline: %+v (%s)", f, changes.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitLargeRepositoryCaptureTime measures a ~6,500-file, ~43 MB committed
|
||||
// repository (the shape that exhausted the budget) and checks the late edit
|
||||
// keeps its patch. Run with -v to see the timing; -short skips it.
|
||||
func TestGitLargeRepositoryCaptureTime(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("large fixture")
|
||||
}
|
||||
dir, git := gitRepo(t)
|
||||
line := strings.Repeat("abcdefghij", 7) + "\n" // 71 bytes
|
||||
body := strings.Repeat(line, 93) // ~6.6 KB per file
|
||||
for d := 0; d < 65; d++ {
|
||||
sub := filepath.Join(dir, "src", fmt.Sprintf("pkg%02d", d))
|
||||
if err := os.MkdirAll(sub, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
put(t, sub, fmt.Sprintf("m%03d.py", i), body)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "tests/server"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(t, dir, "tests/server/test_file_intake_start.py", "def test_one():\n pass\n")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "init")
|
||||
ctx := context.Background()
|
||||
start := time.Now()
|
||||
before := Capture(ctx, dir)
|
||||
beforeTime := time.Since(start)
|
||||
put(t, dir, "tests/server/test_file_intake_start.py", "def test_one():\n assert True\n")
|
||||
put(t, filepath.Join(dir, "src/pkg64"), "m099.py", body+"tail\n")
|
||||
start = time.Now()
|
||||
after := Capture(ctx, dir, before)
|
||||
afterTime := time.Since(start)
|
||||
changes := Compare(before, after)
|
||||
t.Logf("6,501 files / ~43 MB: before %v, after %v", beforeTime, afterTime)
|
||||
if changes.Status != "complete" || len(changes.Files) != 2 {
|
||||
t.Fatalf("capture = %+v", changes)
|
||||
}
|
||||
for _, f := range changes.Files {
|
||||
if f.Patch == "" || f.NoPatch {
|
||||
t.Fatalf("no patch for %s: %+v", f.Path, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
+222
-22
@@ -2,6 +2,7 @@
|
||||
package filediff
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
@@ -52,14 +54,40 @@ func SetSnapshotBudgetForTesting(n int64) (restore func()) {
|
||||
return func() { snapshotBudget.Store(previous) }
|
||||
}
|
||||
|
||||
// No-patch reasons recorded on protocol.FileChange.NoPatchReason.
|
||||
const (
|
||||
reasonBinary = "binary" // binary content; metadata only
|
||||
reasonFileSize = "file_size" // text over the 1 MiB per-file limit
|
||||
reasonBudget = "budget" // the per-snapshot retained-text budget ran out
|
||||
reasonLines = "lines" // over the combined line limit for one comparison
|
||||
reasonEvidence = "evidence_size" // the per-visit retained patch/evidence limit ran out
|
||||
reasonGit = "git_status" // seen only as a Git status change; not compared
|
||||
reasonBaseline = "baseline" // the stage-start content could not be recovered
|
||||
)
|
||||
|
||||
type file struct {
|
||||
data []byte
|
||||
hash [32]byte
|
||||
size int64
|
||||
mode fs.FileMode
|
||||
binary bool
|
||||
// hashOnly marks a text file compared by content hash only: it exceeded the
|
||||
// per-file or per-snapshot text budget, so no patch can be produced.
|
||||
// per-file or per-snapshot text budget, so no patch can be produced. limit
|
||||
// names which (reasonFileSize or reasonBudget).
|
||||
hashOnly bool
|
||||
limit string
|
||||
// blob is the Git index object id of a text file that was clean (matching
|
||||
// the index and HEAD) when captured. Its content is not retained: Git
|
||||
// already holds it, and a later capture recovers it only if the file
|
||||
// changed. This keeps the text budget for files Git has no baseline for.
|
||||
blob string
|
||||
}
|
||||
|
||||
// baseline is stage-start content recovered from Git for a changed file whose
|
||||
// start snapshot kept only its blob id; hash is verified against that snapshot.
|
||||
type baseline struct {
|
||||
data []byte
|
||||
hash [32]byte
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
@@ -72,9 +100,81 @@ type Snapshot struct {
|
||||
// gitStatus maps run-relative paths to their porcelain XY status code when
|
||||
// the run directory is a Git working tree and the status read succeeded
|
||||
// (gitOK). Compare uses it to list changed paths the content snapshot could
|
||||
// not compare (time/file limit, unreadable); it never reads HEAD/index content.
|
||||
// not compare (time/file limit, unreadable).
|
||||
gitStatus map[string]string
|
||||
gitOK bool
|
||||
// baselines holds Git-recovered start content for files that changed since
|
||||
// the previous snapshot; missing records why one could not be recovered.
|
||||
baselines map[string]baseline
|
||||
missing map[string]string
|
||||
}
|
||||
|
||||
// readGitIndex returns the stage-0 index object id of every tracked path in
|
||||
// the run directory (paths relative to it, as ls-files prints them).
|
||||
func readGitIndex(ctx context.Context, directory string) (map[string]string, bool) {
|
||||
cmd := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "ls-files", "--stage", "-z", "--", ".")
|
||||
out := &boundedOutput{limit: 8 << 20}
|
||||
cmd.Stdout = out
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
blobs := map[string]string{}
|
||||
for _, entry := range strings.Split(out.String(), "\x00") {
|
||||
meta, name, ok := strings.Cut(entry, "\t")
|
||||
fields := strings.Fields(meta)
|
||||
if !ok || len(fields) != 3 || fields[2] != "0" || name == "" {
|
||||
continue
|
||||
}
|
||||
blobs[name] = fields[1]
|
||||
}
|
||||
return blobs, true
|
||||
}
|
||||
|
||||
// readBlobs reads Git objects by id with one `git cat-file --batch`. Objects
|
||||
// that are missing, not blobs or larger than maxFileBytes are left out.
|
||||
func readBlobs(ctx context.Context, directory string, ids []string) map[string][]byte {
|
||||
result := map[string][]byte{}
|
||||
if len(ids) == 0 {
|
||||
return result
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "cat-file", "--batch")
|
||||
cmd.Stdin = strings.NewReader(strings.Join(ids, "\n") + "\n")
|
||||
pipe, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return result
|
||||
}
|
||||
reader := bufio.NewReader(pipe)
|
||||
for range ids {
|
||||
header, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
fields := strings.Fields(header)
|
||||
if len(fields) != 3 {
|
||||
continue // "<id> missing" carries no content
|
||||
}
|
||||
size, err := strconv.ParseInt(fields[2], 10, 64)
|
||||
if err != nil || size < 0 {
|
||||
break
|
||||
}
|
||||
if fields[1] != "blob" || size > maxFileBytes {
|
||||
if _, err := io.CopyN(io.Discard, reader, size+1); err != nil {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
data := make([]byte, size+1) // content plus the trailing newline
|
||||
if _, err := io.ReadFull(reader, data); err != nil {
|
||||
break
|
||||
}
|
||||
result[fields[0]] = data[:size]
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, pipe)
|
||||
_ = cmd.Wait()
|
||||
return result
|
||||
}
|
||||
|
||||
// readGitStatus reads `git status --porcelain=v1 -z` for the run directory and
|
||||
@@ -152,8 +252,8 @@ func (b *boundedOutput) Write(p []byte) (int, error) {
|
||||
return b.Buffer.Write(p)
|
||||
}
|
||||
|
||||
// Git only supplies an inventory; never compare against HEAD/index or alter them.
|
||||
// Tracked dirty files and untracked, nonignored files are compared as they exist.
|
||||
// inventory lists the files to snapshot. Git is only read, never altered;
|
||||
// the baseline is still stage-start content (see Capture and recoverBaselines).
|
||||
func inventory(ctx context.Context, directory string, root *os.Root) ([]string, bool, error) {
|
||||
probe := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "rev-parse", "--is-inside-work-tree")
|
||||
if data, err := probe.Output(); err == nil && strings.TrimSpace(string(data)) == "true" {
|
||||
@@ -212,8 +312,22 @@ func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snap
|
||||
s.partial = true
|
||||
s.diagnostic = "Workspace inventory incomplete."
|
||||
}
|
||||
var prev *Snapshot
|
||||
for _, prior := range previous {
|
||||
if prior != nil {
|
||||
prev = prior
|
||||
break
|
||||
}
|
||||
}
|
||||
// A stage-start capture keeps only the blob id of a text file that is clean
|
||||
// in Git (no status entry, stage-0 index entry): Git holds its content. Only
|
||||
// dirty and untracked files, which have no Git baseline, spend the budget.
|
||||
var blobs map[string]string
|
||||
if git {
|
||||
s.gitStatus, s.gitOK = readGitStatus(ctx, directory)
|
||||
if s.gitOK && prev == nil {
|
||||
blobs, _ = readGitIndex(ctx, directory)
|
||||
}
|
||||
}
|
||||
// Keep pre-stage paths even if a stage changes .gitignore/the index.
|
||||
known := map[string]bool{}
|
||||
@@ -289,22 +403,32 @@ func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snap
|
||||
if len(data) <= maxFileBytes {
|
||||
f.Close()
|
||||
binary := bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data)
|
||||
hash := sha256.Sum256(data)
|
||||
hashOnly := false
|
||||
entry := file{hash: sha256.Sum256(data), size: int64(len(data)), mode: opened.Mode().Perm(), binary: binary}
|
||||
_, dirty := s.gitStatus[name]
|
||||
p, seen := file{}, false
|
||||
if prev != nil {
|
||||
p, seen = prev.files[name]
|
||||
}
|
||||
// Binary changes need only metadata and a hash, never retained patch bytes.
|
||||
// Counting screenshots against the retained-text budget can otherwise
|
||||
// exhaust it before any source files are reached in sorted inventory order.
|
||||
// Text beyond the budget keeps only its hash, so a change to it is still
|
||||
// listed (without a patch) instead of being skipped unseen.
|
||||
if binary {
|
||||
data = nil
|
||||
} else if int64(used+len(data)) > snapshotBudget.Load() {
|
||||
data = nil
|
||||
hashOnly = true
|
||||
} else {
|
||||
switch {
|
||||
case binary:
|
||||
case seen && p.hash == entry.hash && !p.binary:
|
||||
// Unchanged since the previous snapshot: share its representation
|
||||
// without spending this snapshot's budget on it again.
|
||||
entry.data, entry.blob, entry.hashOnly, entry.limit = p.data, p.blob, p.hashOnly, p.limit
|
||||
case blobs[name] != "" && !dirty:
|
||||
entry.blob = blobs[name]
|
||||
case int64(used+len(data)) > snapshotBudget.Load():
|
||||
entry.hashOnly, entry.limit = true, reasonBudget
|
||||
default:
|
||||
entry.data = data
|
||||
used += len(data)
|
||||
}
|
||||
s.files[name] = file{data: data, hash: hash, mode: opened.Mode().Perm(), binary: binary, hashOnly: hashOnly}
|
||||
s.files[name] = entry
|
||||
continue
|
||||
}
|
||||
// The file grew past the per-file limit while being read: hash it.
|
||||
@@ -315,11 +439,60 @@ func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snap
|
||||
s.skipped[name] = true
|
||||
continue
|
||||
}
|
||||
s.files[name] = file{hash: hash, mode: opened.Mode().Perm(), binary: binary, hashOnly: !binary}
|
||||
s.files[name] = file{hash: hash, size: opened.Size(), mode: opened.Mode().Perm(), binary: binary, hashOnly: !binary, limit: reasonFileSize}
|
||||
}
|
||||
if prev != nil {
|
||||
s.recoverBaselines(ctx, directory, prev, used)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// recoverBaselines reads, in one Git batch, the stage-start content of every
|
||||
// file prev kept only as a clean blob id that has since changed or gone. The
|
||||
// content is used only if its hash matches what prev read from disk, so
|
||||
// clean/smudge filters, line-ending conversion or an index changed under the
|
||||
// capture can never yield a wrong patch; such files are listed without one.
|
||||
func (s *Snapshot) recoverBaselines(ctx context.Context, directory string, prev *Snapshot, used int) {
|
||||
var names []string
|
||||
for name, p := range prev.files {
|
||||
if p.blob == "" || p.data != nil || p.binary || s.skipped[name] {
|
||||
continue
|
||||
}
|
||||
now, ok := s.files[name]
|
||||
if ok && now.hash == p.hash || !ok && s.partial {
|
||||
continue // unchanged, or an incomplete inventory cannot prove deletion
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return
|
||||
}
|
||||
sort.Strings(names)
|
||||
s.baselines, s.missing = map[string]baseline{}, map[string]string{}
|
||||
ids := make([]string, 0, len(names))
|
||||
wanted := make([]string, 0, len(names))
|
||||
for _, name := range names {
|
||||
p := prev.files[name]
|
||||
if int64(used)+p.size > snapshotBudget.Load() {
|
||||
s.missing[name] = reasonBudget
|
||||
continue
|
||||
}
|
||||
used += int(p.size)
|
||||
ids = append(ids, p.blob)
|
||||
wanted = append(wanted, name)
|
||||
}
|
||||
contents := readBlobs(ctx, directory, ids)
|
||||
for _, name := range wanted {
|
||||
p := prev.files[name]
|
||||
data, ok := contents[p.blob]
|
||||
if !ok || sha256.Sum256(data) != p.hash {
|
||||
s.missing[name] = reasonBaseline
|
||||
continue
|
||||
}
|
||||
s.baselines[name] = baseline{data: data, hash: p.hash}
|
||||
}
|
||||
}
|
||||
|
||||
func Compare(before, after *Snapshot) *protocol.StageFileChanges {
|
||||
result := &protocol.StageFileChanges{Status: "complete", Files: []protocol.FileChange{}}
|
||||
if before == nil || after == nil || !before.available || !after.available {
|
||||
@@ -367,7 +540,10 @@ func Compare(before, after *Snapshot) *protocol.StageFileChanges {
|
||||
}
|
||||
encoded, _ := json.Marshal(change)
|
||||
if retained+len(encoded)+1 > maxPatchBytes {
|
||||
change.Patch = ""
|
||||
if change.Patch != "" {
|
||||
change.Patch = ""
|
||||
change.NoPatchReason = reasonEvidence
|
||||
}
|
||||
change.Truncated = true
|
||||
encoded, _ = json.Marshal(change)
|
||||
}
|
||||
@@ -415,14 +591,38 @@ func Compare(before, after *Snapshot) *protocol.StageFileChanges {
|
||||
if bOK {
|
||||
change.AfterMode = fmt.Sprintf("%04o", b.mode)
|
||||
}
|
||||
if !change.Binary && a.hash != b.hash && (a.hashOnly || b.hashOnly) {
|
||||
// Over-budget content was compared by hash: list it, without a patch.
|
||||
// A clean-at-start file kept only its Git blob id; its start content
|
||||
// comes from the after snapshot's verified Git recovery.
|
||||
aData, unavailable := a.data, ""
|
||||
if aOK && !a.binary && a.blob != "" && a.data == nil {
|
||||
if base, ok := after.baselines[name]; ok && base.hash == a.hash {
|
||||
aData = base.data
|
||||
} else if unavailable = after.missing[name]; unavailable == "" {
|
||||
unavailable = reasonBaseline
|
||||
}
|
||||
}
|
||||
if change.Binary {
|
||||
if a.hash != b.hash {
|
||||
change.NoPatchReason = reasonBinary
|
||||
}
|
||||
} else if a.hash != b.hash && (a.hashOnly || b.hashOnly || unavailable != "") {
|
||||
// Content compared by hash only: list it, without a patch.
|
||||
change.Truncated = true
|
||||
change.NoPatch = true
|
||||
switch {
|
||||
case a.limit == reasonFileSize || b.limit == reasonFileSize:
|
||||
change.NoPatchReason = reasonFileSize
|
||||
case unavailable != "":
|
||||
change.NoPatchReason = unavailable
|
||||
default:
|
||||
change.NoPatchReason = reasonBudget
|
||||
}
|
||||
listed++
|
||||
} else if !change.Binary && a.hash != b.hash {
|
||||
if remaining == 0 || bytes.Count(a.data, []byte("\n"))+bytes.Count(b.data, []byte("\n")) > maxLines {
|
||||
change.Truncated = true
|
||||
} else if a.hash != b.hash {
|
||||
if remaining == 0 {
|
||||
change.Truncated, change.NoPatchReason = true, reasonEvidence
|
||||
} else if bytes.Count(aData, []byte("\n"))+bytes.Count(b.data, []byte("\n")) > maxLines {
|
||||
change.Truncated, change.NoPatchReason = true, reasonLines
|
||||
} else {
|
||||
from, to := "a/"+name, "b/"+name
|
||||
if !aOK {
|
||||
@@ -432,7 +632,7 @@ func Compare(before, after *Snapshot) *protocol.StageFileChanges {
|
||||
to = "/dev/null"
|
||||
}
|
||||
// Quote paths so names cannot forge headers or terminal control sequences.
|
||||
patch, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: diffLines(a.data), B: diffLines(b.data), FromFile: fmt.Sprintf("%q", from), ToFile: fmt.Sprintf("%q", to), Context: 3})
|
||||
patch, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: diffLines(aData), B: diffLines(b.data), FromFile: fmt.Sprintf("%q", from), ToFile: fmt.Sprintf("%q", to), Context: 3})
|
||||
if err != nil {
|
||||
change.Truncated = true
|
||||
} else {
|
||||
@@ -483,7 +683,7 @@ func Compare(before, after *Snapshot) *protocol.StageFileChanges {
|
||||
case was == "" && (now == "??" || strings.HasPrefix(now, "A")):
|
||||
kind = "added"
|
||||
}
|
||||
add(protocol.FileChange{Path: name, Kind: kind, Truncated: true, NoPatch: true})
|
||||
add(protocol.FileChange{Path: name, Kind: kind, Truncated: true, NoPatch: true, NoPatchReason: reasonGit})
|
||||
listed++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,14 +212,14 @@ func gitRepo(t *testing.T) (string, func(...string)) {
|
||||
return dir, run
|
||||
}
|
||||
|
||||
// TestOverBudgetChangeIsListedWithoutPatch is the run-236 regression: a
|
||||
// workspace whose text exceeds the snapshot budget must still list the one
|
||||
// changed file (sorted last, after the budget is exhausted) instead of
|
||||
// TestOverBudgetChangeIsListedWithoutPatch is the run-236 regression outside
|
||||
// Git: a workspace whose text exceeds the snapshot budget must still list the
|
||||
// one changed file (sorted last, after the budget is exhausted) instead of
|
||||
// returning files: [] with every late path skipped.
|
||||
func TestOverBudgetChangeIsListedWithoutPatch(t *testing.T) {
|
||||
restore := SetSnapshotBudgetForTesting(64)
|
||||
defer restore()
|
||||
dir, git := gitRepo(t)
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "web/src"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -227,8 +227,6 @@ func TestOverBudgetChangeIsListedWithoutPatch(t *testing.T) {
|
||||
put(t, dir, fmt.Sprintf("a-%02d.txt", i), strings.Repeat("filler\n", 4))
|
||||
}
|
||||
put(t, dir, "web/src/Inbox.test.tsx", "it('works')\n")
|
||||
git("add", ".")
|
||||
git("commit", "-qm", "init")
|
||||
ctx := context.Background()
|
||||
before := Capture(ctx, dir)
|
||||
put(t, dir, "web/src/Inbox.test.tsx", "it('works')\nit('more')\n")
|
||||
@@ -236,7 +234,7 @@ func TestOverBudgetChangeIsListedWithoutPatch(t *testing.T) {
|
||||
if len(changes.Files) != 1 || changes.Files[0].Path != "web/src/Inbox.test.tsx" {
|
||||
t.Fatalf("over-budget change not listed: %+v", changes)
|
||||
}
|
||||
if f := changes.Files[0]; f.Kind != "modified" || !f.NoPatch || f.Patch != "" {
|
||||
if f := changes.Files[0]; f.Kind != "modified" || !f.NoPatch || f.Patch != "" || f.NoPatchReason != reasonBudget {
|
||||
t.Fatalf("over-budget change should be listed without patch: %+v", f)
|
||||
}
|
||||
if changes.Status != "partial" || changes.Skipped != 0 {
|
||||
|
||||
@@ -469,9 +469,15 @@ type FileChange struct {
|
||||
// NoPatch marks a change listed without any patch: the content exceeded the
|
||||
// capture budget (compared by hash) or the change was seen only in Git
|
||||
// status. Truncated is also set so older readers show "patch omitted".
|
||||
NoPatch bool `json:"no_patch,omitempty"`
|
||||
BeforeMode string `json:"before_mode,omitempty"`
|
||||
AfterMode string `json:"after_mode,omitempty"`
|
||||
NoPatch bool `json:"no_patch,omitempty"`
|
||||
// NoPatchReason says why a content change has no (complete) patch:
|
||||
// "binary", "file_size" (text over 1 MiB), "budget" (snapshot text budget),
|
||||
// "lines" (over 5,000 combined lines), "evidence_size" (per-visit retained
|
||||
// evidence limit), "git_status" (seen only in Git status) or "baseline"
|
||||
// (stage-start content unrecoverable). Empty when a patch was recorded.
|
||||
NoPatchReason string `json:"no_patch_reason,omitempty"`
|
||||
BeforeMode string `json:"before_mode,omitempty"`
|
||||
AfterMode string `json:"after_mode,omitempty"`
|
||||
}
|
||||
|
||||
type NotificationReceipt struct {
|
||||
|
||||
@@ -210,8 +210,8 @@ func TestUISkipPartialCaptureNeverSkips(t *testing.T) {
|
||||
|
||||
// TestUISkipOverBudgetGitWorkspaceRunsModel is the run-236 regression: in a Git
|
||||
// workspace whose text exceeds the snapshot budget, a stage edits one .tsx file
|
||||
// sorted after the budget is exhausted. The capture must list that file
|
||||
// (without a patch) and the next stage, which declares `skipped`, must run its
|
||||
// sorted after the budget would be exhausted. The capture must list that file
|
||||
// (now with its patch: clean files spend no budget) and the next stage, which declares `skipped`, must run its
|
||||
// model rather than be auto-routed with "no recorded file changes".
|
||||
func TestUISkipOverBudgetGitWorkspaceRunsModel(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
@@ -244,13 +244,12 @@ func TestUISkipOverBudgetGitWorkspaceRunsModel(t *testing.T) {
|
||||
git("commit", "-qm", "fixture")
|
||||
detail, visited, verdictCalls := runUISkipIn(t, uiReviewerSource(true), map[string]string{"implement": target}, dir)
|
||||
capture := detail.Steps[0].FileChanges
|
||||
if capture == nil || len(capture.Files) != 1 || capture.Files[0].Path != target || !capture.Files[0].NoPatch {
|
||||
t.Fatalf("over-budget capture did not list the edited file: %+v", capture)
|
||||
// Clean tracked files take their baseline from Git, so the budget no longer
|
||||
// runs out and the edited file keeps a full patch.
|
||||
if capture == nil || capture.Status != "complete" || len(capture.Files) != 1 || capture.Files[0].Path != target || capture.Files[0].NoPatch || !strings.Contains(capture.Files[0].Patch, "+implement") {
|
||||
t.Fatalf("over-budget capture did not patch the edited file: %+v", capture)
|
||||
}
|
||||
assertModelRan(t, detail, visited, verdictCalls)
|
||||
if len(detail.Warnings) == 0 {
|
||||
t.Fatal("partial capture produced no run-level warning")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUISkipOptOut proves auto_skip=false makes a `skipped` stage run its model
|
||||
|
||||
Reference in New Issue
Block a user