c42a19f7d3
Make OpenCode authoring responsive and expand the managed skill references. Add conflict-aware import/export, real-provider reference workflows, and tracked V2 generation for reliable budget accounting.
345 lines
9.4 KiB
Go
345 lines
9.4 KiB
Go
// Package filediff captures bounded, read-only stage-local workspace changes.
|
|
package filediff
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"os/exec"
|
|
"sort"
|
|
"strings"
|
|
"syscall"
|
|
"unicode/utf8"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"github.com/pmezard/go-difflib/difflib"
|
|
)
|
|
|
|
const (
|
|
maxFiles = 10000
|
|
maxFileBytes = 1 << 20
|
|
maxSnapshotBytes = 32 << 20
|
|
maxPatchBytes = 128 << 10
|
|
maxFilePatchBytes = 64 << 10
|
|
maxLines = 5000
|
|
)
|
|
|
|
type file struct {
|
|
data []byte
|
|
hash [32]byte
|
|
mode fs.FileMode
|
|
binary bool
|
|
}
|
|
|
|
type Snapshot struct {
|
|
files map[string]file
|
|
skipped map[string]bool
|
|
partial bool
|
|
available bool
|
|
diagnostic string
|
|
scope string
|
|
}
|
|
|
|
func protected(name string) bool {
|
|
for _, part := range strings.Split(strings.ToLower(name), "/") {
|
|
if part == ".git" || part == "secrets" || part == ".ssh" || part == ".aws" || part == ".gnupg" || part == ".kube" || part == "node_modules" {
|
|
return true
|
|
}
|
|
if part == ".env" || strings.HasPrefix(part, ".env.") || strings.HasSuffix(part, ".env") || strings.HasSuffix(part, ".pem") || strings.HasSuffix(part, ".key") || strings.HasSuffix(part, ".db") || strings.HasSuffix(part, ".db-wal") || strings.HasSuffix(part, ".db-shm") || strings.HasSuffix(part, ".sqlite") || strings.HasSuffix(part, ".sqlite3") || part == "auth.json" || part == "credentials.json" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
type boundedOutput struct {
|
|
bytes.Buffer
|
|
limit int
|
|
}
|
|
|
|
func (b *boundedOutput) Write(p []byte) (int, error) {
|
|
if b.Len()+len(p) > b.limit {
|
|
return 0, fmt.Errorf("file inventory limit exceeded")
|
|
}
|
|
return b.Buffer.Write(p)
|
|
}
|
|
|
|
// Git only supplies an inventory; never compare against HEAD/index or alter them.
|
|
// Tracked dirty files and untracked, nonignored files are compared as they exist.
|
|
func inventory(ctx context.Context, directory string, root *os.Root) ([]string, bool, error) {
|
|
probe := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "rev-parse", "--is-inside-work-tree")
|
|
if data, err := probe.Output(); err == nil && strings.TrimSpace(string(data)) == "true" {
|
|
cmd := exec.CommandContext(ctx, "git", "-c", "core.fsmonitor=false", "-C", directory, "ls-files", "--cached", "--others", "--exclude-standard", "--deduplicate", "-z", "--", ".")
|
|
out := &boundedOutput{limit: 2 << 20}
|
|
cmd.Stdout = out
|
|
if err := cmd.Run(); err != nil {
|
|
return nil, true, fmt.Errorf("Git file inventory unavailable")
|
|
}
|
|
return strings.Split(strings.TrimSuffix(out.String(), "\x00"), "\x00"), true, nil
|
|
}
|
|
if ctx.Err() != nil {
|
|
return nil, false, ctx.Err()
|
|
}
|
|
var names []string
|
|
err := fs.WalkDir(root.FS(), ".", func(name string, entry fs.DirEntry, err error) error {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if name != "." && protected(name) {
|
|
names = append(names, name)
|
|
if entry.IsDir() {
|
|
return fs.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
if !entry.IsDir() {
|
|
names = append(names, name)
|
|
if len(names) > maxFiles {
|
|
return fmt.Errorf("file inventory limit exceeded")
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
return names, false, err
|
|
}
|
|
|
|
func Capture(ctx context.Context, directory string, previous ...*Snapshot) *Snapshot {
|
|
s := &Snapshot{files: map[string]file{}, skipped: map[string]bool{}}
|
|
root, err := os.OpenRoot(directory)
|
|
if err != nil {
|
|
s.diagnostic = "Workspace snapshot unavailable."
|
|
return s
|
|
}
|
|
defer root.Close()
|
|
s.available = true
|
|
names, git, err := inventory(ctx, directory, root)
|
|
s.scope = "Workspace regular files"
|
|
if git {
|
|
s.scope = "Git tracked and nonignored files in the run directory"
|
|
}
|
|
if err != nil {
|
|
s.partial = true
|
|
s.diagnostic = "Workspace inventory incomplete."
|
|
}
|
|
// Keep pre-stage paths even if a stage changes .gitignore/the index.
|
|
known := map[string]bool{}
|
|
for _, name := range names {
|
|
known[name] = true
|
|
}
|
|
for _, prior := range previous {
|
|
if prior != nil {
|
|
for name := range prior.files {
|
|
if !known[name] {
|
|
names = append(names, name)
|
|
known[name] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(names)
|
|
used := 0
|
|
for i, name := range names {
|
|
if name == "" {
|
|
continue
|
|
}
|
|
if i >= maxFiles || ctx.Err() != nil {
|
|
s.partial = true
|
|
s.diagnostic = "Workspace capture reached its time/file limit."
|
|
break
|
|
}
|
|
if !fs.ValidPath(name) || protected(name) {
|
|
s.skipped[name] = true
|
|
continue
|
|
}
|
|
// Reject symlinks in every component, including aliases to sensitive paths.
|
|
valid := true
|
|
parts := strings.Split(name, "/")
|
|
for j := range parts {
|
|
info, err := root.Lstat(strings.Join(parts[:j+1], "/"))
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
valid = false
|
|
break
|
|
} // tracked deletion
|
|
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
|
s.skipped[name] = true
|
|
valid = false
|
|
break
|
|
}
|
|
}
|
|
if !valid {
|
|
continue
|
|
}
|
|
info, err := root.Lstat(name)
|
|
if err != nil || !info.Mode().IsRegular() || info.Size() > maxFileBytes {
|
|
s.skipped[name] = true
|
|
continue
|
|
}
|
|
if used+int(info.Size()) > maxSnapshotBytes {
|
|
s.skipped[name] = true
|
|
s.partial = true
|
|
continue
|
|
}
|
|
f, err := root.OpenFile(name, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
s.skipped[name] = true
|
|
continue
|
|
}
|
|
opened, statErr := f.Stat()
|
|
if statErr != nil || !opened.Mode().IsRegular() {
|
|
f.Close()
|
|
s.skipped[name] = true
|
|
continue
|
|
}
|
|
data, readErr := io.ReadAll(io.LimitReader(f, maxFileBytes+1))
|
|
f.Close()
|
|
if readErr != nil || len(data) > maxFileBytes {
|
|
s.skipped[name] = true
|
|
continue
|
|
}
|
|
used += len(data)
|
|
binary := bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data)
|
|
s.files[name] = file{data: data, hash: sha256.Sum256(data), mode: opened.Mode().Perm(), binary: binary}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func Compare(before, after *Snapshot) *protocol.StageFileChanges {
|
|
result := &protocol.StageFileChanges{Status: "complete", Files: []protocol.FileChange{}}
|
|
if before == nil || after == nil || !before.available || !after.available {
|
|
result.Status = "unavailable"
|
|
result.Diagnostic = "Workspace snapshot unavailable; changes could not be compared."
|
|
return result
|
|
}
|
|
result.Scope = before.scope
|
|
if before.partial || after.partial {
|
|
result.Status = "partial"
|
|
result.Diagnostic = "Workspace capture incomplete; some changes may be missing."
|
|
}
|
|
skipped := map[string]bool{}
|
|
for name := range before.skipped {
|
|
skipped[name] = true
|
|
}
|
|
for name := range after.skipped {
|
|
skipped[name] = true
|
|
}
|
|
result.Skipped = len(skipped)
|
|
if result.Skipped > 0 {
|
|
result.Status = "partial"
|
|
result.Diagnostic = "Excluded protected, symlinked, oversized or unreadable files; this is not a complete workspace diff."
|
|
}
|
|
keys := map[string]bool{}
|
|
for name := range before.files {
|
|
keys[name] = true
|
|
}
|
|
for name := range after.files {
|
|
keys[name] = true
|
|
}
|
|
names := make([]string, 0, len(keys))
|
|
for name := range keys {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
remaining := maxPatchBytes
|
|
retained := 0
|
|
for _, name := range names {
|
|
if skipped[name] {
|
|
continue
|
|
}
|
|
a, aOK := before.files[name]
|
|
b, bOK := after.files[name]
|
|
// Missing files in an incomplete inventory cannot prove deletion/addition.
|
|
if !aOK && before.partial || !bOK && after.partial {
|
|
continue
|
|
}
|
|
if aOK && bOK && a.hash == b.hash && a.mode == b.mode {
|
|
continue
|
|
}
|
|
kind := "modified"
|
|
if !aOK {
|
|
kind = "added"
|
|
}
|
|
if !bOK {
|
|
kind = "deleted"
|
|
}
|
|
change := protocol.FileChange{Path: name, Kind: kind, Binary: a.binary || b.binary}
|
|
if aOK {
|
|
change.BeforeMode = fmt.Sprintf("%04o", a.mode)
|
|
}
|
|
if bOK {
|
|
change.AfterMode = fmt.Sprintf("%04o", b.mode)
|
|
}
|
|
if !change.Binary && a.hash != b.hash {
|
|
if remaining == 0 || bytes.Count(a.data, []byte("\n"))+bytes.Count(b.data, []byte("\n")) > maxLines {
|
|
change.Truncated = true
|
|
} else {
|
|
from, to := "a/"+name, "b/"+name
|
|
if !aOK {
|
|
from = "/dev/null"
|
|
}
|
|
if !bOK {
|
|
to = "/dev/null"
|
|
}
|
|
// Quote paths so names cannot forge headers or terminal control sequences.
|
|
patch, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: diffLines(a.data), B: diffLines(b.data), FromFile: fmt.Sprintf("%q", from), ToFile: fmt.Sprintf("%q", to), Context: 3})
|
|
if err != nil {
|
|
change.Truncated = true
|
|
} else {
|
|
limit := min(remaining, maxFilePatchBytes)
|
|
if len(patch) > limit {
|
|
patch = patch[:limit]
|
|
for !utf8.ValidString(patch) && len(patch) > 0 {
|
|
patch = patch[:len(patch)-1]
|
|
}
|
|
change.Truncated = true
|
|
}
|
|
change.Patch = patch
|
|
remaining -= len(patch)
|
|
}
|
|
}
|
|
}
|
|
if change.Truncated {
|
|
result.Status = "partial"
|
|
result.Diagnostic = "Some patches were omitted/truncated by capture limits."
|
|
}
|
|
encoded, _ := json.Marshal(change)
|
|
if retained+len(encoded)+1 > maxPatchBytes {
|
|
change.Patch = ""
|
|
change.Truncated = true
|
|
encoded, _ = json.Marshal(change)
|
|
}
|
|
if retained+len(encoded)+1 > maxPatchBytes {
|
|
result.Omitted++
|
|
result.Status = "partial"
|
|
result.Diagnostic = "Additional file changes omitted by the retained evidence size limit."
|
|
continue
|
|
}
|
|
if change.Truncated {
|
|
result.Status = "partial"
|
|
}
|
|
retained += len(encoded) + 1
|
|
result.Files = append(result.Files, change)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func diffLines(data []byte) []string {
|
|
if len(data) == 0 {
|
|
return nil
|
|
}
|
|
lines := strings.SplitAfter(string(data), "\n")
|
|
if lines[len(lines)-1] == "" {
|
|
return lines[:len(lines)-1]
|
|
}
|
|
lines[len(lines)-1] += "\n\\ No newline at end of file\n"
|
|
return lines
|
|
}
|