70391b17c2
On a large Git workspace the 32 MiB retained-text budget was spent alphabetically on files that never change, so small late edits were recorded with no_patch. The stage-start snapshot now keeps text only for files dirty or untracked at start; a clean tracked file keeps its hash and stage-0 index blob id. When it changes or is deleted, the after capture reads that blob in one `git cat-file --batch` and uses it only if its SHA-256 matches the start read, so stage commits/resets do not move the baseline and filters or assume-unchanged never yield a wrong patch. The after capture shares unchanged files' representation instead of spending budget on them again. FileChange gains no_patch_reason (binary, file_size, budget, lines, evidence_size, git_status, baseline). Non-Git workspaces keep their behaviour.
362 lines
15 KiB
Go
362 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/filediff"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// uiReviewerSource builds a two-stage implement -> ui_reviewer pipeline. When
|
|
// withSkip is set, ui_reviewer declares an `approved` and a `skipped` route to
|
|
// success; otherwise it only declares the template's `success` route, proving a
|
|
// skip is never invented for a stage without that route.
|
|
func uiReviewerSource(withSkip bool) string {
|
|
source := pipeline.Template([]string{"implement", "ui_reviewer"}, true)
|
|
if withSkip {
|
|
source = strings.Replace(source, "success -> #success", "approved -> #success\nskipped -> #success", 1)
|
|
}
|
|
return source
|
|
}
|
|
|
|
// runUISkip submits source, runs it once with an execute stub that records every
|
|
// visited stage key and writes the file named for that stage into the workspace,
|
|
// then returns the persisted detail, the visited keys and the number of verdict
|
|
// (model) calls the runner made.
|
|
func runUISkip(t *testing.T, source string, files map[string]string) (*protocol.RunDetail, []string, int) {
|
|
t.Helper()
|
|
return runUISkipIn(t, source, files, t.TempDir())
|
|
}
|
|
|
|
// runUISkipIn is runUISkip against a prepared workspace directory.
|
|
func runUISkipIn(t *testing.T, source string, files map[string]string, dir string) (*protocol.RunDetail, []string, int) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "ui.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "ui", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil || len(items) != 1 {
|
|
t.Fatalf("submit/list: %+v %v", items, err)
|
|
}
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r.wg.Wait()
|
|
var mu sync.Mutex
|
|
var visited []string
|
|
verdictCalls := 0
|
|
r.verdict = func(context.Context, map[string]any) string {
|
|
mu.Lock()
|
|
verdictCalls++
|
|
mu.Unlock()
|
|
return "stub verdict"
|
|
}
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
mu.Lock()
|
|
visited = append(visited, def.Key)
|
|
mu.Unlock()
|
|
if name := files[def.Key]; name != "" {
|
|
full := filepath.Join(dir, filepath.FromSlash(name))
|
|
if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
if err := os.WriteFile(full, []byte(def.Key+"\n"), 0o600); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
}
|
|
outcome := "success"
|
|
if def.Key == "ui_reviewer" {
|
|
if _, ok := def.Edges["approved"]; ok {
|
|
outcome = "approved"
|
|
}
|
|
}
|
|
return stage.Result{Outcome: outcome, Output: stage.Values{"summary": def.Key + " done"}}, nil
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "ui-skip", Workspace: dir, Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mu.Lock()
|
|
got := append([]string(nil), visited...)
|
|
calls := verdictCalls
|
|
mu.Unlock()
|
|
return detail, got, calls
|
|
}
|
|
|
|
func executed(visited []string, key string) bool {
|
|
for _, visitedKey := range visited {
|
|
if visitedKey == key {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// assertModelRan fails unless ui_reviewer executed its model and routed through
|
|
// its own outcome rather than the server fallback.
|
|
func assertModelRan(t *testing.T, detail *protocol.RunDetail, visited []string, verdictCalls int) {
|
|
t.Helper()
|
|
if !executed(visited, "ui_reviewer") {
|
|
t.Fatalf("ui_reviewer was auto-skipped; visited=%v steps=%+v", visited, detail.Steps)
|
|
}
|
|
if verdictCalls != 2 {
|
|
t.Fatalf("verdict (model) calls = %d, want 2 for both executed stages", verdictCalls)
|
|
}
|
|
if detail.Status != "succeeded" {
|
|
t.Fatalf("run status = %q, want succeeded: %+v", detail.Status, detail.RunSummary)
|
|
}
|
|
step := detail.Steps[len(detail.Steps)-1]
|
|
if step.Stage != "ui_reviewer" || step.Outcome != "approved" || step.Target != protocol.SuccessEnd {
|
|
t.Fatalf("executed step did not route through approved: %+v", step)
|
|
}
|
|
}
|
|
|
|
// TestUISkipZeroChangesStillSkips proves the fallback short-circuits when every
|
|
// visit recorded a complete capture with no file changes, and that the verdict
|
|
// says it was a server fallback with no model run.
|
|
func TestUISkipZeroChangesStillSkips(t *testing.T) {
|
|
detail, visited, verdictCalls := runUISkip(t, uiReviewerSource(true), nil)
|
|
if executed(visited, "ui_reviewer") {
|
|
t.Fatalf("ui_reviewer executed with no recorded changes; visited=%v", visited)
|
|
}
|
|
if verdictCalls != 1 {
|
|
t.Fatalf("verdict (model) calls = %d, want only implement's 1", verdictCalls)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
|
|
t.Fatalf("run = %q steps=%d, want succeeded/2", detail.Status, len(detail.Steps))
|
|
}
|
|
if first := detail.Steps[0].FileChanges; first == nil || first.Status != "complete" {
|
|
t.Fatalf("implement capture = %+v, want complete", first)
|
|
}
|
|
step := detail.Steps[1]
|
|
if step.Stage != "ui_reviewer" || step.Status != "completed" || step.Outcome != "skipped" || step.Target != protocol.SuccessEnd {
|
|
t.Fatalf("skip step wrong: %+v", step)
|
|
}
|
|
if step.FileChanges != nil {
|
|
t.Fatalf("skip step recorded a capture: %+v", step.FileChanges)
|
|
}
|
|
if step.Verdict == nil || step.Verdict.ReasonSource != protocol.VerdictReasonFallback || !strings.HasPrefix(step.Verdict.Reason, serverSkipReasonPrefix) {
|
|
t.Fatalf("skip verdict does not name the server fallback: %+v", step.Verdict)
|
|
}
|
|
reason, _ := step.Output["summary"].(string)
|
|
if !strings.Contains(reason, "recorded no file changes") {
|
|
t.Fatalf("zero-change reason does not state that: %q", reason)
|
|
}
|
|
if len(detail.Warnings) != 0 {
|
|
t.Fatalf("complete run carries warnings: %v", detail.Warnings)
|
|
}
|
|
}
|
|
|
|
// TestUISkipRunsModelOnAnyRecordedChange proves no path is classified as "not
|
|
// UI": a backend file, a Flutter .dart file, a Kotlin file, a Go TUI file and a
|
|
// web .tsx file all make the stage run its model. The fallback never encodes a
|
|
// repository layout.
|
|
func TestUISkipRunsModelOnAnyRecordedChange(t *testing.T) {
|
|
cases := map[string]string{
|
|
"go backend": "internal/server/fake.go",
|
|
"flutter dart": "app/lib/features/home/home_page.dart",
|
|
"kotlin": "android/app/src/main/kotlin/Main.kt",
|
|
"go tui": "internal/tui/view.go",
|
|
"web tsx": "web/app.tsx",
|
|
"docs": "docs/readme.md",
|
|
}
|
|
for name, path := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
detail, visited, verdictCalls := runUISkip(t, uiReviewerSource(true), map[string]string{"implement": path})
|
|
assertModelRan(t, detail, visited, verdictCalls)
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestUISkipPartialCaptureNeverSkips proves a visit whose capture was partial
|
|
// (here a protected .env file forces skipped>0) blocks the fallback even when
|
|
// it recorded no file changes, and that inspect carries a run-level warning.
|
|
func TestUISkipPartialCaptureNeverSkips(t *testing.T) {
|
|
dir := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("A=1\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
detail, visited, verdictCalls := runUISkipIn(t, uiReviewerSource(true), nil, dir)
|
|
if first := detail.Steps[0].FileChanges; first == nil || first.Status != "partial" || len(first.Files) != 0 {
|
|
t.Fatalf("fixture capture = %+v, want partial with no files", first)
|
|
}
|
|
assertModelRan(t, detail, visited, verdictCalls)
|
|
if len(detail.Warnings) != 1 || !strings.Contains(detail.Warnings[0], "implement #1 (partial)") {
|
|
t.Fatalf("partial capture warning missing: %v", detail.Warnings)
|
|
}
|
|
}
|
|
|
|
// TestUISkipOverBudgetGitWorkspaceRunsModel is the run-236 regression: in a Git
|
|
// workspace whose text exceeds the snapshot budget, a stage edits one .tsx file
|
|
// sorted after the budget would be exhausted. The capture must list that file
|
|
// (now with its patch: clean files spend no budget) and the next stage, which declares `skipped`, must run its
|
|
// model rather than be auto-routed with "no recorded file changes".
|
|
func TestUISkipOverBudgetGitWorkspaceRunsModel(t *testing.T) {
|
|
if _, err := exec.LookPath("git"); err != nil {
|
|
t.Skip("git unavailable")
|
|
}
|
|
restore := filediff.SetSnapshotBudgetForTesting(256)
|
|
defer restore()
|
|
dir := t.TempDir()
|
|
git := func(args ...string) {
|
|
t.Helper()
|
|
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.email=t@example.invalid", "-c", "user.name=t", "-c", "commit.gpgsign=false"}, args...)...)
|
|
if data, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("%v: %v %s", args, err, data)
|
|
}
|
|
}
|
|
git("init", "-q")
|
|
target := "web/src/pages/inbox/PortfolioIntelligenceInbox.test.tsx"
|
|
if err := os.MkdirAll(filepath.Join(dir, filepath.Dir(target)), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 40; i++ {
|
|
if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("a-%02d.txt", i)), []byte(strings.Repeat("filler line\n", 4)), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, filepath.FromSlash(target)), []byte(strings.Repeat("original test line\n", 8)), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
git("add", ".")
|
|
git("commit", "-qm", "fixture")
|
|
detail, visited, verdictCalls := runUISkipIn(t, uiReviewerSource(true), map[string]string{"implement": target}, dir)
|
|
capture := detail.Steps[0].FileChanges
|
|
// Clean tracked files take their baseline from Git, so the budget no longer
|
|
// runs out and the edited file keeps a full patch.
|
|
if capture == nil || capture.Status != "complete" || len(capture.Files) != 1 || capture.Files[0].Path != target || capture.Files[0].NoPatch || !strings.Contains(capture.Files[0].Patch, "+implement") {
|
|
t.Fatalf("over-budget capture did not patch the edited file: %+v", capture)
|
|
}
|
|
assertModelRan(t, detail, visited, verdictCalls)
|
|
}
|
|
|
|
// TestUISkipOptOut proves auto_skip=false makes a `skipped` stage run its model
|
|
// even on complete, empty evidence.
|
|
func TestUISkipOptOut(t *testing.T) {
|
|
source := uiReviewerSource(true)
|
|
at := strings.LastIndex(source, "timeout=1h\n")
|
|
source = source[:at] + "timeout=1h\nauto_skip=false\n" + source[at+len("timeout=1h\n"):]
|
|
detail, visited, verdictCalls := runUISkip(t, source, nil)
|
|
assertModelRan(t, detail, visited, verdictCalls)
|
|
}
|
|
|
|
// TestUISkipNeedsDeclaredEdge proves the guard never invents a skip for a stage
|
|
// whose definition declares no `skipped` route: it always executes, even on
|
|
// complete, empty evidence.
|
|
func TestUISkipNeedsDeclaredEdge(t *testing.T) {
|
|
detail, visited, verdictCalls := runUISkip(t, uiReviewerSource(false), nil)
|
|
if !executed(visited, "ui_reviewer") {
|
|
t.Fatalf("ui_reviewer without a skipped edge did not execute; visited=%v", visited)
|
|
}
|
|
if verdictCalls != 2 {
|
|
t.Fatalf("verdict (model) calls = %d, want 2 for both executed stages", verdictCalls)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 2 {
|
|
t.Fatalf("run = %q steps=%d, want succeeded/2", detail.Status, len(detail.Steps))
|
|
}
|
|
step := detail.Steps[1]
|
|
if step.Stage != "ui_reviewer" || step.Outcome != "success" {
|
|
t.Fatalf("edge-less stage step wrong: %+v", step)
|
|
}
|
|
}
|
|
|
|
// TestUISkipPersistedEvidence proves the resume/retry seed: a stored server
|
|
// fallback step neither adds nor breaks evidence, while a stored visit with no
|
|
// capture (a human decision, a crash, a historical row) or a partial one makes
|
|
// the evidence incomplete.
|
|
func TestUISkipPersistedEvidence(t *testing.T) {
|
|
complete := &protocol.StageFileChanges{Status: "complete", Files: []protocol.FileChange{}}
|
|
reason := skipReasonLine()
|
|
serverSkip := protocol.RunStep{Stage: "ui", Outcome: "skipped", Verdict: &protocol.StageVerdict{Reason: reason, ReasonSource: protocol.VerdictReasonFallback}}
|
|
s := &runState{}
|
|
s.recordPersistedStep(protocol.RunStep{Stage: "implement", FileChanges: complete})
|
|
s.recordPersistedStep(serverSkip)
|
|
if !autoSkipEligible(nil, s) {
|
|
t.Fatalf("complete evidence plus a server skip is not eligible: %+v", s)
|
|
}
|
|
legacy := serverSkip
|
|
legacy.Verdict = &protocol.StageVerdict{Reason: "no UI surface changed (no recorded file changes)", ReasonSource: protocol.VerdictReasonFallback}
|
|
for name, step := range map[string]protocol.RunStep{
|
|
"no capture": {Stage: "gate", Outcome: "approve"},
|
|
"partial": {Stage: "implement", FileChanges: &protocol.StageFileChanges{Status: "partial", Files: []protocol.FileChange{}}},
|
|
"unavailable": {Stage: "implement", FileChanges: &protocol.StageFileChanges{Status: "unavailable"}},
|
|
"legacy skip row": legacy,
|
|
} {
|
|
s := &runState{}
|
|
s.recordPersistedStep(step)
|
|
if autoSkipEligible(nil, s) {
|
|
t.Errorf("%s: incomplete evidence is eligible for the fallback", name)
|
|
}
|
|
}
|
|
if autoSkipEligible(map[string]string{"auto_skip": "false"}, &runState{}) {
|
|
t.Error("auto_skip=false did not opt out")
|
|
}
|
|
}
|
|
|
|
func TestUISurfaceArtifactClassification(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
a protocol.StageArtifact
|
|
want bool
|
|
}{
|
|
{"image content type", protocol.StageArtifact{Name: "shot.bin", ContentType: "image/png"}, true},
|
|
{"image extension only", protocol.StageArtifact{Name: "shot.webp"}, true},
|
|
{"non-image artifact", protocol.StageArtifact{Name: "report.pdf", ContentType: "application/pdf"}, false},
|
|
{"binary artifact", protocol.StageArtifact{Name: "data.bin", ContentType: "application/octet-stream"}, false},
|
|
{"no type no image extension", protocol.StageArtifact{Name: "notes.txt"}, false},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := uiSurfaceArtifact(tc.a); got != tc.want {
|
|
t.Errorf("%s: uiSurfaceArtifact(%+v) = %v, want %v", tc.name, tc.a, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestUISurfaceRecordChanges proves the accumulator latches a UI surface from an
|
|
// image artifact but not from a non-image one, and retains only a bounded sample.
|
|
func TestUISurfaceRecordChanges(t *testing.T) {
|
|
image := &runState{}
|
|
image.recordChanges(&protocol.StageFileChanges{Files: []protocol.FileChange{{Path: "internal/server/fake.go"}}}, []protocol.StageArtifact{{Name: "shot.png", ContentType: "image/png"}})
|
|
if !image.uiSurface {
|
|
t.Fatal("an image artifact did not mark a UI surface")
|
|
}
|
|
|
|
plain := &runState{}
|
|
plain.recordChanges(&protocol.StageFileChanges{Files: []protocol.FileChange{{Path: "internal/server/fake.go"}}}, []protocol.StageArtifact{{Name: "report.pdf", ContentType: "application/pdf"}})
|
|
if plain.uiSurface {
|
|
t.Fatal("a non-image artifact marked a UI surface")
|
|
}
|
|
|
|
bounded := &runState{}
|
|
files := make([]protocol.FileChange, maxRetainedChangedPaths+5)
|
|
for i := range files {
|
|
files[i] = protocol.FileChange{Path: "f"}
|
|
}
|
|
bounded.recordChanges(&protocol.StageFileChanges{Files: files}, nil)
|
|
if len(bounded.changedPaths) != maxRetainedChangedPaths || bounded.changedCount != len(files) {
|
|
t.Fatalf("bounded accumulator = %d paths / count %d, want %d / %d", len(bounded.changedPaths), bounded.changedCount, maxRetainedChangedPaths, len(files))
|
|
}
|
|
}
|