package server import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "strings" "sync/atomic" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) type referenceCase struct { name, source, prompt string files map[string]string check func(*testing.T, *protocol.RunDetail, string) } func referenceCases(model string) []referenceCase { provider, id, _ := strings.Cut(model, "/") block := func(key, output, prompt, tools, edges, extra string) string { return fmt.Sprintf("# stage %s\n\n```output\n%s\n```\n\n```prompt\n%s\n```\n\n```tools\n%s\n```\n\n```configs\nprovider=%s\nmodel=%s\nretries=0\ntimeout=2m\n%s\n```\n\n```edges\n%s\nerror -> #failure\n```\n\n", key, output, prompt, tools, provider, id, extra, edges) } hook := "\n```notify\non=success,failure\n```\n" triage := block("triage", "{ issues }", "Extract the two incident records from input.prompt into issues, an array of objects with id and severity. Preserve the literal IDs/severity. No tools. Return success.", "", "success -> report", "") + block("report", "{ issues, total, high_count, summary }", "Preserve input.issues unchanged. Set total to its length and high_count to the number with severity high. Set summary to exactly: 2 incidents; 1 high priority. Return success. No tools.", "", "success -> #success", "") + hook edit := block("edit", "{ path, summary }", "Read release.txt. Change only status=pending to status=ready and version=1 to version=2 using the edit tool. Preserve the owner line and trailing newline. Do not change any other file. Return success with path release.txt and a short summary of the actual edit.", "read\nedit", "success -> verify", "") + "```permissions\n{\"edit\":[\"release.txt\"]}\n```\n" + block("verify", "{ path, verified, summary }", "Use the shell tool to execute exactly cmp release.txt expected.txt in the run directory. Return success only if it exits zero, with verified=true and path=release.txt. Otherwise return error. Do not edit files. Do not treat the preceding stage's summary as evidence.", "read\nshell", "success -> #success", "") + "```permissions\n{\"shell\":[\"cmp release.txt expected.txt\"]}\n```\n" blocked := block("inspect", "{ reason }", "Use read to check required.txt. It is deliberately missing. If unavailable return blocked with reason required.txt is missing. Do not create files, run shell commands, or invent evidence.", "read", "success -> #success\nblocked -> #failure", "") + hook budget := block("budget", "{ summary }", "Return success with summary exactly: accounting check. No tools.", "", "success -> #success", "budget_tokens=1\nbudget_usd=0.25") return []referenceCase{ {name: "incident-handoff", source: triage, prompt: "INC-101: login failures; severity=high. INC-102: typo in help text; severity=low.", check: func(t *testing.T, r *protocol.RunDetail, _ string) { if r.Status != "succeeded" || len(r.Steps) != 2 || fmt.Sprint(r.Output["total"]) != "2" || fmt.Sprint(r.Output["high_count"]) != "1" || r.Output["summary"] != "2 incidents; 1 high priority." { t.Fatalf("handoff result incorrect: %+v", r) } issues, ok := r.Output["issues"].([]any) if !ok || len(issues) != 2 { t.Fatal("incident evidence missing", r.Output) } found := map[string]string{} for _, v := range issues { m, ok := v.(map[string]any) if !ok { t.Fatal(v) } found[fmt.Sprint(m["id"])] = fmt.Sprint(m["severity"]) } if found["INC-101"] != "high" || found["INC-102"] != "low" { t.Fatal("incident evidence altered", found) } first, _ := json.Marshal(r.Steps[0].Output) second, _ := json.Marshal(r.Steps[1].Input) if string(first) != string(second) { t.Fatal("handoff changed", string(first), string(second)) } assertReferenceAlert(t, r.Steps[1], "success") }}, {name: "edit-and-verify", source: edit, prompt: "Prepare this release record and independently verify it.", files: map[string]string{"release.txt": "status=pending\nversion=1\nowner=team\n", "expected.txt": "status=ready\nversion=2\nowner=team\n", "sentinel.txt": "leave untouched\n"}, check: func(t *testing.T, r *protocol.RunDetail, dir string) { if r.Status != "succeeded" || len(r.Steps) != 2 || r.Output["verified"] != true { t.Fatalf("edit/verification failed: %+v", r) } expected := "status=ready\nversion=2\nowner=team\n" for _, name := range []string{"release.txt", "expected.txt"} { data, err := os.ReadFile(filepath.Join(dir, name)) if err != nil || string(data) != expected { t.Fatal("fixture verification failed", name, string(data), err) } } data, _ := os.ReadFile(filepath.Join(dir, "sentinel.txt")) if string(data) != "leave untouched\n" { t.Fatal("unrelated fixture changed") } cmd := exec.Command("cmp", "release.txt", "expected.txt") cmd.Dir = dir if data, err := cmd.CombinedOutput(); err != nil { t.Fatal("independent cmp failed", err, string(data)) } c := r.Steps[0].FileChanges if c == nil || c.Status != "complete" || len(c.Files) != 1 || c.Files[0].Path != "release.txt" || !strings.Contains(c.Files[0].Patch, "+status=ready") { t.Fatal("edit diff missing", c) } c = r.Steps[1].FileChanges if c == nil || c.Status != "complete" || len(c.Files) != 0 { t.Fatal("verifier edited files", c) } assertReferenceTool(t, r.Steps[0], "read") assertReferenceTool(t, r.Steps[0], "edit") assertReferenceTool(t, r.Steps[1], "shell") }}, {name: "missing-input-failure", source: blocked, prompt: "Inspect the required input; do not fabricate a result.", check: func(t *testing.T, r *protocol.RunDetail, dir string) { if r.Status != "failed" || len(r.Steps) != 1 || r.Steps[0].Outcome != "blocked" || r.Steps[0].Output["reason"] != "required.txt is missing" { t.Fatalf("failure route incorrect: %+v", r) } if _, err := os.Stat(filepath.Join(dir, "required.txt")); !os.IsNotExist(err) { t.Fatal("missing input was invented") } assertReferenceTool(t, r.Steps[0], "read") assertReferenceAlert(t, r.Steps[0], "failure") }}, {name: "accounting-and-budget", source: budget, prompt: "Return the requested short accounting check.", check: func(t *testing.T, r *protocol.RunDetail, _ string) { if r.Status != "failed" || len(r.Steps) != 1 || !strings.Contains(r.Steps[0].Diagnostic, stage.ErrBudgetExceeded.Error()) { t.Fatalf("budget enforcement not confirmed: %+v", r) } var audit struct { Limits struct { Budget stage.BudgetSnapshot `json:"budget"` } `json:"stage_limits"` } if err := json.Unmarshal([]byte(r.Steps[0].Transcript), &audit); err != nil { t.Fatal(err) } b := audit.Limits.Budget if !b.TokensKnown || !b.USDKnown || b.UsedTokens < 1 || b.Attempts != 1 { t.Fatal("complete reported accounting not retained", b) } t.Logf("Reported usage: tokens=%d USD=%.8f (not an independently reconciled bill)", b.UsedTokens, b.UsedUSD) }}, } } func assertReferenceAlert(t *testing.T, step protocol.RunStep, event string) { t.Helper() if len(step.Notifications) != 1 || step.Notifications[0].Status != "sent" || step.Notifications[0].MessageID != "reference-local-receipt" { t.Fatal("local notification receipt missing", step.Notifications) } } func referenceToolFound(text, tool string, allowReadError bool) bool { var value any if json.Unmarshal([]byte(text), &value) != nil { return false } found := false var visit func(any) visit = func(v any) { switch x := v.(type) { case map[string]any: name, _ := x["tool"].(string) if name == "" { name, _ = x["name"].(string) } status, _ := x["status"].(string) if state, ok := x["state"].(map[string]any); ok { status, _ = state["status"].(string) } // Failed read is valid evidence for deliberately missing input, not success. if name == tool && (status == "completed" || status == "success" || allowReadError && tool == "read" && status == "error") { found = true } for _, child := range x { visit(child) } case []any: for _, child := range x { visit(child) } } } visit(value) return found } func assertReferenceTool(t *testing.T, step protocol.RunStep, tool string) { t.Helper() if !referenceToolFound(step.Transcript, tool, step.Outcome == "blocked") { t.Fatalf("no retained terminal %s tool evidence in stage %s", tool, step.Stage) } } func TestReferenceDefinitions(t *testing.T) { for _, c := range referenceCases("opencode-go/deepseek-v4.1-flash") { t.Run(c.name, func(t *testing.T) { if _, err := pipeline.Parse(c.source); err != nil { t.Fatal(err) } }) } } // Explicit opt-in; uses real credentials/inference. Artifacts and sessions are retained. // Six stage attempts plus backend compatibility/format-repair calls may incur charges. func TestReferenceWorkflowsLive(t *testing.T) { if os.Getenv("PIPELINES_REFERENCE_LIVE") != "1" { t.Skip("set PIPELINES_REFERENCE_LIVE=1, PIPELINES_REFERENCE_MODEL and PIPELINES_REFERENCE_DIR for chargeable reference inference") } model := os.Getenv("PIPELINES_REFERENCE_MODEL") provider, id, ok := strings.Cut(model, "/") if !ok || !stage.ValidModel(provider, id) { t.Fatal("explicit valid PIPELINES_REFERENCE_MODEL required") } base := os.Getenv("PIPELINES_REFERENCE_DIR") if base == "" { t.Fatal("PIPELINES_REFERENCE_DIR required for retained private artifacts") } root, err := os.MkdirTemp(base, "pipelines-reference-") if err != nil { t.Fatal(err) } t.Log("Retained reference artifacts:", root) ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) defer cancel() store, err := pipeline.Open(filepath.Join(root, "runs.db")) if err != nil { t.Fatal(err) } defer store.Close() runner, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } defer func() { cancel(); runner.wg.Wait() }() var alerts atomic.Int32 endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { alerts.Add(1) w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"id":"reference-local-receipt","event":"message"}`) })) defer endpoint.Close() t.Setenv("PIPELINES_NTFY_URL", endpoint.URL+"/reference") t.Setenv("PIPELINES_NTFY_TOKEN", "") t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "") type result struct { Name string `json:"name"` Verified bool `json:"verified"` RunID int64 `json:"run_id"` Status string `json:"status"` Duration string `json:"duration"` Diagnostic string `json:"diagnostic,omitempty"` } report := struct { Model string `json:"model"` NotificationScope string `json:"notification_scope"` Results []result `json:"results"` }{Model: model, NotificationScope: "isolated local HTTP endpoint; phone reception not tested", Results: []result{}} write := func(name string, v any) { t.Helper() data, err := json.MarshalIndent(v, "", " ") if err != nil { t.Fatal(err) } if err = os.WriteFile(filepath.Join(root, name), data, 0600); err != nil { t.Fatal(err) } } defer func() { write("report.json", report) }() for _, c := range referenceCases(model) { c := c t.Run(c.name, func(t *testing.T) { started := time.Now() row := result{Name: c.name} defer func() { row.Verified = !t.Failed() if !row.Verified && row.Diagnostic == "" { row.Diagnostic = "Independent acceptance checks failed; inspect the test log and retained run/session JSON." } row.Duration = time.Since(started).String() report.Results = append(report.Results, row) write("report.json", report) }() workspace := filepath.Join(root, c.name) if err := os.Mkdir(workspace, 0700); err != nil { t.Fatal(err) } for name, text := range c.files { if err := os.WriteFile(filepath.Join(workspace, name), []byte(text), 0600); err != nil { t.Fatal(err) } } if err := os.WriteFile(filepath.Join(root, c.name+".md"), []byte(c.source), 0600); err != nil { t.Fatal(err) } if _, err := store.Submit(ctx, c.name, c.source, true); err != nil { t.Fatal(err) } items, err := store.List(ctx) if err != nil { t.Fatal(err) } var pid int64 for _, p := range items { if p.Name == c.name { pid = p.ID } } priorAlerts := alerts.Load() run, err := runner.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: c.name, Workspace: workspace, Input: map[string]any{"prompt": c.prompt}}) if err != nil { row.Diagnostic = err.Error() t.Fatal(err) } runner.wg.Wait() run, err = store.GetRun(context.Background(), run.ID) if err != nil { t.Fatal(err) } row.RunID = run.ID row.Status = run.Status row.Diagnostic = run.Diagnostic write(c.name+".run.json", run) evidence, evidenceErr := runner.evidence(ctx, run.ID) write(c.name+".sessions.json", evidence) // Persist failure evidence before asserting workflow-specific results. c.check(t, run, workspace) if evidenceErr != nil || len(evidence) != len(run.Steps) { t.Fatal("session readback incomplete", evidenceErr) } for _, item := range evidence { if item.Diagnostic != "" || len(item.Data) == 0 { t.Fatal("retained session readback failed", item.Diagnostic) } } for _, step := range run.Steps { if step.SessionID == "" || step.AgentActive { t.Fatal("stage session absent or still active", step) } if step.FileChanges == nil { t.Fatal("workspace capture absent") } } wantAlerts := int32(0) if c.name == "incident-handoff" || c.name == "missing-input-failure" { wantAlerts = 1 } if alerts.Load()-priorAlerts != wantAlerts { t.Fatal("unexpected notification count") } t.Logf("Verified %s: run #%d status=%s visits=%d", c.name, run.ID, run.Status, len(run.Steps)) }) } } // Re-check copied artifacts without additional inference or changing their DB. func TestReferenceArtifactsReview(t *testing.T) { root := os.Getenv("PIPELINES_REFERENCE_REVIEW_DIR") if root == "" { t.Skip("set PIPELINES_REFERENCE_REVIEW_DIR for independent retained-artifact checks (no inference)") } for _, c := range referenceCases("opencode-go/deepseek-v4.1-flash") { c := c t.Run(c.name, func(t *testing.T) { data, err := os.ReadFile(filepath.Join(root, c.name+".run.json")) if os.IsNotExist(err) { t.Skip("case not present in this artifact directory") } if err != nil { t.Fatal(err) } var run protocol.RunDetail if err = json.Unmarshal(data, &run); err != nil { t.Fatal(err) } c.check(t, &run, filepath.Join(root, c.name)) }) } } func TestReferenceToolEvidenceRejectsClaims(t *testing.T) { if referenceToolFound(`{"data":[{"type":"text","text":"I used read and edit successfully"}]}`, "edit", false) { t.Fatal("model prose accepted as tool evidence") } data := `{"data":[{"content":[{"tool":"read","state":{"status":"error"}}]}]}` if referenceToolFound(data, "read", false) || !referenceToolFound(data, "read", true) { t.Fatal("read error confused with successful read") } }