Files
pipelines/internal/server/reference_live_test.go
alan c42a19f7d3 Improve workflow UI, retain stage diffs, and add portable pipeline transfers
Make OpenCode authoring responsive and expand the managed skill references. Add conflict-aware import/export, real-provider reference workflows, and tracked V2 generation for reliable budget accounting.
2026-10-10 20:55:42 +01:00

376 lines
15 KiB
Go

package server
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
type referenceCase struct {
name, source, prompt string
files map[string]string
check func(*testing.T, *protocol.RunDetail, string)
}
func referenceCases(model string) []referenceCase {
provider, id, _ := strings.Cut(model, "/")
block := func(key, output, prompt, tools, edges, extra string) string {
return fmt.Sprintf("# stage %s\n\n```output\n%s\n```\n\n```prompt\n%s\n```\n\n```tools\n%s\n```\n\n```configs\nprovider=%s\nmodel=%s\nretries=0\ntimeout=2m\n%s\n```\n\n```edges\n%s\nerror -> #failure\n```\n\n", key, output, prompt, tools, provider, id, extra, edges)
}
hook := "\n```notify\non=success,failure\n```\n"
triage := block("triage", "{ issues }", "Extract the two incident records from input.prompt into issues, an array of objects with id and severity. Preserve the literal IDs/severity. No tools. Return success.", "", "success -> report", "") +
block("report", "{ issues, total, high_count, summary }", "Preserve input.issues unchanged. Set total to its length and high_count to the number with severity high. Set summary to exactly: 2 incidents; 1 high priority. Return success. No tools.", "", "success -> #success", "") + hook
edit := block("edit", "{ path, summary }", "Read release.txt. Change only status=pending to status=ready and version=1 to version=2 using the edit tool. Preserve the owner line and trailing newline. Do not change any other file. Return success with path release.txt and a short summary of the actual edit.", "read\nedit", "success -> verify", "") +
"```permissions\n{\"edit\":[\"release.txt\"]}\n```\n" +
block("verify", "{ path, verified, summary }", "Use the shell tool to execute exactly cmp release.txt expected.txt in the run directory. Return success only if it exits zero, with verified=true and path=release.txt. Otherwise return error. Do not edit files. Do not treat the preceding stage's summary as evidence.", "read\nshell", "success -> #success", "") +
"```permissions\n{\"shell\":[\"cmp release.txt expected.txt\"]}\n```\n"
blocked := block("inspect", "{ reason }", "Use read to check required.txt. It is deliberately missing. If unavailable return blocked with reason required.txt is missing. Do not create files, run shell commands, or invent evidence.", "read", "success -> #success\nblocked -> #failure", "") + hook
budget := block("budget", "{ summary }", "Return success with summary exactly: accounting check. No tools.", "", "success -> #success", "budget_tokens=1\nbudget_usd=0.25")
return []referenceCase{
{name: "incident-handoff", source: triage, prompt: "INC-101: login failures; severity=high. INC-102: typo in help text; severity=low.", check: func(t *testing.T, r *protocol.RunDetail, _ string) {
if r.Status != "succeeded" || len(r.Steps) != 2 || fmt.Sprint(r.Output["total"]) != "2" || fmt.Sprint(r.Output["high_count"]) != "1" || r.Output["summary"] != "2 incidents; 1 high priority." {
t.Fatalf("handoff result incorrect: %+v", r)
}
issues, ok := r.Output["issues"].([]any)
if !ok || len(issues) != 2 {
t.Fatal("incident evidence missing", r.Output)
}
found := map[string]string{}
for _, v := range issues {
m, ok := v.(map[string]any)
if !ok {
t.Fatal(v)
}
found[fmt.Sprint(m["id"])] = fmt.Sprint(m["severity"])
}
if found["INC-101"] != "high" || found["INC-102"] != "low" {
t.Fatal("incident evidence altered", found)
}
first, _ := json.Marshal(r.Steps[0].Output)
second, _ := json.Marshal(r.Steps[1].Input)
if string(first) != string(second) {
t.Fatal("handoff changed", string(first), string(second))
}
assertReferenceAlert(t, r.Steps[1], "success")
}},
{name: "edit-and-verify", source: edit, prompt: "Prepare this release record and independently verify it.", files: map[string]string{"release.txt": "status=pending\nversion=1\nowner=team\n", "expected.txt": "status=ready\nversion=2\nowner=team\n", "sentinel.txt": "leave untouched\n"}, check: func(t *testing.T, r *protocol.RunDetail, dir string) {
if r.Status != "succeeded" || len(r.Steps) != 2 || r.Output["verified"] != true {
t.Fatalf("edit/verification failed: %+v", r)
}
expected := "status=ready\nversion=2\nowner=team\n"
for _, name := range []string{"release.txt", "expected.txt"} {
data, err := os.ReadFile(filepath.Join(dir, name))
if err != nil || string(data) != expected {
t.Fatal("fixture verification failed", name, string(data), err)
}
}
data, _ := os.ReadFile(filepath.Join(dir, "sentinel.txt"))
if string(data) != "leave untouched\n" {
t.Fatal("unrelated fixture changed")
}
cmd := exec.Command("cmp", "release.txt", "expected.txt")
cmd.Dir = dir
if data, err := cmd.CombinedOutput(); err != nil {
t.Fatal("independent cmp failed", err, string(data))
}
c := r.Steps[0].FileChanges
if c == nil || c.Status != "complete" || len(c.Files) != 1 || c.Files[0].Path != "release.txt" || !strings.Contains(c.Files[0].Patch, "+status=ready") {
t.Fatal("edit diff missing", c)
}
c = r.Steps[1].FileChanges
if c == nil || c.Status != "complete" || len(c.Files) != 0 {
t.Fatal("verifier edited files", c)
}
assertReferenceTool(t, r.Steps[0], "read")
assertReferenceTool(t, r.Steps[0], "edit")
assertReferenceTool(t, r.Steps[1], "shell")
}},
{name: "missing-input-failure", source: blocked, prompt: "Inspect the required input; do not fabricate a result.", check: func(t *testing.T, r *protocol.RunDetail, dir string) {
if r.Status != "failed" || len(r.Steps) != 1 || r.Steps[0].Outcome != "blocked" || r.Steps[0].Output["reason"] != "required.txt is missing" {
t.Fatalf("failure route incorrect: %+v", r)
}
if _, err := os.Stat(filepath.Join(dir, "required.txt")); !os.IsNotExist(err) {
t.Fatal("missing input was invented")
}
assertReferenceTool(t, r.Steps[0], "read")
assertReferenceAlert(t, r.Steps[0], "failure")
}},
{name: "accounting-and-budget", source: budget, prompt: "Return the requested short accounting check.", check: func(t *testing.T, r *protocol.RunDetail, _ string) {
if r.Status != "failed" || len(r.Steps) != 1 || !strings.Contains(r.Steps[0].Diagnostic, stage.ErrBudgetExceeded.Error()) {
t.Fatalf("budget enforcement not confirmed: %+v", r)
}
var audit struct {
Limits struct {
Budget stage.BudgetSnapshot `json:"budget"`
} `json:"stage_limits"`
}
if err := json.Unmarshal([]byte(r.Steps[0].Transcript), &audit); err != nil {
t.Fatal(err)
}
b := audit.Limits.Budget
if !b.TokensKnown || !b.USDKnown || b.UsedTokens < 1 || b.Attempts != 1 {
t.Fatal("complete reported accounting not retained", b)
}
t.Logf("Reported usage: tokens=%d USD=%.8f (not an independently reconciled bill)", b.UsedTokens, b.UsedUSD)
}},
}
}
func assertReferenceAlert(t *testing.T, step protocol.RunStep, event string) {
t.Helper()
if len(step.Notifications) != 1 || step.Notifications[0].Status != "sent" || step.Notifications[0].MessageID != "reference-local-receipt" {
t.Fatal("local notification receipt missing", step.Notifications)
}
}
func referenceToolFound(text, tool string, allowReadError bool) bool {
var value any
if json.Unmarshal([]byte(text), &value) != nil {
return false
}
found := false
var visit func(any)
visit = func(v any) {
switch x := v.(type) {
case map[string]any:
name, _ := x["tool"].(string)
if name == "" {
name, _ = x["name"].(string)
}
status, _ := x["status"].(string)
if state, ok := x["state"].(map[string]any); ok {
status, _ = state["status"].(string)
}
// Failed read is valid evidence for deliberately missing input, not success.
if name == tool && (status == "completed" || status == "success" || allowReadError && tool == "read" && status == "error") {
found = true
}
for _, child := range x {
visit(child)
}
case []any:
for _, child := range x {
visit(child)
}
}
}
visit(value)
return found
}
func assertReferenceTool(t *testing.T, step protocol.RunStep, tool string) {
t.Helper()
if !referenceToolFound(step.Transcript, tool, step.Outcome == "blocked") {
t.Fatalf("no retained terminal %s tool evidence in stage %s", tool, step.Stage)
}
}
func TestReferenceDefinitions(t *testing.T) {
for _, c := range referenceCases("opencode-go/deepseek-v4.1-flash") {
t.Run(c.name, func(t *testing.T) {
if _, err := pipeline.Parse(c.source); err != nil {
t.Fatal(err)
}
})
}
}
// Explicit opt-in; uses real credentials/inference. Artifacts and sessions are retained.
// Six stage attempts plus backend compatibility/format-repair calls may incur charges.
func TestReferenceWorkflowsLive(t *testing.T) {
if os.Getenv("PIPELINES_REFERENCE_LIVE") != "1" {
t.Skip("set PIPELINES_REFERENCE_LIVE=1, PIPELINES_REFERENCE_MODEL and PIPELINES_REFERENCE_DIR for chargeable reference inference")
}
model := os.Getenv("PIPELINES_REFERENCE_MODEL")
provider, id, ok := strings.Cut(model, "/")
if !ok || !stage.ValidModel(provider, id) {
t.Fatal("explicit valid PIPELINES_REFERENCE_MODEL required")
}
base := os.Getenv("PIPELINES_REFERENCE_DIR")
if base == "" {
t.Fatal("PIPELINES_REFERENCE_DIR required for retained private artifacts")
}
root, err := os.MkdirTemp(base, "pipelines-reference-")
if err != nil {
t.Fatal(err)
}
t.Log("Retained reference artifacts:", root)
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
defer cancel()
store, err := pipeline.Open(filepath.Join(root, "runs.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
runner, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); runner.wg.Wait() }()
var alerts atomic.Int32
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
alerts.Add(1)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"id":"reference-local-receipt","event":"message"}`)
}))
defer endpoint.Close()
t.Setenv("PIPELINES_NTFY_URL", endpoint.URL+"/reference")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
type result struct {
Name string `json:"name"`
Verified bool `json:"verified"`
RunID int64 `json:"run_id"`
Status string `json:"status"`
Duration string `json:"duration"`
Diagnostic string `json:"diagnostic,omitempty"`
}
report := struct {
Model string `json:"model"`
NotificationScope string `json:"notification_scope"`
Results []result `json:"results"`
}{Model: model, NotificationScope: "isolated local HTTP endpoint; phone reception not tested", Results: []result{}}
write := func(name string, v any) {
t.Helper()
data, err := json.MarshalIndent(v, "", " ")
if err != nil {
t.Fatal(err)
}
if err = os.WriteFile(filepath.Join(root, name), data, 0600); err != nil {
t.Fatal(err)
}
}
defer func() { write("report.json", report) }()
for _, c := range referenceCases(model) {
c := c
t.Run(c.name, func(t *testing.T) {
started := time.Now()
row := result{Name: c.name}
defer func() {
row.Verified = !t.Failed()
if !row.Verified && row.Diagnostic == "" {
row.Diagnostic = "Independent acceptance checks failed; inspect the test log and retained run/session JSON."
}
row.Duration = time.Since(started).String()
report.Results = append(report.Results, row)
write("report.json", report)
}()
workspace := filepath.Join(root, c.name)
if err := os.Mkdir(workspace, 0700); err != nil {
t.Fatal(err)
}
for name, text := range c.files {
if err := os.WriteFile(filepath.Join(workspace, name), []byte(text), 0600); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(root, c.name+".md"), []byte(c.source), 0600); err != nil {
t.Fatal(err)
}
if _, err := store.Submit(ctx, c.name, c.source, true); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
var pid int64
for _, p := range items {
if p.Name == c.name {
pid = p.ID
}
}
priorAlerts := alerts.Load()
run, err := runner.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: c.name, Workspace: workspace, Input: map[string]any{"prompt": c.prompt}})
if err != nil {
row.Diagnostic = err.Error()
t.Fatal(err)
}
runner.wg.Wait()
run, err = store.GetRun(context.Background(), run.ID)
if err != nil {
t.Fatal(err)
}
row.RunID = run.ID
row.Status = run.Status
row.Diagnostic = run.Diagnostic
write(c.name+".run.json", run)
evidence, evidenceErr := runner.evidence(ctx, run.ID)
write(c.name+".sessions.json", evidence)
// Persist failure evidence before asserting workflow-specific results.
c.check(t, run, workspace)
if evidenceErr != nil || len(evidence) != len(run.Steps) {
t.Fatal("session readback incomplete", evidenceErr)
}
for _, item := range evidence {
if item.Diagnostic != "" || len(item.Data) == 0 {
t.Fatal("retained session readback failed", item.Diagnostic)
}
}
for _, step := range run.Steps {
if step.SessionID == "" || step.AgentActive {
t.Fatal("stage session absent or still active", step)
}
if step.FileChanges == nil {
t.Fatal("workspace capture absent")
}
}
wantAlerts := int32(0)
if c.name == "incident-handoff" || c.name == "missing-input-failure" {
wantAlerts = 1
}
if alerts.Load()-priorAlerts != wantAlerts {
t.Fatal("unexpected notification count")
}
t.Logf("Verified %s: run #%d status=%s visits=%d", c.name, run.ID, run.Status, len(run.Steps))
})
}
}
// Re-check copied artifacts without additional inference or changing their DB.
func TestReferenceArtifactsReview(t *testing.T) {
root := os.Getenv("PIPELINES_REFERENCE_REVIEW_DIR")
if root == "" {
t.Skip("set PIPELINES_REFERENCE_REVIEW_DIR for independent retained-artifact checks (no inference)")
}
for _, c := range referenceCases("opencode-go/deepseek-v4.1-flash") {
c := c
t.Run(c.name, func(t *testing.T) {
data, err := os.ReadFile(filepath.Join(root, c.name+".run.json"))
if os.IsNotExist(err) {
t.Skip("case not present in this artifact directory")
}
if err != nil {
t.Fatal(err)
}
var run protocol.RunDetail
if err = json.Unmarshal(data, &run); err != nil {
t.Fatal(err)
}
c.check(t, &run, filepath.Join(root, c.name))
})
}
}
func TestReferenceToolEvidenceRejectsClaims(t *testing.T) {
if referenceToolFound(`{"data":[{"type":"text","text":"I used read and edit successfully"}]}`, "edit", false) {
t.Fatal("model prose accepted as tool evidence")
}
data := `{"data":[{"content":[{"tool":"read","state":{"status":"error"}}]}]}`
if referenceToolFound(data, "read", false) || !referenceToolFound(data, "read", true) {
t.Fatal("read error confused with successful read")
}
}