c42a19f7d3
Make OpenCode authoring responsive and expand the managed skill references. Add conflict-aware import/export, real-provider reference workflows, and tracked V2 generation for reliable budget accounting.
376 lines
15 KiB
Go
376 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
type referenceCase struct {
|
|
name, source, prompt string
|
|
files map[string]string
|
|
check func(*testing.T, *protocol.RunDetail, string)
|
|
}
|
|
|
|
func referenceCases(model string) []referenceCase {
|
|
provider, id, _ := strings.Cut(model, "/")
|
|
block := func(key, output, prompt, tools, edges, extra string) string {
|
|
return fmt.Sprintf("# stage %s\n\n```output\n%s\n```\n\n```prompt\n%s\n```\n\n```tools\n%s\n```\n\n```configs\nprovider=%s\nmodel=%s\nretries=0\ntimeout=2m\n%s\n```\n\n```edges\n%s\nerror -> #failure\n```\n\n", key, output, prompt, tools, provider, id, extra, edges)
|
|
}
|
|
hook := "\n```notify\non=success,failure\n```\n"
|
|
triage := block("triage", "{ issues }", "Extract the two incident records from input.prompt into issues, an array of objects with id and severity. Preserve the literal IDs/severity. No tools. Return success.", "", "success -> report", "") +
|
|
block("report", "{ issues, total, high_count, summary }", "Preserve input.issues unchanged. Set total to its length and high_count to the number with severity high. Set summary to exactly: 2 incidents; 1 high priority. Return success. No tools.", "", "success -> #success", "") + hook
|
|
edit := block("edit", "{ path, summary }", "Read release.txt. Change only status=pending to status=ready and version=1 to version=2 using the edit tool. Preserve the owner line and trailing newline. Do not change any other file. Return success with path release.txt and a short summary of the actual edit.", "read\nedit", "success -> verify", "") +
|
|
"```permissions\n{\"edit\":[\"release.txt\"]}\n```\n" +
|
|
block("verify", "{ path, verified, summary }", "Use the shell tool to execute exactly cmp release.txt expected.txt in the run directory. Return success only if it exits zero, with verified=true and path=release.txt. Otherwise return error. Do not edit files. Do not treat the preceding stage's summary as evidence.", "read\nshell", "success -> #success", "") +
|
|
"```permissions\n{\"shell\":[\"cmp release.txt expected.txt\"]}\n```\n"
|
|
blocked := block("inspect", "{ reason }", "Use read to check required.txt. It is deliberately missing. If unavailable return blocked with reason required.txt is missing. Do not create files, run shell commands, or invent evidence.", "read", "success -> #success\nblocked -> #failure", "") + hook
|
|
budget := block("budget", "{ summary }", "Return success with summary exactly: accounting check. No tools.", "", "success -> #success", "budget_tokens=1\nbudget_usd=0.25")
|
|
return []referenceCase{
|
|
{name: "incident-handoff", source: triage, prompt: "INC-101: login failures; severity=high. INC-102: typo in help text; severity=low.", check: func(t *testing.T, r *protocol.RunDetail, _ string) {
|
|
if r.Status != "succeeded" || len(r.Steps) != 2 || fmt.Sprint(r.Output["total"]) != "2" || fmt.Sprint(r.Output["high_count"]) != "1" || r.Output["summary"] != "2 incidents; 1 high priority." {
|
|
t.Fatalf("handoff result incorrect: %+v", r)
|
|
}
|
|
issues, ok := r.Output["issues"].([]any)
|
|
if !ok || len(issues) != 2 {
|
|
t.Fatal("incident evidence missing", r.Output)
|
|
}
|
|
found := map[string]string{}
|
|
for _, v := range issues {
|
|
m, ok := v.(map[string]any)
|
|
if !ok {
|
|
t.Fatal(v)
|
|
}
|
|
found[fmt.Sprint(m["id"])] = fmt.Sprint(m["severity"])
|
|
}
|
|
if found["INC-101"] != "high" || found["INC-102"] != "low" {
|
|
t.Fatal("incident evidence altered", found)
|
|
}
|
|
first, _ := json.Marshal(r.Steps[0].Output)
|
|
second, _ := json.Marshal(r.Steps[1].Input)
|
|
if string(first) != string(second) {
|
|
t.Fatal("handoff changed", string(first), string(second))
|
|
}
|
|
assertReferenceAlert(t, r.Steps[1], "success")
|
|
}},
|
|
{name: "edit-and-verify", source: edit, prompt: "Prepare this release record and independently verify it.", files: map[string]string{"release.txt": "status=pending\nversion=1\nowner=team\n", "expected.txt": "status=ready\nversion=2\nowner=team\n", "sentinel.txt": "leave untouched\n"}, check: func(t *testing.T, r *protocol.RunDetail, dir string) {
|
|
if r.Status != "succeeded" || len(r.Steps) != 2 || r.Output["verified"] != true {
|
|
t.Fatalf("edit/verification failed: %+v", r)
|
|
}
|
|
expected := "status=ready\nversion=2\nowner=team\n"
|
|
for _, name := range []string{"release.txt", "expected.txt"} {
|
|
data, err := os.ReadFile(filepath.Join(dir, name))
|
|
if err != nil || string(data) != expected {
|
|
t.Fatal("fixture verification failed", name, string(data), err)
|
|
}
|
|
}
|
|
data, _ := os.ReadFile(filepath.Join(dir, "sentinel.txt"))
|
|
if string(data) != "leave untouched\n" {
|
|
t.Fatal("unrelated fixture changed")
|
|
}
|
|
cmd := exec.Command("cmp", "release.txt", "expected.txt")
|
|
cmd.Dir = dir
|
|
if data, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatal("independent cmp failed", err, string(data))
|
|
}
|
|
c := r.Steps[0].FileChanges
|
|
if c == nil || c.Status != "complete" || len(c.Files) != 1 || c.Files[0].Path != "release.txt" || !strings.Contains(c.Files[0].Patch, "+status=ready") {
|
|
t.Fatal("edit diff missing", c)
|
|
}
|
|
c = r.Steps[1].FileChanges
|
|
if c == nil || c.Status != "complete" || len(c.Files) != 0 {
|
|
t.Fatal("verifier edited files", c)
|
|
}
|
|
assertReferenceTool(t, r.Steps[0], "read")
|
|
assertReferenceTool(t, r.Steps[0], "edit")
|
|
assertReferenceTool(t, r.Steps[1], "shell")
|
|
}},
|
|
{name: "missing-input-failure", source: blocked, prompt: "Inspect the required input; do not fabricate a result.", check: func(t *testing.T, r *protocol.RunDetail, dir string) {
|
|
if r.Status != "failed" || len(r.Steps) != 1 || r.Steps[0].Outcome != "blocked" || r.Steps[0].Output["reason"] != "required.txt is missing" {
|
|
t.Fatalf("failure route incorrect: %+v", r)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "required.txt")); !os.IsNotExist(err) {
|
|
t.Fatal("missing input was invented")
|
|
}
|
|
assertReferenceTool(t, r.Steps[0], "read")
|
|
assertReferenceAlert(t, r.Steps[0], "failure")
|
|
}},
|
|
{name: "accounting-and-budget", source: budget, prompt: "Return the requested short accounting check.", check: func(t *testing.T, r *protocol.RunDetail, _ string) {
|
|
if r.Status != "failed" || len(r.Steps) != 1 || !strings.Contains(r.Steps[0].Diagnostic, stage.ErrBudgetExceeded.Error()) {
|
|
t.Fatalf("budget enforcement not confirmed: %+v", r)
|
|
}
|
|
var audit struct {
|
|
Limits struct {
|
|
Budget stage.BudgetSnapshot `json:"budget"`
|
|
} `json:"stage_limits"`
|
|
}
|
|
if err := json.Unmarshal([]byte(r.Steps[0].Transcript), &audit); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b := audit.Limits.Budget
|
|
if !b.TokensKnown || !b.USDKnown || b.UsedTokens < 1 || b.Attempts != 1 {
|
|
t.Fatal("complete reported accounting not retained", b)
|
|
}
|
|
t.Logf("Reported usage: tokens=%d USD=%.8f (not an independently reconciled bill)", b.UsedTokens, b.UsedUSD)
|
|
}},
|
|
}
|
|
}
|
|
|
|
func assertReferenceAlert(t *testing.T, step protocol.RunStep, event string) {
|
|
t.Helper()
|
|
if len(step.Notifications) != 1 || step.Notifications[0].Status != "sent" || step.Notifications[0].MessageID != "reference-local-receipt" {
|
|
t.Fatal("local notification receipt missing", step.Notifications)
|
|
}
|
|
}
|
|
|
|
func referenceToolFound(text, tool string, allowReadError bool) bool {
|
|
var value any
|
|
if json.Unmarshal([]byte(text), &value) != nil {
|
|
return false
|
|
}
|
|
found := false
|
|
var visit func(any)
|
|
visit = func(v any) {
|
|
switch x := v.(type) {
|
|
case map[string]any:
|
|
name, _ := x["tool"].(string)
|
|
if name == "" {
|
|
name, _ = x["name"].(string)
|
|
}
|
|
status, _ := x["status"].(string)
|
|
if state, ok := x["state"].(map[string]any); ok {
|
|
status, _ = state["status"].(string)
|
|
}
|
|
// Failed read is valid evidence for deliberately missing input, not success.
|
|
if name == tool && (status == "completed" || status == "success" || allowReadError && tool == "read" && status == "error") {
|
|
found = true
|
|
}
|
|
for _, child := range x {
|
|
visit(child)
|
|
}
|
|
case []any:
|
|
for _, child := range x {
|
|
visit(child)
|
|
}
|
|
}
|
|
}
|
|
visit(value)
|
|
return found
|
|
}
|
|
func assertReferenceTool(t *testing.T, step protocol.RunStep, tool string) {
|
|
t.Helper()
|
|
if !referenceToolFound(step.Transcript, tool, step.Outcome == "blocked") {
|
|
t.Fatalf("no retained terminal %s tool evidence in stage %s", tool, step.Stage)
|
|
}
|
|
}
|
|
|
|
func TestReferenceDefinitions(t *testing.T) {
|
|
for _, c := range referenceCases("opencode-go/deepseek-v4.1-flash") {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
if _, err := pipeline.Parse(c.source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Explicit opt-in; uses real credentials/inference. Artifacts and sessions are retained.
|
|
// Six stage attempts plus backend compatibility/format-repair calls may incur charges.
|
|
func TestReferenceWorkflowsLive(t *testing.T) {
|
|
if os.Getenv("PIPELINES_REFERENCE_LIVE") != "1" {
|
|
t.Skip("set PIPELINES_REFERENCE_LIVE=1, PIPELINES_REFERENCE_MODEL and PIPELINES_REFERENCE_DIR for chargeable reference inference")
|
|
}
|
|
model := os.Getenv("PIPELINES_REFERENCE_MODEL")
|
|
provider, id, ok := strings.Cut(model, "/")
|
|
if !ok || !stage.ValidModel(provider, id) {
|
|
t.Fatal("explicit valid PIPELINES_REFERENCE_MODEL required")
|
|
}
|
|
base := os.Getenv("PIPELINES_REFERENCE_DIR")
|
|
if base == "" {
|
|
t.Fatal("PIPELINES_REFERENCE_DIR required for retained private artifacts")
|
|
}
|
|
root, err := os.MkdirTemp(base, "pipelines-reference-")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Log("Retained reference artifacts:", root)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(root, "runs.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runner, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); runner.wg.Wait() }()
|
|
var alerts atomic.Int32
|
|
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
alerts.Add(1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprint(w, `{"id":"reference-local-receipt","event":"message"}`)
|
|
}))
|
|
defer endpoint.Close()
|
|
t.Setenv("PIPELINES_NTFY_URL", endpoint.URL+"/reference")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
|
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
|
type result struct {
|
|
Name string `json:"name"`
|
|
Verified bool `json:"verified"`
|
|
RunID int64 `json:"run_id"`
|
|
Status string `json:"status"`
|
|
Duration string `json:"duration"`
|
|
Diagnostic string `json:"diagnostic,omitempty"`
|
|
}
|
|
report := struct {
|
|
Model string `json:"model"`
|
|
NotificationScope string `json:"notification_scope"`
|
|
Results []result `json:"results"`
|
|
}{Model: model, NotificationScope: "isolated local HTTP endpoint; phone reception not tested", Results: []result{}}
|
|
write := func(name string, v any) {
|
|
t.Helper()
|
|
data, err := json.MarshalIndent(v, "", " ")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.WriteFile(filepath.Join(root, name), data, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
defer func() { write("report.json", report) }()
|
|
for _, c := range referenceCases(model) {
|
|
c := c
|
|
t.Run(c.name, func(t *testing.T) {
|
|
started := time.Now()
|
|
row := result{Name: c.name}
|
|
defer func() {
|
|
row.Verified = !t.Failed()
|
|
if !row.Verified && row.Diagnostic == "" {
|
|
row.Diagnostic = "Independent acceptance checks failed; inspect the test log and retained run/session JSON."
|
|
}
|
|
row.Duration = time.Since(started).String()
|
|
report.Results = append(report.Results, row)
|
|
write("report.json", report)
|
|
}()
|
|
workspace := filepath.Join(root, c.name)
|
|
if err := os.Mkdir(workspace, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, text := range c.files {
|
|
if err := os.WriteFile(filepath.Join(workspace, name), []byte(text), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := os.WriteFile(filepath.Join(root, c.name+".md"), []byte(c.source), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, c.name, c.source, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var pid int64
|
|
for _, p := range items {
|
|
if p.Name == c.name {
|
|
pid = p.ID
|
|
}
|
|
}
|
|
priorAlerts := alerts.Load()
|
|
run, err := runner.start(protocol.Message{PipelineID: pid, Version: 1, RequestKey: c.name, Workspace: workspace, Input: map[string]any{"prompt": c.prompt}})
|
|
if err != nil {
|
|
row.Diagnostic = err.Error()
|
|
t.Fatal(err)
|
|
}
|
|
runner.wg.Wait()
|
|
run, err = store.GetRun(context.Background(), run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
row.RunID = run.ID
|
|
row.Status = run.Status
|
|
row.Diagnostic = run.Diagnostic
|
|
write(c.name+".run.json", run)
|
|
evidence, evidenceErr := runner.evidence(ctx, run.ID)
|
|
write(c.name+".sessions.json", evidence)
|
|
// Persist failure evidence before asserting workflow-specific results.
|
|
c.check(t, run, workspace)
|
|
if evidenceErr != nil || len(evidence) != len(run.Steps) {
|
|
t.Fatal("session readback incomplete", evidenceErr)
|
|
}
|
|
for _, item := range evidence {
|
|
if item.Diagnostic != "" || len(item.Data) == 0 {
|
|
t.Fatal("retained session readback failed", item.Diagnostic)
|
|
}
|
|
}
|
|
for _, step := range run.Steps {
|
|
if step.SessionID == "" || step.AgentActive {
|
|
t.Fatal("stage session absent or still active", step)
|
|
}
|
|
if step.FileChanges == nil {
|
|
t.Fatal("workspace capture absent")
|
|
}
|
|
}
|
|
wantAlerts := int32(0)
|
|
if c.name == "incident-handoff" || c.name == "missing-input-failure" {
|
|
wantAlerts = 1
|
|
}
|
|
if alerts.Load()-priorAlerts != wantAlerts {
|
|
t.Fatal("unexpected notification count")
|
|
}
|
|
t.Logf("Verified %s: run #%d status=%s visits=%d", c.name, run.ID, run.Status, len(run.Steps))
|
|
})
|
|
}
|
|
}
|
|
|
|
// Re-check copied artifacts without additional inference or changing their DB.
|
|
func TestReferenceArtifactsReview(t *testing.T) {
|
|
root := os.Getenv("PIPELINES_REFERENCE_REVIEW_DIR")
|
|
if root == "" {
|
|
t.Skip("set PIPELINES_REFERENCE_REVIEW_DIR for independent retained-artifact checks (no inference)")
|
|
}
|
|
for _, c := range referenceCases("opencode-go/deepseek-v4.1-flash") {
|
|
c := c
|
|
t.Run(c.name, func(t *testing.T) {
|
|
data, err := os.ReadFile(filepath.Join(root, c.name+".run.json"))
|
|
if os.IsNotExist(err) {
|
|
t.Skip("case not present in this artifact directory")
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var run protocol.RunDetail
|
|
if err = json.Unmarshal(data, &run); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c.check(t, &run, filepath.Join(root, c.name))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReferenceToolEvidenceRejectsClaims(t *testing.T) {
|
|
if referenceToolFound(`{"data":[{"type":"text","text":"I used read and edit successfully"}]}`, "edit", false) {
|
|
t.Fatal("model prose accepted as tool evidence")
|
|
}
|
|
data := `{"data":[{"content":[{"tool":"read","state":{"status":"error"}}]}]}`
|
|
if referenceToolFound(data, "read", false) || !referenceToolFound(data, "read", true) {
|
|
t.Fatal("read error confused with successful read")
|
|
}
|
|
}
|