Files
pipelines/internal/server/native_inference_test.go
T

97 lines
4.7 KiB
Go

package server
import (
"context"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Explicitly opt in: this uses existing native-provider credentials, makes ten
// small model calls, and retains all stage/probe sessions. No tools are enabled.
func TestNativeMathDemoLive(t *testing.T) {
executable := os.Getenv("PIPELINES_TEST_OPENCODE_V1")
if executable == "" || os.Getenv("PIPELINES_TEST_NATIVE_INFERENCE") != "1" {
t.Skip("set PIPELINES_TEST_OPENCODE_V1 and PIPELINES_TEST_NATIVE_INFERENCE=1 for real native math inference")
}
t.Setenv("PIPELINES_OPENCODE", executable)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
directory := t.TempDir()
store, err := pipeline.Open(filepath.Join(directory, "math.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
var source strings.Builder
add := func(key, output, prompt, edges string) {
fmt.Fprintf(&source, "# stage %s\n\n```output\n%s\n```\n\n```prompt\n%s\n```\n\n```tools\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n```edges\n%s\nerror -> #failure\n```\n\n", key, output, prompt, edges)
}
contract := "Return only the small output map {a,b,value,steps}; value is an integer, steps an array of equation strings."
add("sum", contract, "Parse input.prompt as a + b. Compute value=a+b and steps=[\"a + b = value\"] with actual numbers. Preserve a and b.", "success -> multiply")
add("multiply", contract, "Preserve a,b and steps. Set value=input.value*3. Append the decimal input.value, followed by ' * 3 = ', followed by the decimal new value, as one equation string. For example, input.value=9 yields '9 * 3 = 27'. Never use placeholder words in equations.", "success -> parity")
add("parity", contract, "Do not change any output fields. Select outcome even when input.value is even, odd when it is odd.", "even -> halve\nodd -> add_one")
add("halve", contract, "Preserve a,b and steps. Set value=input.value/2. Append the decimal input.value, ' / 2 = ', the decimal new value, and ' (even branch)' as one equation string. For example, input.value=30 yields '30 / 2 = 15 (even branch)'. Never use placeholder words in equations.", "success -> report")
add("add_one", contract, "Preserve a,b and steps. Set value=input.value+1. Append the decimal input.value, ' + 1 = ', the decimal new value, and ' (odd branch)' as one equation string. For example, input.value=27 yields '27 + 1 = 28 (odd branch)'. Never use placeholder words in equations.", "success -> report")
add("report", "Return only {a,b,value,steps,summary}.", "Preserve a,b,value,steps. Set summary to the exact equation strings in input.steps joined by \"; \".", "success -> #success")
if _, err = store.Submit(ctx, "native math regression", source.String()); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait() }()
for _, test := range []struct {
input, value, summary string
path []string
}{
{"7 + 2", "28", "7 + 2 = 9; 9 * 3 = 27; 27 + 1 = 28 (odd branch)", []string{"sum", "multiply", "parity", "add_one", "report"}},
{"7 + 3", "15", "7 + 3 = 10; 10 * 3 = 30; 30 / 2 = 15 (even branch)", []string{"sum", "multiply", "parity", "halve", "report"}},
} {
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "native-math-" + test.value, Workspace: directory, Input: map[string]any{"prompt": test.input}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
run, err = store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if run.Status != "succeeded" || fmt.Sprint(run.Output["value"]) != test.value || run.Output["summary"] != test.summary {
t.Fatalf("native math failed: input=%s status=%s diagnostic=%s output=%v", test.input, run.Status, run.Diagnostic, run.Output)
}
var path []string
for _, step := range run.Steps {
path = append(path, step.Stage)
if step.SessionID == "" || step.AgentActive {
t.Fatal("session not retained/stopped", step)
}
}
if !reflect.DeepEqual(path, test.path) {
t.Fatal("unexpected stage replay/path", path, test.path)
}
evidence, err := r.evidence(ctx, run.ID)
if err != nil || len(evidence) == 0 {
t.Fatal("live session evidence unavailable", err)
}
for _, item := range evidence {
if item.Diagnostic != "" {
t.Fatal("native session readback failed", item.Diagnostic)
}
}
t.Logf("PASS input=%s result=%s path=%v summary=%s; native session readback passed", test.input, test.value, path, test.summary)
}
}