97 lines
4.7 KiB
Go
97 lines
4.7 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// Explicitly opt in: this uses existing native-provider credentials, makes ten
|
|
// small model calls, and retains all stage/probe sessions. No tools are enabled.
|
|
func TestNativeMathDemoLive(t *testing.T) {
|
|
executable := os.Getenv("PIPELINES_TEST_OPENCODE_V1")
|
|
if executable == "" || os.Getenv("PIPELINES_TEST_NATIVE_INFERENCE") != "1" {
|
|
t.Skip("set PIPELINES_TEST_OPENCODE_V1 and PIPELINES_TEST_NATIVE_INFERENCE=1 for real native math inference")
|
|
}
|
|
t.Setenv("PIPELINES_OPENCODE", executable)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
|
defer cancel()
|
|
directory := t.TempDir()
|
|
store, err := pipeline.Open(filepath.Join(directory, "math.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
var source strings.Builder
|
|
add := func(key, output, prompt, edges string) {
|
|
fmt.Fprintf(&source, "# stage %s\n\n```output\n%s\n```\n\n```prompt\n%s\n```\n\n```tools\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n```edges\n%s\nerror -> #failure\n```\n\n", key, output, prompt, edges)
|
|
}
|
|
contract := "Return only the small output map {a,b,value,steps}; value is an integer, steps an array of equation strings."
|
|
add("sum", contract, "Parse input.prompt as a + b. Compute value=a+b and steps=[\"a + b = value\"] with actual numbers. Preserve a and b.", "success -> multiply")
|
|
add("multiply", contract, "Preserve a,b and steps. Set value=input.value*3. Append the decimal input.value, followed by ' * 3 = ', followed by the decimal new value, as one equation string. For example, input.value=9 yields '9 * 3 = 27'. Never use placeholder words in equations.", "success -> parity")
|
|
add("parity", contract, "Do not change any output fields. Select outcome even when input.value is even, odd when it is odd.", "even -> halve\nodd -> add_one")
|
|
add("halve", contract, "Preserve a,b and steps. Set value=input.value/2. Append the decimal input.value, ' / 2 = ', the decimal new value, and ' (even branch)' as one equation string. For example, input.value=30 yields '30 / 2 = 15 (even branch)'. Never use placeholder words in equations.", "success -> report")
|
|
add("add_one", contract, "Preserve a,b and steps. Set value=input.value+1. Append the decimal input.value, ' + 1 = ', the decimal new value, and ' (odd branch)' as one equation string. For example, input.value=27 yields '27 + 1 = 28 (odd branch)'. Never use placeholder words in equations.", "success -> report")
|
|
add("report", "Return only {a,b,value,steps,summary}.", "Preserve a,b,value,steps. Set summary to the exact equation strings in input.steps joined by \"; \".", "success -> #success")
|
|
if _, err = store.Submit(ctx, "native math regression", source.String()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.wg.Wait() }()
|
|
for _, test := range []struct {
|
|
input, value, summary string
|
|
path []string
|
|
}{
|
|
{"7 + 2", "28", "7 + 2 = 9; 9 * 3 = 27; 27 + 1 = 28 (odd branch)", []string{"sum", "multiply", "parity", "add_one", "report"}},
|
|
{"7 + 3", "15", "7 + 3 = 10; 10 * 3 = 30; 30 / 2 = 15 (even branch)", []string{"sum", "multiply", "parity", "halve", "report"}},
|
|
} {
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "native-math-" + test.value, Workspace: directory, Input: map[string]any{"prompt": test.input}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
run, err = store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if run.Status != "succeeded" || fmt.Sprint(run.Output["value"]) != test.value || run.Output["summary"] != test.summary {
|
|
t.Fatalf("native math failed: input=%s status=%s diagnostic=%s output=%v", test.input, run.Status, run.Diagnostic, run.Output)
|
|
}
|
|
var path []string
|
|
for _, step := range run.Steps {
|
|
path = append(path, step.Stage)
|
|
if step.SessionID == "" || step.AgentActive {
|
|
t.Fatal("session not retained/stopped", step)
|
|
}
|
|
}
|
|
if !reflect.DeepEqual(path, test.path) {
|
|
t.Fatal("unexpected stage replay/path", path, test.path)
|
|
}
|
|
evidence, err := r.evidence(ctx, run.ID)
|
|
if err != nil || len(evidence) == 0 {
|
|
t.Fatal("live session evidence unavailable", err)
|
|
}
|
|
for _, item := range evidence {
|
|
if item.Diagnostic != "" {
|
|
t.Fatal("native session readback failed", item.Diagnostic)
|
|
}
|
|
}
|
|
t.Logf("PASS input=%s result=%s path=%v summary=%s; native session readback passed", test.input, test.value, path, test.summary)
|
|
}
|
|
}
|