package server import ( "context" "fmt" "os" "path/filepath" "reflect" "strings" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" ) // Explicitly opt in: this uses existing native-provider credentials, makes ten // small model calls, and retains all stage/probe sessions. No tools are enabled. func TestNativeMathDemoLive(t *testing.T) { executable := os.Getenv("PIPELINES_TEST_OPENCODE_V1") if executable == "" || os.Getenv("PIPELINES_TEST_NATIVE_INFERENCE") != "1" { t.Skip("set PIPELINES_TEST_OPENCODE_V1 and PIPELINES_TEST_NATIVE_INFERENCE=1 for real native math inference") } t.Setenv("PIPELINES_OPENCODE", executable) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() directory := t.TempDir() store, err := pipeline.Open(filepath.Join(directory, "math.db")) if err != nil { t.Fatal(err) } defer store.Close() var source strings.Builder add := func(key, output, prompt, edges string) { fmt.Fprintf(&source, "# stage %s\n\n```output\n%s\n```\n\n```prompt\n%s\n```\n\n```tools\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n```edges\n%s\nerror -> #failure\n```\n\n", key, output, prompt, edges) } contract := "Return only the small output map {a,b,value,steps}; value is an integer, steps an array of equation strings." add("sum", contract, "Parse input.prompt as a + b. Compute value=a+b and steps=[\"a + b = value\"] with actual numbers. Preserve a and b.", "success -> multiply") add("multiply", contract, "Preserve a,b and steps. Set value=input.value*3. Append the decimal input.value, followed by ' * 3 = ', followed by the decimal new value, as one equation string. For example, input.value=9 yields '9 * 3 = 27'. Never use placeholder words in equations.", "success -> parity") add("parity", contract, "Do not change any output fields. Select outcome even when input.value is even, odd when it is odd.", "even -> halve\nodd -> add_one") add("halve", contract, "Preserve a,b and steps. Set value=input.value/2. Append the decimal input.value, ' / 2 = ', the decimal new value, and ' (even branch)' as one equation string. For example, input.value=30 yields '30 / 2 = 15 (even branch)'. Never use placeholder words in equations.", "success -> report") add("add_one", contract, "Preserve a,b and steps. Set value=input.value+1. Append the decimal input.value, ' + 1 = ', the decimal new value, and ' (odd branch)' as one equation string. For example, input.value=27 yields '27 + 1 = 28 (odd branch)'. Never use placeholder words in equations.", "success -> report") add("report", "Return only {a,b,value,steps,summary}.", "Preserve a,b,value,steps. Set summary to the exact equation strings in input.steps joined by \"; \".", "success -> #success") if _, err = store.Submit(ctx, "native math regression", source.String()); err != nil { t.Fatal(err) } items, err := store.List(ctx) if err != nil { t.Fatal(err) } r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait() }() for _, test := range []struct { input, value, summary string path []string }{ {"7 + 2", "28", "7 + 2 = 9; 9 * 3 = 27; 27 + 1 = 28 (odd branch)", []string{"sum", "multiply", "parity", "add_one", "report"}}, {"7 + 3", "15", "7 + 3 = 10; 10 * 3 = 30; 30 / 2 = 15 (even branch)", []string{"sum", "multiply", "parity", "halve", "report"}}, } { run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "native-math-" + test.value, Workspace: directory, Input: map[string]any{"prompt": test.input}}) if err != nil { t.Fatal(err) } r.wg.Wait() run, err = store.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) } if run.Status != "succeeded" || fmt.Sprint(run.Output["value"]) != test.value || run.Output["summary"] != test.summary { t.Fatalf("native math failed: input=%s status=%s diagnostic=%s output=%v", test.input, run.Status, run.Diagnostic, run.Output) } var path []string for _, step := range run.Steps { path = append(path, step.Stage) if step.SessionID == "" || step.AgentActive { t.Fatal("session not retained/stopped", step) } } if !reflect.DeepEqual(path, test.path) { t.Fatal("unexpected stage replay/path", path, test.path) } evidence, err := r.evidence(ctx, run.ID) if err != nil || len(evidence) == 0 { t.Fatal("live session evidence unavailable", err) } for _, item := range evidence { if item.Diagnostic != "" { t.Fatal("native session readback failed", item.Diagnostic) } } t.Logf("PASS input=%s result=%s path=%v summary=%s; native session readback passed", test.input, test.value, path, test.summary) } }