Files
pipelines/internal/opencode/format_test.go
T

93 lines
4.2 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package opencode
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestNativeFormatProbeDowngradesOnlyReadbackIncompatibility(t *testing.T) {
for _, status := range []int{200, 400, 401, 500} {
t.Run(http.StatusText(status), func(t *testing.T) {
probes, generations := 0, 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.Method + " " + r.URL.Path {
case "POST /session":
probes++
var body struct{ Permission []map[string]string }
json.NewDecoder(r.Body).Decode(&body)
if len(body.Permission) != 1 || body.Permission[0]["action"] != "deny" {
t.Fatal("probe has tools", body)
}
w.Write([]byte(`{"id":"ses_probe"}`))
case "POST /session/ses_probe/message":
var body struct {
NoReply bool `json:"noReply"`
}
json.NewDecoder(r.Body).Decode(&body)
if !body.NoReply {
t.Fatal("probe requested inference")
}
w.Write([]byte(`{}`))
case "GET /session/ses_probe/message":
w.WriteHeader(status)
if status == 200 {
w.Write([]byte(`[{"info":{"role":"user","format":{"type":"json_schema"}}}]`))
} else {
w.Write([]byte(`{"name":"BadRequest","data":{"message":"Expected OutputFormatJsonSchema at info.format"}}`))
}
case "POST /session/ses_primary/message":
generations++
var body map[string]json.RawMessage
json.NewDecoder(r.Body).Decode(&body)
if (len(body["format"]) > 0) != (status == 200) {
t.Error("incorrect format compatibility mode", string(body["format"]))
}
w.Write([]byte(`{"info":{"role":"assistant","finish":"stop","time":{"completed":1},"structured":{"outcome":"success","output":{"value":9}}},"parts":[]}`))
default:
t.Error("unexpected operation", r.Method, r.URL.Path)
}
}))
defer srv.Close()
c := &Connection{Backend: "v1", URL: srv.URL, models: map[string]map[string]string{"ses_primary": {"providerID": "test", "modelID": "test"}}, responses: map[string][]byte{}}
body := map[string]any{"prompt": "task", "format": map[string]any{"type": "json_schema", "schema": map[string]any{"type": "object"}, "retryCount": 0}}
data, err := c.Call(context.Background(), "post", "/api/session/ses_primary/generate", body)
if status == 401 || status == 500 {
if err == nil || generations != 0 {
t.Fatal("unrelated failure silently downgraded", err, generations)
}
return
}
if err != nil || !strings.Contains(string(data), "value") || generations != 1 {
t.Fatal(err, string(data), generations)
}
if c.FormatProbe == nil || c.FormatProbe.SessionID != "ses_probe" || c.FormatProbe.Supported != (status == 200) {
t.Fatal("probe evidence lost", c.FormatProbe)
}
if _, err = c.Call(context.Background(), "post", "/api/session/ses_primary/generate", body); err != nil || probes != 1 {
t.Fatal("capability probe was not cached", err, probes)
}
})
}
}
func TestNativeStructuredFailureIsClearAndDoesNotReplayGeneration(t *testing.T) {
response := []byte(`{"info":{"role":"assistant","finish":"stop","time":{"completed":1},"error":{"name":"StructuredOutputError","data":{"message":"Model did not produce structured output; secret-token"}}},"parts":[{"type":"text","text":"<|DSML|invoke name=\"StructuredOutput\">not JSON</|DSML|invoke>"}]}`)
_, err := messageText(response)
if err == nil || !strings.Contains(err.Error(), "StructuredOutputError: Model did not produce structured output") || strings.Contains(err.Error(), "secret-token") {
t.Fatal(err)
}
if _, partialErr := messageText([]byte(strings.Replace(string(response), `"completed":1`, `"completed":0`, 1))); partialErr == nil || !strings.Contains(partialErr.Error(), "StructuredOutputError") {
t.Fatal("incomplete timestamp hid provider failure", partialErr)
}
c := &Connection{Backend: "v1", responses: map[string][]byte{"ses_failure": response}}
retained, err := c.Call(context.Background(), "post", "/api/experimental/session/ses_failure/wait", nil)
if err == nil || string(retained) != string(response) {
t.Fatal("wait failure discarded provider response", err, string(retained))
}
}