Files
pipelines/internal/opencode/format_test.go
T
alan 7b72b8f20f stage: retry pre-admission harness reach failures and widen repair deadline
Treat a failure to reach the harness before any prompt or tool call is
admitted (connecting or creating the session) as a distinct retryable
reach failure, so a transport blip at the first stage no longer ends a
mutating run; keep post-admission readback/poll/agent failures retried
only on an explicit read-only stage and never replay admitted tool work.
Preserve the true cause when the context is already done (join ctx.Err
with the typed transport error) and report a sanitized unknown provider
error name/status instead of a blanket summary.

Make the isolated format-repair deadline proportional to the stage
timeout (floored, capped and clamped to remaining minus a margin) so a
slow-but-repairable result is recovered without ever extending the stage
deadline; the bounded ladder, value guard, deny-all formatter session,
cumulative usage accounting and no-tool-replay rule are unchanged.

Update docs/model-results.md, docs/pipeline-performance-review.md and the
packaged skill references; add stage, opencode and server fixtures.
2026-10-17 14:51:09 +01:00

116 lines
5.7 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package opencode
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestNativeFormatProbeDowngradesOnlyReadbackIncompatibility(t *testing.T) {
for _, status := range []int{200, 400, 401, 500} {
t.Run(http.StatusText(status), func(t *testing.T) {
probes, generations := 0, 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.Method + " " + r.URL.Path {
case "POST /session":
probes++
var body struct{ Permission []map[string]string }
json.NewDecoder(r.Body).Decode(&body)
if len(body.Permission) != 1 || body.Permission[0]["action"] != "deny" {
t.Fatal("probe has tools", body)
}
w.Write([]byte(`{"id":"ses_probe"}`))
case "POST /session/ses_probe/message":
var body struct {
NoReply bool `json:"noReply"`
}
json.NewDecoder(r.Body).Decode(&body)
if !body.NoReply {
t.Fatal("probe requested inference")
}
w.Write([]byte(`{}`))
case "GET /session/ses_probe/message":
w.WriteHeader(status)
if status == 200 {
w.Write([]byte(`[{"info":{"role":"user","format":{"type":"json_schema"}}}]`))
} else {
w.Write([]byte(`{"name":"BadRequest","data":{"message":"Expected OutputFormatJsonSchema at info.format"}}`))
}
case "POST /session/ses_primary/message":
generations++
var body map[string]json.RawMessage
json.NewDecoder(r.Body).Decode(&body)
if (len(body["format"]) > 0) != (status == 200) {
t.Error("incorrect format compatibility mode", string(body["format"]))
}
w.Write([]byte(`{"info":{"role":"assistant","finish":"stop","time":{"completed":1},"structured":{"outcome":"success","output":{"value":9}}},"parts":[]}`))
default:
t.Error("unexpected operation", r.Method, r.URL.Path)
}
}))
defer srv.Close()
c := &Connection{Backend: "v1", URL: srv.URL, models: map[string]map[string]string{"ses_primary": {"providerID": "test", "modelID": "test"}}, responses: map[string][]byte{}}
body := map[string]any{"prompt": "task", "format": map[string]any{"type": "json_schema", "schema": map[string]any{"type": "object"}, "retryCount": 0}}
data, err := c.Call(context.Background(), "post", "/api/session/ses_primary/generate", body)
if status == 401 || status == 500 {
if err == nil || generations != 0 {
t.Fatal("unrelated failure silently downgraded", err, generations)
}
return
}
if err != nil || !strings.Contains(string(data), "value") || generations != 1 {
t.Fatal(err, string(data), generations)
}
if c.FormatProbe == nil || c.FormatProbe.SessionID != "ses_probe" || c.FormatProbe.Supported != (status == 200) {
t.Fatal("probe evidence lost", c.FormatProbe)
}
if _, err = c.Call(context.Background(), "post", "/api/session/ses_primary/generate", body); err != nil || probes != 1 {
t.Fatal("capability probe was not cached", err, probes)
}
})
}
}
func TestNativeStructuredFailureIsClearAndDoesNotReplayGeneration(t *testing.T) {
response := []byte(`{"info":{"role":"assistant","finish":"stop","time":{"completed":1},"error":{"name":"StructuredOutputError","data":{"message":"Model did not produce structured output; secret-token"}}},"parts":[{"type":"text","text":"<|DSML|invoke name=\"StructuredOutput\">not JSON</|DSML|invoke>"}]}`)
_, err := messageText(response)
if err == nil || !strings.Contains(err.Error(), "StructuredOutputError: Model did not produce structured output") || strings.Contains(err.Error(), "secret-token") {
t.Fatal(err)
}
if _, partialErr := messageText([]byte(strings.Replace(string(response), `"completed":1`, `"completed":0`, 1))); partialErr == nil || !strings.Contains(partialErr.Error(), "StructuredOutputError") {
t.Fatal("incomplete timestamp hid provider failure", partialErr)
}
c := &Connection{Backend: "v1", responses: map[string][]byte{"ses_failure": response}}
retained, err := c.Call(context.Background(), "post", "/api/experimental/session/ses_failure/wait", nil)
if err == nil || string(retained) != string(response) {
t.Fatal("wait failure discarded provider response", err, string(retained))
}
}
// TestExecutionErrorReportsSanitizedNameAndStatusWithoutBody proves an
// unrecognized provider error name is reported as a bounded, safe identifier
// plus any HTTP status instead of a blanket summary, while no provider message,
// prompt or credential is copied and known names keep their exact wording.
func TestExecutionErrorReportsSanitizedNameAndStatusWithoutBody(t *testing.T) {
err := ExecutionError(json.RawMessage(`{"name":"TotallyUnknownError","data":{"statusCode":502,"message":"secret provider body"}}`))
if err == nil || !strings.Contains(err.Error(), "TotallyUnknownError") || !strings.Contains(err.Error(), "HTTP 502") {
t.Fatalf("unknown error name/status not reported: %v", err)
}
if strings.Contains(err.Error(), "secret provider body") {
t.Fatalf("diagnostic leaked the provider body: %v", err)
}
if err := ExecutionError(json.RawMessage(`{"_tag":"SomeTagError"}`)); err == nil || !strings.Contains(err.Error(), "SomeTagError") {
t.Fatalf("unknown tag not reported: %v", err)
}
if err := ExecutionError(json.RawMessage(`{"name":"Injected Error: Bearer secret-token"}`)); err == nil || strings.Contains(err.Error(), "secret-token") || strings.Contains(err.Error(), "Bearer") || err.Error() != "OpenCode agent reported an execution error" {
t.Fatalf("unsafe name was copied into the diagnostic: %v", err)
}
if err := ExecutionError(json.RawMessage(`{"name":"APIError","data":{"statusCode":500}}`)); err == nil || !strings.Contains(err.Error(), "APIError (HTTP 500)") {
t.Fatalf("known safe name changed: %v", err)
}
}