7b72b8f20f
Treat a failure to reach the harness before any prompt or tool call is admitted (connecting or creating the session) as a distinct retryable reach failure, so a transport blip at the first stage no longer ends a mutating run; keep post-admission readback/poll/agent failures retried only on an explicit read-only stage and never replay admitted tool work. Preserve the true cause when the context is already done (join ctx.Err with the typed transport error) and report a sanitized unknown provider error name/status instead of a blanket summary. Make the isolated format-repair deadline proportional to the stage timeout (floored, capped and clamped to remaining minus a margin) so a slow-but-repairable result is recovered without ever extending the stage deadline; the bounded ladder, value guard, deny-all formatter session, cumulative usage accounting and no-tool-replay rule are unchanged. Update docs/model-results.md, docs/pipeline-performance-review.md and the packaged skill references; add stage, opencode and server fixtures.
116 lines
5.7 KiB
Go
116 lines
5.7 KiB
Go
package opencode
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
func TestNativeFormatProbeDowngradesOnlyReadbackIncompatibility(t *testing.T) {
|
||
for _, status := range []int{200, 400, 401, 500} {
|
||
t.Run(http.StatusText(status), func(t *testing.T) {
|
||
probes, generations := 0, 0
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
switch r.Method + " " + r.URL.Path {
|
||
case "POST /session":
|
||
probes++
|
||
var body struct{ Permission []map[string]string }
|
||
json.NewDecoder(r.Body).Decode(&body)
|
||
if len(body.Permission) != 1 || body.Permission[0]["action"] != "deny" {
|
||
t.Fatal("probe has tools", body)
|
||
}
|
||
w.Write([]byte(`{"id":"ses_probe"}`))
|
||
case "POST /session/ses_probe/message":
|
||
var body struct {
|
||
NoReply bool `json:"noReply"`
|
||
}
|
||
json.NewDecoder(r.Body).Decode(&body)
|
||
if !body.NoReply {
|
||
t.Fatal("probe requested inference")
|
||
}
|
||
w.Write([]byte(`{}`))
|
||
case "GET /session/ses_probe/message":
|
||
w.WriteHeader(status)
|
||
if status == 200 {
|
||
w.Write([]byte(`[{"info":{"role":"user","format":{"type":"json_schema"}}}]`))
|
||
} else {
|
||
w.Write([]byte(`{"name":"BadRequest","data":{"message":"Expected OutputFormatJsonSchema at info.format"}}`))
|
||
}
|
||
case "POST /session/ses_primary/message":
|
||
generations++
|
||
var body map[string]json.RawMessage
|
||
json.NewDecoder(r.Body).Decode(&body)
|
||
if (len(body["format"]) > 0) != (status == 200) {
|
||
t.Error("incorrect format compatibility mode", string(body["format"]))
|
||
}
|
||
w.Write([]byte(`{"info":{"role":"assistant","finish":"stop","time":{"completed":1},"structured":{"outcome":"success","output":{"value":9}}},"parts":[]}`))
|
||
default:
|
||
t.Error("unexpected operation", r.Method, r.URL.Path)
|
||
}
|
||
}))
|
||
defer srv.Close()
|
||
c := &Connection{Backend: "v1", URL: srv.URL, models: map[string]map[string]string{"ses_primary": {"providerID": "test", "modelID": "test"}}, responses: map[string][]byte{}}
|
||
body := map[string]any{"prompt": "task", "format": map[string]any{"type": "json_schema", "schema": map[string]any{"type": "object"}, "retryCount": 0}}
|
||
data, err := c.Call(context.Background(), "post", "/api/session/ses_primary/generate", body)
|
||
if status == 401 || status == 500 {
|
||
if err == nil || generations != 0 {
|
||
t.Fatal("unrelated failure silently downgraded", err, generations)
|
||
}
|
||
return
|
||
}
|
||
if err != nil || !strings.Contains(string(data), "value") || generations != 1 {
|
||
t.Fatal(err, string(data), generations)
|
||
}
|
||
if c.FormatProbe == nil || c.FormatProbe.SessionID != "ses_probe" || c.FormatProbe.Supported != (status == 200) {
|
||
t.Fatal("probe evidence lost", c.FormatProbe)
|
||
}
|
||
if _, err = c.Call(context.Background(), "post", "/api/session/ses_primary/generate", body); err != nil || probes != 1 {
|
||
t.Fatal("capability probe was not cached", err, probes)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestNativeStructuredFailureIsClearAndDoesNotReplayGeneration(t *testing.T) {
|
||
response := []byte(`{"info":{"role":"assistant","finish":"stop","time":{"completed":1},"error":{"name":"StructuredOutputError","data":{"message":"Model did not produce structured output; secret-token"}}},"parts":[{"type":"text","text":"<|DSML|invoke name=\"StructuredOutput\">not JSON</|DSML|invoke>"}]}`)
|
||
_, err := messageText(response)
|
||
if err == nil || !strings.Contains(err.Error(), "StructuredOutputError: Model did not produce structured output") || strings.Contains(err.Error(), "secret-token") {
|
||
t.Fatal(err)
|
||
}
|
||
if _, partialErr := messageText([]byte(strings.Replace(string(response), `"completed":1`, `"completed":0`, 1))); partialErr == nil || !strings.Contains(partialErr.Error(), "StructuredOutputError") {
|
||
t.Fatal("incomplete timestamp hid provider failure", partialErr)
|
||
}
|
||
c := &Connection{Backend: "v1", responses: map[string][]byte{"ses_failure": response}}
|
||
retained, err := c.Call(context.Background(), "post", "/api/experimental/session/ses_failure/wait", nil)
|
||
if err == nil || string(retained) != string(response) {
|
||
t.Fatal("wait failure discarded provider response", err, string(retained))
|
||
}
|
||
}
|
||
|
||
// TestExecutionErrorReportsSanitizedNameAndStatusWithoutBody proves an
|
||
// unrecognized provider error name is reported as a bounded, safe identifier
|
||
// plus any HTTP status instead of a blanket summary, while no provider message,
|
||
// prompt or credential is copied and known names keep their exact wording.
|
||
func TestExecutionErrorReportsSanitizedNameAndStatusWithoutBody(t *testing.T) {
|
||
err := ExecutionError(json.RawMessage(`{"name":"TotallyUnknownError","data":{"statusCode":502,"message":"secret provider body"}}`))
|
||
if err == nil || !strings.Contains(err.Error(), "TotallyUnknownError") || !strings.Contains(err.Error(), "HTTP 502") {
|
||
t.Fatalf("unknown error name/status not reported: %v", err)
|
||
}
|
||
if strings.Contains(err.Error(), "secret provider body") {
|
||
t.Fatalf("diagnostic leaked the provider body: %v", err)
|
||
}
|
||
if err := ExecutionError(json.RawMessage(`{"_tag":"SomeTagError"}`)); err == nil || !strings.Contains(err.Error(), "SomeTagError") {
|
||
t.Fatalf("unknown tag not reported: %v", err)
|
||
}
|
||
if err := ExecutionError(json.RawMessage(`{"name":"Injected Error: Bearer secret-token"}`)); err == nil || strings.Contains(err.Error(), "secret-token") || strings.Contains(err.Error(), "Bearer") || err.Error() != "OpenCode agent reported an execution error" {
|
||
t.Fatalf("unsafe name was copied into the diagnostic: %v", err)
|
||
}
|
||
if err := ExecutionError(json.RawMessage(`{"name":"APIError","data":{"statusCode":500}}`)); err == nil || !strings.Contains(err.Error(), "APIError (HTTP 500)") {
|
||
t.Fatalf("known safe name changed: %v", err)
|
||
}
|
||
}
|