16643b9f1b
Ten runs each burned five visits in seconds on "OpenCode agent reported an execution error": the provider answered HTTP 402 provider.quota and the classifier read neither field. That refusal is now its own error with fixed text, is not retried inside the stage, and ends the run instead of following the error edge.
138 lines
6.7 KiB
Go
138 lines
6.7 KiB
Go
package opencode
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
func TestNativeFormatProbeDowngradesOnlyReadbackIncompatibility(t *testing.T) {
|
||
for _, status := range []int{200, 400, 401, 500} {
|
||
t.Run(http.StatusText(status), func(t *testing.T) {
|
||
probes, generations := 0, 0
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
switch r.Method + " " + r.URL.Path {
|
||
case "POST /session":
|
||
probes++
|
||
var body struct{ Permission []map[string]string }
|
||
json.NewDecoder(r.Body).Decode(&body)
|
||
if len(body.Permission) != 1 || body.Permission[0]["action"] != "deny" {
|
||
t.Fatal("probe has tools", body)
|
||
}
|
||
w.Write([]byte(`{"id":"ses_probe"}`))
|
||
case "POST /session/ses_probe/message":
|
||
var body struct {
|
||
NoReply bool `json:"noReply"`
|
||
}
|
||
json.NewDecoder(r.Body).Decode(&body)
|
||
if !body.NoReply {
|
||
t.Fatal("probe requested inference")
|
||
}
|
||
w.Write([]byte(`{}`))
|
||
case "GET /session/ses_probe/message":
|
||
w.WriteHeader(status)
|
||
if status == 200 {
|
||
w.Write([]byte(`[{"info":{"role":"user","format":{"type":"json_schema"}}}]`))
|
||
} else {
|
||
w.Write([]byte(`{"name":"BadRequest","data":{"message":"Expected OutputFormatJsonSchema at info.format"}}`))
|
||
}
|
||
case "POST /session/ses_primary/message":
|
||
generations++
|
||
var body map[string]json.RawMessage
|
||
json.NewDecoder(r.Body).Decode(&body)
|
||
if (len(body["format"]) > 0) != (status == 200) {
|
||
t.Error("incorrect format compatibility mode", string(body["format"]))
|
||
}
|
||
w.Write([]byte(`{"info":{"role":"assistant","finish":"stop","time":{"completed":1},"structured":{"outcome":"success","output":{"value":9}}},"parts":[]}`))
|
||
default:
|
||
t.Error("unexpected operation", r.Method, r.URL.Path)
|
||
}
|
||
}))
|
||
defer srv.Close()
|
||
c := &Connection{Backend: "v1", URL: srv.URL, models: map[string]map[string]string{"ses_primary": {"providerID": "test", "modelID": "test"}}, responses: map[string][]byte{}}
|
||
body := map[string]any{"prompt": "task", "format": map[string]any{"type": "json_schema", "schema": map[string]any{"type": "object"}, "retryCount": 0}}
|
||
data, err := c.Call(context.Background(), "post", "/api/session/ses_primary/generate", body)
|
||
if status == 401 || status == 500 {
|
||
if err == nil || generations != 0 {
|
||
t.Fatal("unrelated failure silently downgraded", err, generations)
|
||
}
|
||
return
|
||
}
|
||
if err != nil || !strings.Contains(string(data), "value") || generations != 1 {
|
||
t.Fatal(err, string(data), generations)
|
||
}
|
||
if c.FormatProbe == nil || c.FormatProbe.SessionID != "ses_probe" || c.FormatProbe.Supported != (status == 200) {
|
||
t.Fatal("probe evidence lost", c.FormatProbe)
|
||
}
|
||
if _, err = c.Call(context.Background(), "post", "/api/session/ses_primary/generate", body); err != nil || probes != 1 {
|
||
t.Fatal("capability probe was not cached", err, probes)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestNativeStructuredFailureIsClearAndDoesNotReplayGeneration(t *testing.T) {
|
||
response := []byte(`{"info":{"role":"assistant","finish":"stop","time":{"completed":1},"error":{"name":"StructuredOutputError","data":{"message":"Model did not produce structured output; secret-token"}}},"parts":[{"type":"text","text":"<|DSML|invoke name=\"StructuredOutput\">not JSON</|DSML|invoke>"}]}`)
|
||
_, err := messageText(response)
|
||
if err == nil || !strings.Contains(err.Error(), "StructuredOutputError: Model did not produce structured output") || strings.Contains(err.Error(), "secret-token") {
|
||
t.Fatal(err)
|
||
}
|
||
if _, partialErr := messageText([]byte(strings.Replace(string(response), `"completed":1`, `"completed":0`, 1))); partialErr == nil || !strings.Contains(partialErr.Error(), "StructuredOutputError") {
|
||
t.Fatal("incomplete timestamp hid provider failure", partialErr)
|
||
}
|
||
c := &Connection{Backend: "v1", responses: map[string][]byte{"ses_failure": response}}
|
||
retained, err := c.Call(context.Background(), "post", "/api/experimental/session/ses_failure/wait", nil)
|
||
if err == nil || string(retained) != string(response) {
|
||
t.Fatal("wait failure discarded provider response", err, string(retained))
|
||
}
|
||
}
|
||
|
||
// TestExecutionErrorReportsSanitizedNameAndStatusWithoutBody proves an
|
||
// unrecognized provider error name is reported as a bounded, safe identifier
|
||
// plus any HTTP status instead of a blanket summary, while no provider message,
|
||
// prompt or credential is copied and known names keep their exact wording.
|
||
func TestExecutionErrorReportsSanitizedNameAndStatusWithoutBody(t *testing.T) {
|
||
err := ExecutionError(json.RawMessage(`{"name":"TotallyUnknownError","data":{"statusCode":502,"message":"secret provider body"}}`))
|
||
if err == nil || !strings.Contains(err.Error(), "TotallyUnknownError") || !strings.Contains(err.Error(), "HTTP 502") {
|
||
t.Fatalf("unknown error name/status not reported: %v", err)
|
||
}
|
||
if strings.Contains(err.Error(), "secret provider body") {
|
||
t.Fatalf("diagnostic leaked the provider body: %v", err)
|
||
}
|
||
if err := ExecutionError(json.RawMessage(`{"_tag":"SomeTagError"}`)); err == nil || !strings.Contains(err.Error(), "SomeTagError") {
|
||
t.Fatalf("unknown tag not reported: %v", err)
|
||
}
|
||
if err := ExecutionError(json.RawMessage(`{"name":"Injected Error: Bearer secret-token"}`)); err == nil || strings.Contains(err.Error(), "secret-token") || strings.Contains(err.Error(), "Bearer") || err.Error() != "OpenCode agent reported an execution error" {
|
||
t.Fatalf("unsafe name was copied into the diagnostic: %v", err)
|
||
}
|
||
if err := ExecutionError(json.RawMessage(`{"name":"APIError","data":{"statusCode":500}}`)); err == nil || !strings.Contains(err.Error(), "APIError (HTTP 500)") {
|
||
t.Fatalf("known safe name changed: %v", err)
|
||
}
|
||
}
|
||
|
||
// A provider that refuses for the account (no balance, quota used up) is its
|
||
// own error with fixed text: the provider's message and billing link are not
|
||
// copied, and callers can tell it must not be retried.
|
||
func TestExecutionErrorClassifiesProviderQuota(t *testing.T) {
|
||
for _, body := range []string{
|
||
`{"message":"Insufficient balance. Top up at https://example.test/billing?credits=1.","status":402,"type":"provider.quota"}`,
|
||
`{"name":"APIError","data":{"statusCode":402,"message":"pay"}}`,
|
||
} {
|
||
err := ExecutionError(json.RawMessage(body))
|
||
if !errors.Is(err, ErrProviderQuota) {
|
||
t.Fatalf("%s: err = %v", body, err)
|
||
}
|
||
if text := err.Error(); strings.Contains(text, "example.test") || strings.Contains(text, "Insufficient") || !strings.Contains(text, "HTTP 402") {
|
||
t.Fatalf("diagnostic = %q", text)
|
||
}
|
||
}
|
||
if err := ExecutionError(json.RawMessage(`{"name":"APIError","data":{"statusCode":500}}`)); errors.Is(err, ErrProviderQuota) {
|
||
t.Fatalf("a server error is not a quota refusal: %v", err)
|
||
}
|
||
}
|