Distinguish active stage deadlines from harness transport failures
Identify configured attempt expiry during V2 active-session idle waits and retain bounded phase, timeout and session diagnostics without claiming a lost response. Preserve genuine operation failures, error identities, redaction, cleanup and admitted-work replay protections. Add stage regressions and scratch-server CLI/websocket retention coverage. Checked packaged templates in examples and the pipeline template generator: none needs updating because timeout configuration and contracts are unchanged. Checked packaged authoring skill references: debugging.md needs a STAGE DEADLINE playbook and clarified transport guidance. Recorded that stale shared documentation in docs/notes/stage-timeout-diagnostics.md for docs-fold; shared documentation was deliberately not edited. Verification: go build ./...; go vet ./...; go test ./...; go test -count=1 ./internal/server ./internal/stage ./internal/pipeline ./internal/protocol; go test -count=1 -run TestStageDeadlineDiagnosticThroughCLIAndWebsocket -v ./internal/server; gofmt -l .; git diff --check. All passed. Live-provider behavior not tested; no deploy or deployed-service restart.
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
# Stage-timeout diagnostics
|
||||
|
||||
## Source behavior
|
||||
|
||||
The V2 active-session wait now reports `STAGE DEADLINE` when the configured
|
||||
attempt timer expires between successful polls of a running admitted session.
|
||||
Its bounded diagnostic names the idle-wait phase, configured timeout, retained
|
||||
session, and that the stage had not finished (active at the last poll). It does
|
||||
not claim that a completed response was lost. Cancellation is no longer recorded
|
||||
as a stage deadline. A bare deadline alone is not `HARNESS TRANSPORT`.
|
||||
|
||||
Actual request/HTTP/operation failures keep `HARNESS TRANSPORT`, their sanitized
|
||||
operation/status evidence and underlying error identities, including when joined
|
||||
with a deadline. A parent deadline no longer discards an accompanying transport
|
||||
failure. Final-result validation, budget accounting, cleanup, interrupts,
|
||||
side effects and replay policy are unchanged. No timeout was increased.
|
||||
|
||||
## Documentation to fold manually
|
||||
|
||||
`internal/author/skill/references/debugging.md` needs a separate stage-deadline
|
||||
playbook alongside its HARNESS TRANSPORT section: inspect the retained session,
|
||||
last activity, configured limits and unfinished final-result evidence before
|
||||
deciding what to do. Do not blindly retry an admitted mutating stage; edits/tool
|
||||
side effects remain. Explain the distinction between a configured idle-wait
|
||||
overrun, cancellation, an actual failed request with a deadline, budget failure,
|
||||
and final-result validation. Do not infer that all deadline errors are transport
|
||||
failures or that a successful last tool means the stage returned a result.
|
||||
|
||||
## Evidence and deployment
|
||||
|
||||
2026-10-18: installed server build metadata (`go version -m
|
||||
/Users/alan/.local/share/pipelines/bin/pipelines-server`) identifies revision
|
||||
`0d1ddc73a444fa24110ba5223646856c8f053d29`. Comparing that revision with this
|
||||
worktree's initial HEAD showed identical model-failure classification, model
|
||||
attempt timing, V2 polling and agent wait/cleanup paths. The runner difference
|
||||
was unrelated session-export HTML-only metadata.
|
||||
|
||||
Regression tests use isolated loopback servers and scratch SQLite databases,
|
||||
with a controlled V2 CLI fixture for the admitted-active-session deadline and
|
||||
real loopback HTTP/refused-connection checks for native transport evidence.
|
||||
This is not a live-provider timeout test. No deployment or deployed-service
|
||||
restart is part of this item; retained historical runs are not rewritten.
|
||||
@@ -0,0 +1,138 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// Exercise the real V2 CLI/poll path, runner, SQLite retention and websocket CLI
|
||||
// against an isolated loopback server/database, never the configured deployment.
|
||||
func TestStageDeadlineDiagnosticThroughCLIAndWebsocket(t *testing.T) {
|
||||
ntfy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"id":"scratch-receipt","event":"message"}`))
|
||||
}))
|
||||
defer ntfy.Close()
|
||||
t.Setenv("PIPELINES_NTFY_URL", ntfy.URL+"/scratch")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN", "")
|
||||
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
|
||||
for _, mode := range []string{"active", "http", "connection"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dir := t.TempDir()
|
||||
executable := filepath.Join(dir, "opencode")
|
||||
script := `#!/bin/sh
|
||||
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
||||
printf '%s %s\n' "$2" "$3" >> "@STATE@/calls"
|
||||
case "$3" in
|
||||
/api/session) echo '{"data":{"id":"ses_deadline"}}' ;;
|
||||
*/prompt) echo 'edited once' >> "@STATE@/edited"; echo '{}' ;;
|
||||
/api/session/active)
|
||||
if [ -f "@STATE@/interrupted" ]; then echo '{"data":{}}'; exit 0; fi
|
||||
case "@MODE@" in
|
||||
active) echo '{"data":{"ses_deadline":{"type":"running"}}}' ;;
|
||||
http) echo 'HTTP 503 token=private-token' >&2; exit 1 ;;
|
||||
connection) echo 'connection refused token=private-token' >&2; exit 1 ;;
|
||||
esac ;;
|
||||
*/interrupt) touch "@STATE@/interrupted"; echo '{}' ;;
|
||||
*/message*) echo '{"data":[{"content":[{"type":"text","text":"still working"}]}]}' ;;
|
||||
/api/session/ses_deadline) echo '{"data":{"id":"ses_deadline"}}' ;;
|
||||
*) echo '{}' ;;
|
||||
esac
|
||||
`
|
||||
script = strings.ReplaceAll(strings.ReplaceAll(script, "@STATE@", dir), "@MODE@", mode)
|
||||
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PIPELINES_OPENCODE", executable)
|
||||
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "```tools\n```", "```tools\nedit\n```", 1)
|
||||
source = strings.Replace(source, "timeout=1h", "timeout=500ms", 1)
|
||||
if _, err := store.Submit(ctx, "deadline", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
var out, stderr bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")}
|
||||
if err := app.Run(ctx, []string{"--url", url, "run", "deadline", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") {
|
||||
t.Fatalf("run not failed: %v, %s, %s", err, out.String(), stderr.String())
|
||||
}
|
||||
var response protocol.Message
|
||||
if err := json.Unmarshal(out.Bytes(), &response); err != nil {
|
||||
t.Fatal(err, out.String())
|
||||
}
|
||||
detail := response.RunDetail
|
||||
if detail == nil || detail.Status != "failed" || len(detail.Steps) != 1 {
|
||||
t.Fatal("missing failed run/step", out.String())
|
||||
}
|
||||
step := detail.Steps[0]
|
||||
if step.SessionID != "ses_deadline" || step.AgentActive || step.Output["error"] != step.Diagnostic || step.Status != "failed" || detail.Diagnostic != step.Diagnostic {
|
||||
t.Fatalf("incorrect retained failure: %+v", detail)
|
||||
}
|
||||
wants := []string{"HARNESS TRANSPORT", "get /api/session/active", "CLI exit code 1"}
|
||||
if mode == "active" {
|
||||
wants = []string{"STAGE DEADLINE", "timeout 500ms", "waiting for session idle", "ses_deadline", "stage had not finished"}
|
||||
if strings.Contains(detail.Diagnostic, "HARNESS TRANSPORT") || strings.Contains(detail.Diagnostic, "response lost") {
|
||||
t.Fatal("active stage overrun reported as transport", detail.Diagnostic)
|
||||
}
|
||||
if strings.Contains(detail.Diagnostic, "\n") {
|
||||
t.Fatal("duplicate context error appended to bounded stage diagnostic", detail.Diagnostic)
|
||||
}
|
||||
for _, want := range []string{"opencode_failures", "wait for session idle", "timeout 500ms", "retained_session", "still working"} {
|
||||
if !strings.Contains(step.Transcript, want) {
|
||||
t.Fatalf("missing retained evidence %q: %s", want, step.Transcript)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if !strings.Contains(step.Transcript, "[REDACTED]") {
|
||||
t.Fatal("request failure evidence missing", step.Transcript)
|
||||
}
|
||||
if mode == "http" {
|
||||
wants = append(wants, "HTTP 503")
|
||||
} else if !strings.Contains(step.Transcript, "connection refused") {
|
||||
t.Fatal("underlying connection evidence missing", step.Transcript)
|
||||
}
|
||||
}
|
||||
for _, want := range wants {
|
||||
if !strings.Contains(detail.Diagnostic, want) {
|
||||
t.Fatalf("missing diagnostic %q: %s", want, detail.Diagnostic)
|
||||
}
|
||||
}
|
||||
if strings.Contains(detail.Diagnostic+step.Transcript, "private-token") {
|
||||
t.Fatal("credential leaked in retained diagnostic/evidence")
|
||||
}
|
||||
edited, err := os.ReadFile(filepath.Join(dir, "edited"))
|
||||
if err != nil || string(edited) != "edited once\n" {
|
||||
t.Fatalf("admitted tool work replayed or lost: %q, %v", edited, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "interrupted")); err != nil {
|
||||
t.Fatal("cleanup did not interrupt", err)
|
||||
}
|
||||
stored, err := store.GetRun(ctx, detail.ID)
|
||||
if err != nil || stored.Diagnostic != detail.Diagnostic || stored.Steps[0].Transcript != step.Transcript {
|
||||
t.Fatal("diagnostic/evidence not persisted in scratch database", err)
|
||||
}
|
||||
t.Logf("scratch run %s: %s", mode, stored.Diagnostic)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -36,8 +36,8 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) {
|
||||
if typed := wrapModelStageFailure("worker", &openCodeError{operation: "get /api/session", status: 503}); !strings.Contains(typed.Error(), "HARNESS TRANSPORT") || !errors.As(typed, new(*openCodeError)) {
|
||||
t.Fatalf("typed harness error was not classified: %v", typed)
|
||||
}
|
||||
if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); !strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) {
|
||||
t.Fatalf("context deadline was not classified as harness: %v", deadline)
|
||||
if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) {
|
||||
t.Fatalf("bare context deadline was misclassified or lost: %v", deadline)
|
||||
}
|
||||
if modelFailureCategory(nil) != "" || wrapModelStageFailure("worker", nil) != nil {
|
||||
t.Fatal("nil failure was classified")
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Retained model-stage failure categories. A strict final-result validation
|
||||
@@ -13,8 +14,27 @@ import (
|
||||
const (
|
||||
finalResultValidation = "FINAL RESULT VALIDATION"
|
||||
harnessTransport = "HARNESS TRANSPORT"
|
||||
stageDeadline = "STAGE DEADLINE"
|
||||
)
|
||||
|
||||
// stageTimeoutCause identifies this attempt's configured timer, not a parent
|
||||
// cancellation or a request's own deadline. It is carried through budget contexts.
|
||||
type stageTimeoutCause struct{ timeout time.Duration }
|
||||
|
||||
func (e *stageTimeoutCause) Error() string { return "configured stage timeout reached" }
|
||||
func (e *stageTimeoutCause) Unwrap() error { return context.DeadlineExceeded }
|
||||
|
||||
type stageDeadlineError struct {
|
||||
timeout time.Duration
|
||||
session string
|
||||
}
|
||||
|
||||
func (e *stageDeadlineError) Error() string {
|
||||
return fmt.Sprintf("configured stage deadline reached (timeout %s) while waiting for session idle; retained OpenCode session %.128s was still active at the last poll; stage had not finished", e.timeout, RedactEvidence(e.session))
|
||||
}
|
||||
|
||||
func (e *stageDeadlineError) Unwrap() error { return context.DeadlineExceeded }
|
||||
|
||||
// finalResultGuidance is a fixed sanitized sentence appended only to validation
|
||||
// diagnostics. It states the retained side effects and the cost of a full retry.
|
||||
const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work"
|
||||
@@ -51,11 +71,14 @@ func (c *PiClient) workAdmitted() bool { return c.WorkStarted }
|
||||
// the label stays sanitized and bounded. Validation is checked first: a
|
||||
// structured-output failure is never reported as a transport failure and vice versa.
|
||||
func modelFailureCategory(err error) string {
|
||||
var deadline *stageDeadlineError
|
||||
switch {
|
||||
case errors.Is(err, ErrInvalidModelResult):
|
||||
return finalResultValidation
|
||||
case harnessTransportFailure(err), errors.Is(err, context.DeadlineExceeded):
|
||||
case harnessTransportFailure(err):
|
||||
return harnessTransport
|
||||
case errors.As(err, &deadline):
|
||||
return stageDeadline
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
@@ -73,6 +96,8 @@ func wrapModelStageFailure(key string, err error) error {
|
||||
return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance)
|
||||
case harnessTransport:
|
||||
return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err)
|
||||
case stageDeadline:
|
||||
return fmt.Errorf("model stage %s: %s: %w", key, stageDeadline, err)
|
||||
default:
|
||||
return fmt.Errorf("model stage %s: %w", key, err)
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
|
||||
return Result{}, err
|
||||
}
|
||||
spec.Config.Budget.Attempt()
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout)
|
||||
attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout})
|
||||
text, generateErr := client.Generate(attemptCtx, spec.Config, prompt)
|
||||
if generateErr == nil && attemptCtx.Err() != nil {
|
||||
generateErr = attemptCtx.Err()
|
||||
@@ -96,11 +96,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
|
||||
}
|
||||
} else {
|
||||
err = generateErr
|
||||
if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
|
||||
// Do not append the same context error twice: the wait/transport
|
||||
// error may already preserve it, with a more precise diagnostic.
|
||||
if attemptCtx.Err() != nil && !errors.Is(err, attemptCtx.Err()) && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
|
||||
err = errors.Join(err, attemptCtx.Err())
|
||||
}
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
if harnessTransportFailure(err) {
|
||||
return Result{}, wrapModelStageFailure(def.Key, err)
|
||||
}
|
||||
return Result{}, ctx.Err()
|
||||
}
|
||||
if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) {
|
||||
|
||||
@@ -217,6 +217,11 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
waitErr := fmt.Errorf("waiting for session idle; retained OpenCode session %.128s; stage had not finished: %w", RedactEvidence(sessionID), ctx.Err())
|
||||
var timeout *stageTimeoutCause
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.As(context.Cause(ctx), &timeout) {
|
||||
waitErr = &stageDeadlineError{timeout: timeout.timeout, session: sessionID}
|
||||
}
|
||||
// A settled session is the only success signal; retain one bounded
|
||||
// sanitized record so a timed-out wait is not an empty transcript.
|
||||
c.Operations.record(OperationEvidence{
|
||||
@@ -225,9 +230,9 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
|
||||
Executable: opencode.Executable(),
|
||||
Directory: c.Directory,
|
||||
ExitCode: -1,
|
||||
Diagnostic: "stage deadline reached while waiting for the agent session to become idle",
|
||||
Diagnostic: waitErr.Error(),
|
||||
}, started)
|
||||
return ctx.Err()
|
||||
return waitErr
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,7 +251,7 @@ func TestWaitForSessionIdleRecordsOneTimeoutEvidence(t *testing.T) {
|
||||
t.Fatalf("expected exactly one timeout evidence, got %d", len(log.Failures))
|
||||
}
|
||||
entry := log.Failures[0]
|
||||
if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "deadline reached") {
|
||||
if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "context deadline exceeded") {
|
||||
t.Fatalf("timeout evidence not sanitized as expected: %+v", entry)
|
||||
}
|
||||
clean := &OperationLog{}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
)
|
||||
|
||||
func TestConfiguredStageDeadlineWhileSessionActive(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Blocks["tools"] = "edit"
|
||||
def.Configs["timeout"] = "20ms"
|
||||
def.Configs["retries"] = "2"
|
||||
log := &OperationLog{}
|
||||
creates, prompts, interrupts := 0, 0, 0
|
||||
agent := &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", Operations: log, sessionPollInterval: time.Hour}}
|
||||
agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
switch args[2] {
|
||||
case "/api/session":
|
||||
creates++
|
||||
return []byte(`{"data":{"id":"ses_deadline"}}`), nil
|
||||
case "/api/session/ses_deadline/prompt":
|
||||
prompts++
|
||||
return []byte(`{}`), nil
|
||||
case "/api/session/active":
|
||||
return []byte(`{"data":{"ses_deadline":{"type":"running"}}}`), nil
|
||||
case "/api/session/ses_deadline/interrupt":
|
||||
interrupts++
|
||||
return []byte(`{}`), nil
|
||||
default:
|
||||
// Best-effort timeout readback/usage cannot convert the failure to success.
|
||||
return []byte(`{}`), nil
|
||||
}
|
||||
}
|
||||
result, err := (ModelStage{Client: agent}).Run(context.Background(), def, nil)
|
||||
if !errors.Is(err, context.DeadlineExceeded) || modelFailureCategory(err) != stageDeadline || errors.Is(err, ErrHarnessTransport) {
|
||||
t.Fatalf("deadline identity/category: %v", err)
|
||||
}
|
||||
for _, want := range []string{"STAGE DEADLINE", "worker", "timeout 20ms", "waiting for session idle", "ses_deadline", "stage had not finished"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("missing %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
if result.Output != nil || creates != 1 || prompts != 1 || interrupts != 1 || !agent.WorkStarted {
|
||||
t.Fatalf("work replayed, cleanup skipped or incomplete turn accepted: %+v creates=%d prompts=%d interrupts=%d", result, creates, prompts, interrupts)
|
||||
}
|
||||
if len(log.Failures) != 1 || !strings.Contains(log.Failures[0].Diagnostic, "timeout 20ms") {
|
||||
t.Fatalf("deadline evidence missing: %+v", log.Failures)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitCancellationIsNotConfiguredStageDeadline(t *testing.T) {
|
||||
for _, cause := range []error{context.Canceled, ErrBudgetExceeded, ErrBudgetAccounting} {
|
||||
t.Run(cause.Error(), func(t *testing.T) {
|
||||
attempt, stop := context.WithTimeoutCause(context.Background(), time.Hour, &stageTimeoutCause{timeout: time.Hour})
|
||||
defer stop()
|
||||
ctx, cancel := context.WithCancelCause(attempt)
|
||||
log := &OperationLog{}
|
||||
client := OpenCodeClient{Operations: log, command: func(context.Context, string, ...string) ([]byte, error) {
|
||||
cancel(cause)
|
||||
return []byte(`{"data":{"ses_wait":{"type":"running"}}}`), nil
|
||||
}}
|
||||
err := client.waitForSessionIdle(ctx, "ses_wait")
|
||||
if !errors.Is(err, context.Canceled) || modelFailureCategory(err) != "" || len(log.Failures) != 1 {
|
||||
t.Fatalf("cancellation misclassified: %v, %+v", err, log.Failures)
|
||||
}
|
||||
if strings.Contains(log.Failures[0].Diagnostic, "deadline") {
|
||||
t.Fatalf("cancellation recorded as deadline: %+v", log.Failures)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadlineWithTransportRetainsUnderlyingIdentity(t *testing.T) {
|
||||
transport := &openCodeError{operation: "get /api/session/active", status: 503, cause: errors.New("request failed")}
|
||||
err := wrapModelStageFailure("worker", errors.Join(&stageDeadlineError{timeout: time.Second, session: "ses_wait"}, transport))
|
||||
var got *openCodeError
|
||||
if modelFailureCategory(err) != harnessTransport || !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || !errors.Is(err, transport.cause) {
|
||||
t.Fatalf("combined failure lost transport/deadline identity: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "HTTP 503") || !strings.Contains(err.Error(), "get /api/session/active") {
|
||||
t.Fatalf("combined failure lost operation/status: %v", err)
|
||||
}
|
||||
bounded := (&stageDeadlineError{timeout: time.Hour, session: strings.Repeat("s", 10000) + " Bearer private-token"}).Error()
|
||||
if len(bounded) > 512 || strings.Contains(bounded, "private-token") {
|
||||
t.Fatal("deadline diagnostic was not bounded/redacted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParentDeadlinePreservesModelTransportFailure(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
transport := &openCodeError{operation: "get /api/session/active", status: 504, cause: errors.New("request failed")}
|
||||
client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) {
|
||||
<-ctx.Done()
|
||||
return "", errors.Join(ctx.Err(), transport)
|
||||
})
|
||||
_, err := (ModelStage{Client: client}).Run(ctx, modelDefinition(), nil)
|
||||
var got *openCodeError
|
||||
if !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || modelFailureCategory(err) != harnessTransport {
|
||||
t.Fatalf("parent deadline erased underlying request failure: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeHTTPAndConnectionFailuresRemainTransport(t *testing.T) {
|
||||
for _, mode := range []string{"http", "connection"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = w.Write([]byte(`{"message":"service unavailable","token":"private-token"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
if mode == "connection" {
|
||||
srv.Close()
|
||||
}
|
||||
log := &OperationLog{}
|
||||
client := OpenCodeClient{Backend: "v1", Operations: log, connection: &opencode.Connection{Backend: "v1", URL: srv.URL}}
|
||||
_, err := client.call(context.Background(), "get", "/api/session", nil)
|
||||
var typed *openCodeError
|
||||
if !errors.As(err, &typed) || modelFailureCategory(err) != harnessTransport || len(log.Failures) != 1 {
|
||||
t.Fatalf("real request failure misclassified: %v %+v", err, log.Failures)
|
||||
}
|
||||
if mode == "http" && (typed.status != 503 || log.Failures[0].HTTPStatus != 503 || !strings.Contains(log.Failures[0].Stdout, "service unavailable")) {
|
||||
t.Fatalf("HTTP status/evidence lost: %v %+v", err, log.Failures)
|
||||
}
|
||||
if mode == "connection" && (log.Failures[0].Diagnostic != "OpenCode native API request failed" || log.Failures[0].NativeOperation != "GET /session") {
|
||||
t.Fatalf("connection evidence lost: %+v", log.Failures)
|
||||
}
|
||||
if strings.Contains(err.Error()+log.Transcript(""), "private-token") {
|
||||
t.Fatal("credential leaked in transport diagnostic/evidence")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user