Distinguish active stage deadlines from harness transport failures

Identify configured attempt expiry during V2 active-session idle waits and retain bounded phase, timeout and session diagnostics without claiming a lost response. Preserve genuine operation failures, error identities, redaction, cleanup and admitted-work replay protections. Add stage regressions and scratch-server CLI/websocket retention coverage.

Checked packaged templates in examples and the pipeline template generator: none needs updating because timeout configuration and contracts are unchanged. Checked packaged authoring skill references: debugging.md needs a STAGE DEADLINE playbook and clarified transport guidance. Recorded that stale shared documentation in docs/notes/stage-timeout-diagnostics.md for docs-fold; shared documentation was deliberately not edited.

Verification: go build ./...; go vet ./...; go test ./...; go test -count=1 ./internal/server ./internal/stage ./internal/pipeline ./internal/protocol; go test -count=1 -run TestStageDeadlineDiagnosticThroughCLIAndWebsocket -v ./internal/server; gofmt -l .; git diff --check. All passed. Live-provider behavior not tested; no deploy or deployed-service restart.
This commit is contained in:
2026-10-18 11:17:01 +01:00
parent b93eee234c
commit 31d8f4a6d5
8 changed files with 363 additions and 8 deletions
+42
View File
@@ -0,0 +1,42 @@
# Stage-timeout diagnostics
## Source behavior
The V2 active-session wait now reports `STAGE DEADLINE` when the configured
attempt timer expires between successful polls of a running admitted session.
Its bounded diagnostic names the idle-wait phase, configured timeout, retained
session, and that the stage had not finished (active at the last poll). It does
not claim that a completed response was lost. Cancellation is no longer recorded
as a stage deadline. A bare deadline alone is not `HARNESS TRANSPORT`.
Actual request/HTTP/operation failures keep `HARNESS TRANSPORT`, their sanitized
operation/status evidence and underlying error identities, including when joined
with a deadline. A parent deadline no longer discards an accompanying transport
failure. Final-result validation, budget accounting, cleanup, interrupts,
side effects and replay policy are unchanged. No timeout was increased.
## Documentation to fold manually
`internal/author/skill/references/debugging.md` needs a separate stage-deadline
playbook alongside its HARNESS TRANSPORT section: inspect the retained session,
last activity, configured limits and unfinished final-result evidence before
deciding what to do. Do not blindly retry an admitted mutating stage; edits/tool
side effects remain. Explain the distinction between a configured idle-wait
overrun, cancellation, an actual failed request with a deadline, budget failure,
and final-result validation. Do not infer that all deadline errors are transport
failures or that a successful last tool means the stage returned a result.
## Evidence and deployment
2026-10-18: installed server build metadata (`go version -m
/Users/alan/.local/share/pipelines/bin/pipelines-server`) identifies revision
`0d1ddc73a444fa24110ba5223646856c8f053d29`. Comparing that revision with this
worktree's initial HEAD showed identical model-failure classification, model
attempt timing, V2 polling and agent wait/cleanup paths. The runner difference
was unrelated session-export HTML-only metadata.
Regression tests use isolated loopback servers and scratch SQLite databases,
with a controlled V2 CLI fixture for the admitted-active-session deadline and
real loopback HTTP/refused-connection checks for native transport evidence.
This is not a live-provider timeout test. No deployment or deployed-service
restart is part of this item; retained historical runs are not rewritten.
+138
View File
@@ -0,0 +1,138 @@
package server
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Exercise the real V2 CLI/poll path, runner, SQLite retention and websocket CLI
// against an isolated loopback server/database, never the configured deployment.
func TestStageDeadlineDiagnosticThroughCLIAndWebsocket(t *testing.T) {
ntfy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"id":"scratch-receipt","event":"message"}`))
}))
defer ntfy.Close()
t.Setenv("PIPELINES_NTFY_URL", ntfy.URL+"/scratch")
t.Setenv("PIPELINES_NTFY_TOKEN", "")
t.Setenv("PIPELINES_NTFY_TOKEN_FILE", "")
for _, mode := range []string{"active", "http", "connection"} {
t.Run(mode, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir := t.TempDir()
executable := filepath.Join(dir, "opencode")
script := `#!/bin/sh
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
printf '%s %s\n' "$2" "$3" >> "@STATE@/calls"
case "$3" in
/api/session) echo '{"data":{"id":"ses_deadline"}}' ;;
*/prompt) echo 'edited once' >> "@STATE@/edited"; echo '{}' ;;
/api/session/active)
if [ -f "@STATE@/interrupted" ]; then echo '{"data":{}}'; exit 0; fi
case "@MODE@" in
active) echo '{"data":{"ses_deadline":{"type":"running"}}}' ;;
http) echo 'HTTP 503 token=private-token' >&2; exit 1 ;;
connection) echo 'connection refused token=private-token' >&2; exit 1 ;;
esac ;;
*/interrupt) touch "@STATE@/interrupted"; echo '{}' ;;
*/message*) echo '{"data":[{"content":[{"type":"text","text":"still working"}]}]}' ;;
/api/session/ses_deadline) echo '{"data":{"id":"ses_deadline"}}' ;;
*) echo '{}' ;;
esac
`
script = strings.ReplaceAll(strings.ReplaceAll(script, "@STATE@", dir), "@MODE@", mode)
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_OPENCODE", executable)
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := pipeline.Template([]string{"worker"}, true)
source = strings.Replace(source, "```tools\n```", "```tools\nedit\n```", 1)
source = strings.Replace(source, "timeout=1h", "timeout=500ms", 1)
if _, err := store.Submit(ctx, "deadline", source); err != nil {
t.Fatal(err)
}
handler := Handler(ctx, store)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
var out, stderr bytes.Buffer
app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")}
if err := app.Run(ctx, []string{"--url", url, "run", "deadline", "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", dir}); err == nil || !strings.Contains(err.Error(), "run ended failed") {
t.Fatalf("run not failed: %v, %s, %s", err, out.String(), stderr.String())
}
var response protocol.Message
if err := json.Unmarshal(out.Bytes(), &response); err != nil {
t.Fatal(err, out.String())
}
detail := response.RunDetail
if detail == nil || detail.Status != "failed" || len(detail.Steps) != 1 {
t.Fatal("missing failed run/step", out.String())
}
step := detail.Steps[0]
if step.SessionID != "ses_deadline" || step.AgentActive || step.Output["error"] != step.Diagnostic || step.Status != "failed" || detail.Diagnostic != step.Diagnostic {
t.Fatalf("incorrect retained failure: %+v", detail)
}
wants := []string{"HARNESS TRANSPORT", "get /api/session/active", "CLI exit code 1"}
if mode == "active" {
wants = []string{"STAGE DEADLINE", "timeout 500ms", "waiting for session idle", "ses_deadline", "stage had not finished"}
if strings.Contains(detail.Diagnostic, "HARNESS TRANSPORT") || strings.Contains(detail.Diagnostic, "response lost") {
t.Fatal("active stage overrun reported as transport", detail.Diagnostic)
}
if strings.Contains(detail.Diagnostic, "\n") {
t.Fatal("duplicate context error appended to bounded stage diagnostic", detail.Diagnostic)
}
for _, want := range []string{"opencode_failures", "wait for session idle", "timeout 500ms", "retained_session", "still working"} {
if !strings.Contains(step.Transcript, want) {
t.Fatalf("missing retained evidence %q: %s", want, step.Transcript)
}
}
} else {
if !strings.Contains(step.Transcript, "[REDACTED]") {
t.Fatal("request failure evidence missing", step.Transcript)
}
if mode == "http" {
wants = append(wants, "HTTP 503")
} else if !strings.Contains(step.Transcript, "connection refused") {
t.Fatal("underlying connection evidence missing", step.Transcript)
}
}
for _, want := range wants {
if !strings.Contains(detail.Diagnostic, want) {
t.Fatalf("missing diagnostic %q: %s", want, detail.Diagnostic)
}
}
if strings.Contains(detail.Diagnostic+step.Transcript, "private-token") {
t.Fatal("credential leaked in retained diagnostic/evidence")
}
edited, err := os.ReadFile(filepath.Join(dir, "edited"))
if err != nil || string(edited) != "edited once\n" {
t.Fatalf("admitted tool work replayed or lost: %q, %v", edited, err)
}
if _, err := os.Stat(filepath.Join(dir, "interrupted")); err != nil {
t.Fatal("cleanup did not interrupt", err)
}
stored, err := store.GetRun(ctx, detail.ID)
if err != nil || stored.Diagnostic != detail.Diagnostic || stored.Steps[0].Transcript != step.Transcript {
t.Fatal("diagnostic/evidence not persisted in scratch database", err)
}
t.Logf("scratch run %s: %s", mode, stored.Diagnostic)
})
}
}
+2 -2
View File
@@ -36,8 +36,8 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) {
if typed := wrapModelStageFailure("worker", &openCodeError{operation: "get /api/session", status: 503}); !strings.Contains(typed.Error(), "HARNESS TRANSPORT") || !errors.As(typed, new(*openCodeError)) {
t.Fatalf("typed harness error was not classified: %v", typed)
}
if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); !strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) {
t.Fatalf("context deadline was not classified as harness: %v", deadline)
if deadline := wrapModelStageFailure("worker", context.DeadlineExceeded); strings.Contains(deadline.Error(), "HARNESS TRANSPORT") || !errors.Is(deadline, context.DeadlineExceeded) {
t.Fatalf("bare context deadline was misclassified or lost: %v", deadline)
}
if modelFailureCategory(nil) != "" || wrapModelStageFailure("worker", nil) != nil {
t.Fatal("nil failure was classified")
+26 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"time"
)
// Retained model-stage failure categories. A strict final-result validation
@@ -13,8 +14,27 @@ import (
const (
finalResultValidation = "FINAL RESULT VALIDATION"
harnessTransport = "HARNESS TRANSPORT"
stageDeadline = "STAGE DEADLINE"
)
// stageTimeoutCause identifies this attempt's configured timer, not a parent
// cancellation or a request's own deadline. It is carried through budget contexts.
type stageTimeoutCause struct{ timeout time.Duration }
func (e *stageTimeoutCause) Error() string { return "configured stage timeout reached" }
func (e *stageTimeoutCause) Unwrap() error { return context.DeadlineExceeded }
type stageDeadlineError struct {
timeout time.Duration
session string
}
func (e *stageDeadlineError) Error() string {
return fmt.Sprintf("configured stage deadline reached (timeout %s) while waiting for session idle; retained OpenCode session %.128s was still active at the last poll; stage had not finished", e.timeout, RedactEvidence(e.session))
}
func (e *stageDeadlineError) Unwrap() error { return context.DeadlineExceeded }
// finalResultGuidance is a fixed sanitized sentence appended only to validation
// diagnostics. It states the retained side effects and the cost of a full retry.
const finalResultGuidance = "completed edits and tool side effects are retained and not undone; a full retry repeats work"
@@ -51,11 +71,14 @@ func (c *PiClient) workAdmitted() bool { return c.WorkStarted }
// the label stays sanitized and bounded. Validation is checked first: a
// structured-output failure is never reported as a transport failure and vice versa.
func modelFailureCategory(err error) string {
var deadline *stageDeadlineError
switch {
case errors.Is(err, ErrInvalidModelResult):
return finalResultValidation
case harnessTransportFailure(err), errors.Is(err, context.DeadlineExceeded):
case harnessTransportFailure(err):
return harnessTransport
case errors.As(err, &deadline):
return stageDeadline
default:
return ""
}
@@ -73,6 +96,8 @@ func wrapModelStageFailure(key string, err error) error {
return fmt.Errorf("model stage %s: %s: %w; %s", key, finalResultValidation, err, finalResultGuidance)
case harnessTransport:
return fmt.Errorf("model stage %s: %s: %w", key, harnessTransport, err)
case stageDeadline:
return fmt.Errorf("model stage %s: %s: %w", key, stageDeadline, err)
default:
return fmt.Errorf("model stage %s: %w", key, err)
}
+7 -2
View File
@@ -61,7 +61,7 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
return Result{}, err
}
spec.Config.Budget.Attempt()
attemptCtx, cancel := context.WithTimeout(ctx, spec.Config.Timeout)
attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout})
text, generateErr := client.Generate(attemptCtx, spec.Config, prompt)
if generateErr == nil && attemptCtx.Err() != nil {
generateErr = attemptCtx.Err()
@@ -96,11 +96,16 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu
}
} else {
err = generateErr
if attemptCtx.Err() != nil && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
// Do not append the same context error twice: the wait/transport
// error may already preserve it, with a more precise diagnostic.
if attemptCtx.Err() != nil && !errors.Is(err, attemptCtx.Err()) && !errors.Is(err, ErrBudgetExceeded) && !errors.Is(err, ErrBudgetAccounting) {
err = errors.Join(err, attemptCtx.Err())
}
cancel()
if ctx.Err() != nil {
if harnessTransportFailure(err) {
return Result{}, wrapModelStageFailure(def.Key, err)
}
return Result{}, ctx.Err()
}
if errors.Is(err, ErrBudgetExceeded) || errors.Is(err, ErrBudgetAccounting) || errors.Is(err, ErrStageCleanup) {
+7 -2
View File
@@ -217,6 +217,11 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
select {
case <-ctx.Done():
timer.Stop()
waitErr := fmt.Errorf("waiting for session idle; retained OpenCode session %.128s; stage had not finished: %w", RedactEvidence(sessionID), ctx.Err())
var timeout *stageTimeoutCause
if errors.Is(ctx.Err(), context.DeadlineExceeded) && errors.As(context.Cause(ctx), &timeout) {
waitErr = &stageDeadlineError{timeout: timeout.timeout, session: sessionID}
}
// A settled session is the only success signal; retain one bounded
// sanitized record so a timed-out wait is not an empty transcript.
c.Operations.record(OperationEvidence{
@@ -225,9 +230,9 @@ func (c OpenCodeClient) waitForSessionIdle(ctx context.Context, sessionID string
Executable: opencode.Executable(),
Directory: c.Directory,
ExitCode: -1,
Diagnostic: "stage deadline reached while waiting for the agent session to become idle",
Diagnostic: waitErr.Error(),
}, started)
return ctx.Err()
return waitErr
case <-timer.C:
}
}
+1 -1
View File
@@ -251,7 +251,7 @@ func TestWaitForSessionIdleRecordsOneTimeoutEvidence(t *testing.T) {
t.Fatalf("expected exactly one timeout evidence, got %d", len(log.Failures))
}
entry := log.Failures[0]
if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "deadline reached") {
if entry.Operation != "wait for session idle" || !strings.Contains(entry.Diagnostic, "context deadline exceeded") {
t.Fatalf("timeout evidence not sanitized as expected: %+v", entry)
}
clean := &OperationLog{}
+140
View File
@@ -0,0 +1,140 @@
package stage
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/opencode"
)
func TestConfiguredStageDeadlineWhileSessionActive(t *testing.T) {
def := modelDefinition()
def.Blocks["tools"] = "edit"
def.Configs["timeout"] = "20ms"
def.Configs["retries"] = "2"
log := &OperationLog{}
creates, prompts, interrupts := 0, 0, 0
agent := &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", Operations: log, sessionPollInterval: time.Hour}}
agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
switch args[2] {
case "/api/session":
creates++
return []byte(`{"data":{"id":"ses_deadline"}}`), nil
case "/api/session/ses_deadline/prompt":
prompts++
return []byte(`{}`), nil
case "/api/session/active":
return []byte(`{"data":{"ses_deadline":{"type":"running"}}}`), nil
case "/api/session/ses_deadline/interrupt":
interrupts++
return []byte(`{}`), nil
default:
// Best-effort timeout readback/usage cannot convert the failure to success.
return []byte(`{}`), nil
}
}
result, err := (ModelStage{Client: agent}).Run(context.Background(), def, nil)
if !errors.Is(err, context.DeadlineExceeded) || modelFailureCategory(err) != stageDeadline || errors.Is(err, ErrHarnessTransport) {
t.Fatalf("deadline identity/category: %v", err)
}
for _, want := range []string{"STAGE DEADLINE", "worker", "timeout 20ms", "waiting for session idle", "ses_deadline", "stage had not finished"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("missing %q: %v", want, err)
}
}
if result.Output != nil || creates != 1 || prompts != 1 || interrupts != 1 || !agent.WorkStarted {
t.Fatalf("work replayed, cleanup skipped or incomplete turn accepted: %+v creates=%d prompts=%d interrupts=%d", result, creates, prompts, interrupts)
}
if len(log.Failures) != 1 || !strings.Contains(log.Failures[0].Diagnostic, "timeout 20ms") {
t.Fatalf("deadline evidence missing: %+v", log.Failures)
}
}
func TestWaitCancellationIsNotConfiguredStageDeadline(t *testing.T) {
for _, cause := range []error{context.Canceled, ErrBudgetExceeded, ErrBudgetAccounting} {
t.Run(cause.Error(), func(t *testing.T) {
attempt, stop := context.WithTimeoutCause(context.Background(), time.Hour, &stageTimeoutCause{timeout: time.Hour})
defer stop()
ctx, cancel := context.WithCancelCause(attempt)
log := &OperationLog{}
client := OpenCodeClient{Operations: log, command: func(context.Context, string, ...string) ([]byte, error) {
cancel(cause)
return []byte(`{"data":{"ses_wait":{"type":"running"}}}`), nil
}}
err := client.waitForSessionIdle(ctx, "ses_wait")
if !errors.Is(err, context.Canceled) || modelFailureCategory(err) != "" || len(log.Failures) != 1 {
t.Fatalf("cancellation misclassified: %v, %+v", err, log.Failures)
}
if strings.Contains(log.Failures[0].Diagnostic, "deadline") {
t.Fatalf("cancellation recorded as deadline: %+v", log.Failures)
}
})
}
}
func TestDeadlineWithTransportRetainsUnderlyingIdentity(t *testing.T) {
transport := &openCodeError{operation: "get /api/session/active", status: 503, cause: errors.New("request failed")}
err := wrapModelStageFailure("worker", errors.Join(&stageDeadlineError{timeout: time.Second, session: "ses_wait"}, transport))
var got *openCodeError
if modelFailureCategory(err) != harnessTransport || !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || !errors.Is(err, transport.cause) {
t.Fatalf("combined failure lost transport/deadline identity: %v", err)
}
if !strings.Contains(err.Error(), "HTTP 503") || !strings.Contains(err.Error(), "get /api/session/active") {
t.Fatalf("combined failure lost operation/status: %v", err)
}
bounded := (&stageDeadlineError{timeout: time.Hour, session: strings.Repeat("s", 10000) + " Bearer private-token"}).Error()
if len(bounded) > 512 || strings.Contains(bounded, "private-token") {
t.Fatal("deadline diagnostic was not bounded/redacted")
}
}
func TestParentDeadlinePreservesModelTransportFailure(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
transport := &openCodeError{operation: "get /api/session/active", status: 504, cause: errors.New("request failed")}
client := generatorFunc(func(ctx context.Context, _ ModelConfig, _ string) (string, error) {
<-ctx.Done()
return "", errors.Join(ctx.Err(), transport)
})
_, err := (ModelStage{Client: client}).Run(ctx, modelDefinition(), nil)
var got *openCodeError
if !errors.Is(err, context.DeadlineExceeded) || !errors.As(err, &got) || got != transport || modelFailureCategory(err) != harnessTransport {
t.Fatalf("parent deadline erased underlying request failure: %v", err)
}
}
func TestNativeHTTPAndConnectionFailuresRemainTransport(t *testing.T) {
for _, mode := range []string{"http", "connection"} {
t.Run(mode, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"message":"service unavailable","token":"private-token"}`))
}))
defer srv.Close()
if mode == "connection" {
srv.Close()
}
log := &OperationLog{}
client := OpenCodeClient{Backend: "v1", Operations: log, connection: &opencode.Connection{Backend: "v1", URL: srv.URL}}
_, err := client.call(context.Background(), "get", "/api/session", nil)
var typed *openCodeError
if !errors.As(err, &typed) || modelFailureCategory(err) != harnessTransport || len(log.Failures) != 1 {
t.Fatalf("real request failure misclassified: %v %+v", err, log.Failures)
}
if mode == "http" && (typed.status != 503 || log.Failures[0].HTTPStatus != 503 || !strings.Contains(log.Failures[0].Stdout, "service unavailable")) {
t.Fatalf("HTTP status/evidence lost: %v %+v", err, log.Failures)
}
if mode == "connection" && (log.Failures[0].Diagnostic != "OpenCode native API request failed" || log.Failures[0].NativeOperation != "GET /session") {
t.Fatalf("connection evidence lost: %+v", log.Failures)
}
if strings.Contains(err.Error()+log.Transcript(""), "private-token") {
t.Fatal("credential leaked in transport diagnostic/evidence")
}
})
}
}