stage: retry pre-admission harness reach failures and widen repair deadline
Treat a failure to reach the harness before any prompt or tool call is admitted (connecting or creating the session) as a distinct retryable reach failure, so a transport blip at the first stage no longer ends a mutating run; keep post-admission readback/poll/agent failures retried only on an explicit read-only stage and never replay admitted tool work. Preserve the true cause when the context is already done (join ctx.Err with the typed transport error) and report a sanitized unknown provider error name/status instead of a blanket summary. Make the isolated format-repair deadline proportional to the stage timeout (floored, capped and clamped to remaining minus a margin) so a slow-but-repairable result is recovered without ever extending the stage deadline; the bounded ladder, value guard, deny-all formatter session, cumulative usage accounting and no-tool-replay rule are unchanged. Update docs/model-results.md, docs/pipeline-performance-review.md and the packaged skill references; add stage, opencode and server fixtures.
This commit is contained in:
@@ -247,3 +247,97 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
|
||||
t.Fatal("nonexistent run accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves,
|
||||
// through the real runner and CLI fixture, that a read-only stage retries a
|
||||
// post-admission transport (final-readback) failure with a fresh session while
|
||||
// an edit stage never replays the admitted prompt, and that the failed step's
|
||||
// diagnostic retains the underlying transport cause.
|
||||
func TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dir := t.TempDir()
|
||||
executable := filepath.Join(dir, "opencode")
|
||||
t.Setenv("PIPELINES_OPENCODE", executable)
|
||||
scriptTemplate := `#!/bin/sh
|
||||
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
||||
printf '%s %s\n' "$2" "$3" >> "@STATE@/calls"
|
||||
case "$3" in
|
||||
/api/session)
|
||||
n=0; if [ -f "@STATE@/sessions" ]; then n=$(cat "@STATE@/sessions"); fi
|
||||
n=$((n+1)); echo "$n" > "@STATE@/sessions"
|
||||
printf '{"data":{"id":"ses_bounded_%s"}}' "$n" ;;
|
||||
*/prompt) echo '{}' ;;
|
||||
/api/session/active) echo '{"data":{}}' ;;
|
||||
*/message*)
|
||||
r=0; if [ -f "@STATE@/readbacks" ]; then r=$(cat "@STATE@/readbacks"); fi
|
||||
r=$((r+1)); echo "$r" > "@STATE@/readbacks"
|
||||
if [ "$r" = "1" ]; then
|
||||
echo '{"data":[{"finish":"stop","content":'
|
||||
else
|
||||
echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\"outcome\":\"success\",\"output\":{\"summary\":\"done\"}}"}]}]}'
|
||||
fi ;;
|
||||
*/interrupt) echo '{}' ;;
|
||||
*) exit 99 ;;
|
||||
esac
|
||||
`
|
||||
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait() }()
|
||||
toolsBlock := func(tools string) string {
|
||||
return "# stage worker\n\n```output\nResult.\n```\n\n```prompt\nDo the work.\n```\n\n```tools\n" + tools + "\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=1\ntimeout=30s\n```\n"
|
||||
}
|
||||
runStage := func(name, tools string) (status, sessions, diagnostic string) {
|
||||
t.Helper()
|
||||
workspace := t.TempDir()
|
||||
script := strings.ReplaceAll(scriptTemplate, "@STATE@", workspace)
|
||||
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Submit(ctx, name, toolsBlock(tools)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
var id int64
|
||||
for _, item := range items {
|
||||
if item.Name == name {
|
||||
id = item.ID
|
||||
}
|
||||
}
|
||||
started, err := r.start(protocol.Message{PipelineID: id, Version: 1, RequestKey: name, Workspace: workspace, Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err := store.GetRun(ctx, started.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(detail.Steps) > 0 {
|
||||
diagnostic = detail.Steps[0].Diagnostic
|
||||
}
|
||||
if data, err := os.ReadFile(filepath.Join(workspace, "sessions")); err == nil {
|
||||
sessions = strings.TrimSpace(string(data))
|
||||
}
|
||||
return detail.Status, sessions, diagnostic
|
||||
}
|
||||
|
||||
status, sessions, diagnostic := runStage("readonly", "read")
|
||||
if status != "succeeded" || sessions != "2" || diagnostic != "" {
|
||||
t.Fatalf("read-only stage did not retry the transport readback: status=%q sessions=%q diagnostic=%q", status, sessions, diagnostic)
|
||||
}
|
||||
status, sessions, diagnostic = runStage("edit", "edit")
|
||||
if status != "failed" || sessions != "1" {
|
||||
t.Fatalf("edit stage replayed admitted work: status=%q sessions=%q diagnostic=%q", status, sessions, diagnostic)
|
||||
}
|
||||
if !strings.Contains(diagnostic, "HARNESS TRANSPORT") || !strings.Contains(diagnostic, "malformed or incomplete") {
|
||||
t.Fatalf("step diagnostic lost the underlying transport cause: %q", diagnostic)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user