Files
pipelines/internal/server/retry_test.go
T
alan 7b72b8f20f stage: retry pre-admission harness reach failures and widen repair deadline
Treat a failure to reach the harness before any prompt or tool call is
admitted (connecting or creating the session) as a distinct retryable
reach failure, so a transport blip at the first stage no longer ends a
mutating run; keep post-admission readback/poll/agent failures retried
only on an explicit read-only stage and never replay admitted tool work.
Preserve the true cause when the context is already done (join ctx.Err
with the typed transport error) and report a sanitized unknown provider
error name/status instead of a blanket summary.

Make the isolated format-repair deadline proportional to the stage
timeout (floored, capped and clamped to remaining minus a margin) so a
slow-but-repairable result is recovered without ever extending the stage
deadline; the bounded ladder, value guard, deny-all formatter session,
cumulative usage accounting and no-tool-replay rule are unchanged.

Update docs/model-results.md, docs/pipeline-performance-review.md and the
packaged skill references; add stage, opencode and server fixtures.
2026-10-17 14:51:09 +01:00

344 lines
14 KiB
Go

package server
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/cli"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestRetryCreatesFreshOpenCodeSessionAndRetainsFailedSession(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir := t.TempDir()
log := filepath.Join(dir, "calls")
count := filepath.Join(dir, "count")
executable := filepath.Join(dir, "opencode")
script := fmt.Sprintf(`#!/bin/sh
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
printf '%%s %%s %%s\n' "$PWD" "$2" "$3" >> %q
case "$3" in
/api/session)
n=0; if [ -f %q ]; then n=$(cat %q); fi
n=$((n+1)); echo "$n" > %q
printf '{"data":{"id":"ses_fresh_%%s"}}' "$n" ;;
/api/session/ses_fresh_1/generate) echo '{"_tag":"InvalidRequestError"}'; exit 1 ;;
/api/session/ses_fresh_1/interrupt) echo '{}' ;;
/api/session/ses_fresh_1) echo '{"data":{"id":"ses_fresh_1"}}' ;;
/api/session/ses_fresh_1/message*) echo '{"data":[{"error":{"message":"provider rejected generation","token":"private-token"}}]}' ;;
/api/session/ses_fresh_2/generate) echo '{"data":{"text":"{\"outcome\":\"success\",\"output\":{\"summary\":\"done\"}}"}}' ;;
*) exit 99 ;;
esac
`, log, count, count, count)
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_OPENCODE", executable)
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "fresh", pipeline.Template([]string{"plan"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait() }()
workspace, err := filepath.EvalSymlinks(dir)
if err != nil {
t.Fatal(err)
}
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "first", Workspace: workspace, Input: map[string]any{"prompt": "original task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
original, err = store.GetRun(ctx, original.ID)
if err != nil || original.Status != "failed" || len(original.Steps) != 1 || original.Steps[0].SessionID != "ses_fresh_1" || original.Steps[0].AgentActive {
t.Fatal("failed session not retained/stopped", original, err)
}
if !strings.Contains(original.Steps[0].Transcript, "opencode_failures") || !strings.Contains(original.Steps[0].Transcript, "InvalidRequestError") {
t.Fatal("failure evidence not persisted", original.Steps[0].Transcript)
}
evidence, evidenceErr := r.evidence(ctx, original.ID)
if evidenceErr != nil || len(evidence) != 1 || evidence[0].Diagnostic != "" || !strings.Contains(evidence[0].Data, "provider rejected generation") || strings.Contains(evidence[0].Data, "private-token") {
t.Fatal(evidence, evidenceErr)
}
retry, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "second"})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
retry, err = store.GetRun(ctx, retry.ID)
if err != nil || retry.ID == original.ID || retry.Status != "succeeded" || len(retry.Steps) != 1 || retry.Steps[0].SessionID != "ses_fresh_2" || retry.Workspace != original.Workspace || !reflect.DeepEqual(retry.Input, original.Input) {
t.Fatal("retry reused run/session state", retry, err)
}
unchanged, err := store.GetRun(ctx, original.ID)
if err != nil || !reflect.DeepEqual(original, unchanged) {
t.Fatal("retry modified old run", err)
}
calls, err := os.ReadFile(log)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(calls), "delete") || strings.Count(string(calls), "post /api/session\n") != 2 || strings.Count(string(calls), "ses_fresh_1/generate") != 1 {
t.Fatal("session deleted or reused", string(calls))
}
for _, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
if !strings.HasPrefix(line, workspace+" ") {
t.Fatal("wrong invocation directory", line)
}
}
}
func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "workflow", pipeline.Template([]string{"plan", "implement"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
handler := Handler(ctx, store)
r := handler.(*runHandler).runner
defer func() { cancel(); WaitRuns(handler) }()
workspace, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
initial := map[string]any{"prompt": "exact original task\nwith two lines", "pwd": "spoofed", "custom": map[string]any{"value": "keep me"}}
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
if def.Key == "plan" {
if input["pwd"] != workspace || input["prompt"] != initial["prompt"] {
t.Error("wrong initial input", input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "intermediate, not the task"}}, nil
}
return stage.Result{}, fmt.Errorf("deliberate failure")
}
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, WritePaths: []string{"README.md"}, Input: initial})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
original, err = store.GetRun(ctx, original.ID)
if err != nil {
t.Fatal(err)
}
if original.Status != "failed" || original.Input["pwd"] != workspace || initial["pwd"] != "spoofed" {
t.Fatal("initial input was not recorded independently", original)
}
if _, err := store.Submit(ctx, "workflow", pipeline.Template([]string{"different"}, true)); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(handler)
defer srv.Close()
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
if def.Key == "plan" {
if !reflect.DeepEqual(map[string]any(input), original.Input) {
t.Error("retry used changed input", input, original.Input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "new handoff"}}, nil
}
if def.Key != "implement" || input["summary"] != "new handoff" {
t.Error("retry did not restart at entry", def.Key, input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
}
runCLI := func(args ...string) (protocol.Message, error) {
var out bytes.Buffer
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
err := app.Run(ctx, append([]string{"--url", url}, args...))
var result protocol.Message
if err == nil {
err = json.Unmarshal(out.Bytes(), &result)
}
return result, err
}
if _, err := runCLI("retry", fmt.Sprint(original.ID), "--json"); err == nil || !strings.Contains(err.Error(), "action canceled") {
t.Fatal("unconfirmed retry accepted", err)
}
beforeRetry, err := store.ListRuns(ctx)
if err != nil || len(beforeRetry) != 1 {
t.Fatal("unconfirmed retry created a run", err)
}
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key")
if err != nil {
t.Fatal(err)
}
retried := result.RunDetail
if retried == nil || retried.ID == original.ID || retried.Status != "succeeded" || retried.Version != 1 || retried.RetryOf != original.ID || retried.Workspace != workspace || !reflect.DeepEqual(retried.Input, original.Input) || !reflect.DeepEqual(retried.WritePaths, original.WritePaths) || retried.AllowTools != original.AllowTools {
t.Fatal("retry did not preserve recorded run settings", retried)
}
r.wg.Wait()
again, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--request-key", "retry-key")
if err != nil || again.RunDetail.ID != retried.ID {
t.Fatal("retry lost idempotency", again, err)
}
unchanged, err := store.GetRun(ctx, original.ID)
if err != nil || !reflect.DeepEqual(original, unchanged) {
t.Fatal("retry modified failed history", unchanged, err)
}
if _, err := runCLI("retry", fmt.Sprint(retried.ID), "--yes", "--json"); err == nil {
t.Fatal("succeeded run accepted for retry")
}
if _, err := r.retry(protocol.Message{RunID: original.ID}); err == nil {
t.Fatal("retry accepted without idempotency key")
}
if _, err := r.start(protocol.Message{PipelineID: original.PipelineID, Version: 1, RequestKey: "retry-key", Workspace: workspace, Input: original.Input, WritePaths: original.WritePaths}); err == nil {
t.Fatal("ordinary run reused a retry's key")
}
}
func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
store.Submit(ctx, "workflow", pipeline.Template([]string{"one"}, true))
items, _ := store.List(ctx)
d, _ := store.Get(ctx, items[0].ID, 1)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
for _, status := range []string{"queued", "running", "succeeded", "canceled", "failed"} {
id, _, err := store.CreateRun(ctx, d, status, filepath.Join(t.TempDir(), "missing"), false, nil, map[string]any{"prompt": "old task"})
if err != nil {
t.Fatal(err)
}
if status != "queued" {
if err := store.FinishRun(ctx, id, status, "error", "", nil); err != nil {
t.Fatal(err)
}
}
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status})
if err == nil {
t.Fatal("invalid retry accepted", status)
}
if status == "failed" && !strings.Contains(err.Error(), "workspace is unavailable") {
t.Fatal("retry did not validate original directory", err)
}
}
if _, err := r.retry(protocol.Message{RunID: 999, RequestKey: "missing"}); err == nil {
t.Fatal("nonexistent run accepted")
}
}
// TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves,
// through the real runner and CLI fixture, that a read-only stage retries a
// post-admission transport (final-readback) failure with a fresh session while
// an edit stage never replays the admitted prompt, and that the failed step's
// diagnostic retains the underlying transport cause.
func TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir := t.TempDir()
executable := filepath.Join(dir, "opencode")
t.Setenv("PIPELINES_OPENCODE", executable)
scriptTemplate := `#!/bin/sh
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
printf '%s %s\n' "$2" "$3" >> "@STATE@/calls"
case "$3" in
/api/session)
n=0; if [ -f "@STATE@/sessions" ]; then n=$(cat "@STATE@/sessions"); fi
n=$((n+1)); echo "$n" > "@STATE@/sessions"
printf '{"data":{"id":"ses_bounded_%s"}}' "$n" ;;
*/prompt) echo '{}' ;;
/api/session/active) echo '{"data":{}}' ;;
*/message*)
r=0; if [ -f "@STATE@/readbacks" ]; then r=$(cat "@STATE@/readbacks"); fi
r=$((r+1)); echo "$r" > "@STATE@/readbacks"
if [ "$r" = "1" ]; then
echo '{"data":[{"finish":"stop","content":'
else
echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\"outcome\":\"success\",\"output\":{\"summary\":\"done\"}}"}]}]}'
fi ;;
*/interrupt) echo '{}' ;;
*) exit 99 ;;
esac
`
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait() }()
toolsBlock := func(tools string) string {
return "# stage worker\n\n```output\nResult.\n```\n\n```prompt\nDo the work.\n```\n\n```tools\n" + tools + "\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=1\ntimeout=30s\n```\n"
}
runStage := func(name, tools string) (status, sessions, diagnostic string) {
t.Helper()
workspace := t.TempDir()
script := strings.ReplaceAll(scriptTemplate, "@STATE@", workspace)
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
t.Fatal(err)
}
if _, err := store.Submit(ctx, name, toolsBlock(tools)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
var id int64
for _, item := range items {
if item.Name == name {
id = item.ID
}
}
started, err := r.start(protocol.Message{PipelineID: id, Version: 1, RequestKey: name, Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, started.ID)
if err != nil {
t.Fatal(err)
}
if len(detail.Steps) > 0 {
diagnostic = detail.Steps[0].Diagnostic
}
if data, err := os.ReadFile(filepath.Join(workspace, "sessions")); err == nil {
sessions = strings.TrimSpace(string(data))
}
return detail.Status, sessions, diagnostic
}
status, sessions, diagnostic := runStage("readonly", "read")
if status != "succeeded" || sessions != "2" || diagnostic != "" {
t.Fatalf("read-only stage did not retry the transport readback: status=%q sessions=%q diagnostic=%q", status, sessions, diagnostic)
}
status, sessions, diagnostic = runStage("edit", "edit")
if status != "failed" || sessions != "1" {
t.Fatalf("edit stage replayed admitted work: status=%q sessions=%q diagnostic=%q", status, sessions, diagnostic)
}
if !strings.Contains(diagnostic, "HARNESS TRANSPORT") || !strings.Contains(diagnostic, "malformed or incomplete") {
t.Fatalf("step diagnostic lost the underlying transport cause: %q", diagnostic)
}
}