622 lines
27 KiB
Go
622 lines
27 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/cli"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func TestRetryInputOverrideRejectsReservedAndOversizedValues(t *testing.T) {
|
|
base := map[string]any{"handoff": "retained"}
|
|
for _, reserved := range []string{"workspace", "server_changed_files"} {
|
|
if _, err := retryInputOverride(base, map[string]any{reserved: "/escape"}); err == nil {
|
|
t.Fatalf("reserved retry input %s was accepted", reserved)
|
|
}
|
|
}
|
|
if _, err := retryInputOverride(base, map[string]any{"large": strings.Repeat("x", maxRetryInputBytes)}); err == nil {
|
|
t.Fatal("oversized retry input was accepted")
|
|
}
|
|
merged, err := retryInputOverride(base, map[string]any{"correction": "new"})
|
|
if err != nil || merged["handoff"] != "retained" || merged["correction"] != "new" {
|
|
t.Fatalf("retry input merge lost retained context: %#v, %v", merged, err)
|
|
}
|
|
}
|
|
|
|
func TestRetryStageContractIncludesPromptConfigAndEdges(t *testing.T) {
|
|
original := protocol.Stage{Key: "b", Type: "model", Prompt: "old", Configs: map[string]string{"model": "one"}, Edges: map[string]string{"success": "c"}}
|
|
if !retryStageContractEqual(original, original) {
|
|
t.Fatal("identical stage contracts differ")
|
|
}
|
|
changed := original
|
|
changed.Prompt = "corrected"
|
|
if retryStageContractEqual(original, changed) {
|
|
t.Fatal("changed stage prompt was treated as compatible")
|
|
}
|
|
changed = original
|
|
changed.Edges = map[string]string{"success": "other"}
|
|
if retryStageContractEqual(original, changed) {
|
|
t.Fatal("changed stage edge was treated as compatible")
|
|
}
|
|
}
|
|
|
|
func TestRetryCreatesFreshOpenCodeSessionAndRetainsFailedSession(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
dir := t.TempDir()
|
|
log := filepath.Join(dir, "calls")
|
|
count := filepath.Join(dir, "count")
|
|
executable := filepath.Join(dir, "opencode")
|
|
script := fmt.Sprintf(`#!/bin/sh
|
|
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
|
printf '%%s %%s %%s\n' "$PWD" "$2" "$3" >> %q
|
|
case "$3" in
|
|
/api/session/active) echo '{"data":{}}' ;;
|
|
/api/session)
|
|
n=0; if [ -f %q ]; then n=$(cat %q); fi
|
|
n=$((n+1)); echo "$n" > %q
|
|
printf '{"data":{"id":"ses_fresh_%%s"}}' "$n" ;;
|
|
/api/session/ses_fresh_1/prompt) echo '{"_tag":"InvalidRequestError"}'; exit 1 ;;
|
|
/api/session/ses_fresh_1/interrupt) echo '{}' ;;
|
|
/api/session/ses_fresh_1) echo '{"data":{"id":"ses_fresh_1"}}' ;;
|
|
/api/session/ses_fresh_1/message*) echo '{"data":[{"error":{"message":"provider rejected generation","token":"private-token"}}]}' ;;
|
|
/api/session/ses_fresh_2/prompt) echo '{}' ;;
|
|
/api/session/ses_fresh_2/message*) printf '%%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n"}]}]}' ;;
|
|
/api/session/ses_fresh_2) echo '{"data":{"id":"ses_fresh_2"}}' ;;
|
|
*) exit 99 ;;
|
|
esac
|
|
`, log, count, count, count)
|
|
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PIPELINES_OPENCODE", executable)
|
|
localExportService(t, "http://127.0.0.1:1")
|
|
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "fresh", pipeline.Template([]string{"plan"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.wg.Wait() }()
|
|
r.exportSession = validGateExport // Export is a separate mandatory handoff gate.
|
|
workspace, err := filepath.EvalSymlinks(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "first", Workspace: workspace, Input: map[string]any{"prompt": "original task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
original, err = store.GetRun(ctx, original.ID)
|
|
if err != nil || original.Status != "failed" || len(original.Steps) != 1 || original.Steps[0].SessionID != "ses_fresh_1" || original.Steps[0].AgentActive {
|
|
t.Fatal("failed session not retained/stopped", original, err)
|
|
}
|
|
if !strings.Contains(original.Steps[0].Transcript, "opencode_failures") || !strings.Contains(original.Steps[0].Transcript, "InvalidRequestError") {
|
|
t.Fatal("failure evidence not persisted", original.Steps[0].Transcript)
|
|
}
|
|
evidence, evidenceErr := r.evidence(ctx, original.ID)
|
|
if evidenceErr != nil || len(evidence) != 1 || evidence[0].Diagnostic != "" || !strings.Contains(evidence[0].Data, "provider rejected generation") || strings.Contains(evidence[0].Data, "private-token") {
|
|
t.Fatal(evidence, evidenceErr)
|
|
}
|
|
retry, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "second"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
retry, err = store.GetRun(ctx, retry.ID)
|
|
if err != nil || retry.ID == original.ID || retry.Status != "succeeded" || len(retry.Steps) != 1 || retry.Steps[0].SessionID != "ses_fresh_2" || retry.Workspace != original.Workspace || !reflect.DeepEqual(retry.Input, original.Input) {
|
|
t.Fatal("retry reused run/session state", retry, err)
|
|
}
|
|
unchanged, err := store.GetRun(ctx, original.ID)
|
|
if err != nil || !reflect.DeepEqual(original, unchanged) {
|
|
t.Fatal("retry modified old run", err)
|
|
}
|
|
calls, err := os.ReadFile(log)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(calls), "delete") || strings.Contains(string(calls), "/generate") || strings.Count(string(calls), "post /api/session\n") != 2 || strings.Count(string(calls), "ses_fresh_1/prompt") != 1 {
|
|
t.Fatal("session deleted or reused", string(calls))
|
|
}
|
|
for _, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
|
|
if !strings.HasPrefix(line, workspace+" ") {
|
|
t.Fatal("wrong invocation directory", line)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "workflow", pipeline.Template([]string{"plan", "implement"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
handler := Handler(ctx, store)
|
|
r := handler.(*runHandler).runner
|
|
defer func() { cancel(); WaitRuns(handler) }()
|
|
workspace, err := filepath.EvalSymlinks(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
initial := map[string]any{"prompt": "exact original task\nwith two lines", "pwd": "spoofed", "custom": map[string]any{"value": "keep me"}}
|
|
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
|
if def.Key == "plan" {
|
|
if input["pwd"] != workspace || input["prompt"] != initial["prompt"] {
|
|
t.Error("wrong initial input", input)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "intermediate, not the task"}}, nil
|
|
}
|
|
return stage.Result{}, fmt.Errorf("deliberate failure")
|
|
}
|
|
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, WritePaths: []string{"README.md"}, Input: initial})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
original, err = store.GetRun(ctx, original.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if original.Status != "failed" || original.Input["pwd"] != workspace || initial["pwd"] != "spoofed" {
|
|
t.Fatal("initial input was not recorded independently", original)
|
|
}
|
|
if _, err := store.Submit(ctx, "workflow", pipeline.Template([]string{"different"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
|
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
|
if def.Key == "plan" {
|
|
if !reflect.DeepEqual(map[string]any(input), original.Input) {
|
|
t.Error("retry used changed input", input, original.Input)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "new handoff"}}, nil
|
|
}
|
|
if def.Key != "implement" || input["summary"] != "new handoff" {
|
|
t.Error("retry did not restart at entry", def.Key, input)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
|
}
|
|
runCLI := func(args ...string) (protocol.Message, error) {
|
|
var out bytes.Buffer
|
|
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
|
err := app.Run(ctx, append([]string{"--url", url}, args...))
|
|
var result protocol.Message
|
|
if err == nil {
|
|
err = json.Unmarshal(out.Bytes(), &result)
|
|
}
|
|
return result, err
|
|
}
|
|
if _, err := runCLI("retry", fmt.Sprint(original.ID), "--json"); err == nil || !strings.Contains(err.Error(), "action canceled") {
|
|
t.Fatal("unconfirmed retry accepted", err)
|
|
}
|
|
beforeRetry, err := store.ListRuns(ctx)
|
|
if err != nil || len(beforeRetry) != 1 {
|
|
t.Fatal("unconfirmed retry created a run", err)
|
|
}
|
|
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key", "--mode", "restart")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
retried := result.RunDetail
|
|
if retried == nil || retried.ID == original.ID || retried.Status != "succeeded" || retried.Version != 1 || retried.RetryOf != original.ID || retried.Workspace != workspace || !reflect.DeepEqual(retried.Input, original.Input) || !reflect.DeepEqual(retried.WritePaths, original.WritePaths) || retried.AllowTools != original.AllowTools {
|
|
t.Fatal("retry did not preserve recorded run settings", retried)
|
|
}
|
|
r.wg.Wait()
|
|
again, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--request-key", "retry-key")
|
|
if err != nil || again.RunDetail.ID != retried.ID {
|
|
t.Fatal("retry lost idempotency", again, err)
|
|
}
|
|
unchanged, err := store.GetRun(ctx, original.ID)
|
|
if err != nil || !reflect.DeepEqual(original, unchanged) {
|
|
t.Fatal("retry modified failed history", unchanged, err)
|
|
}
|
|
if _, err := runCLI("retry", fmt.Sprint(retried.ID), "--yes", "--json"); err == nil {
|
|
t.Fatal("succeeded run accepted for retry")
|
|
}
|
|
if _, err := r.retry(protocol.Message{RunID: original.ID}); err == nil {
|
|
t.Fatal("retry accepted without idempotency key")
|
|
}
|
|
if _, err := r.start(protocol.Message{PipelineID: original.PipelineID, Version: 1, RequestKey: "retry-key", Workspace: workspace, Input: original.Input, WritePaths: original.WritePaths}); err == nil {
|
|
t.Fatal("ordinary run reused a retry's key")
|
|
}
|
|
}
|
|
|
|
func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
store.Submit(ctx, "workflow", pipeline.Template([]string{"one"}, true))
|
|
items, _ := store.List(ctx)
|
|
d, _ := store.Get(ctx, items[0].ID, 1)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, status := range []string{"queued", "running", "succeeded", "canceled", "failed"} {
|
|
id, _, err := store.CreateRun(ctx, d, status, filepath.Join(t.TempDir(), "missing"), false, nil, map[string]any{"prompt": "old task"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status != "queued" {
|
|
if err := store.FinishRun(ctx, id, status, "error", "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status, RetryMode: "restart"})
|
|
if err == nil {
|
|
t.Fatal("invalid retry accepted", status)
|
|
}
|
|
if status == "failed" && !strings.Contains(err.Error(), "workspace is unavailable") {
|
|
t.Fatal("retry did not validate original directory", err)
|
|
}
|
|
}
|
|
if _, err := r.retry(protocol.Message{RunID: 999, RequestKey: "missing"}); err == nil {
|
|
t.Fatal("nonexistent run accepted")
|
|
}
|
|
}
|
|
|
|
func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
dir := t.TempDir()
|
|
store, err := pipeline.Open(filepath.Join(dir, "resume.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
source := "# stage a\n\n```prompt\nDo A.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage b\n\n```prompt\nDo B.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage c\n\n```prompt\nDo C.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n"
|
|
if _, err := store.Submit(ctx, "resume", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
workspace := t.TempDir()
|
|
var aCalls, bCalls, cCalls int
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
|
switch def.Key {
|
|
case "a":
|
|
aCalls++
|
|
if input["prompt"] != "task" {
|
|
t.Errorf("entry received altered input: %#v", input)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"task": "task", "handoff": "from-a"}}, nil
|
|
case "b":
|
|
bCalls++
|
|
if input["handoff"] != "from-a" {
|
|
t.Errorf("resume lost exact prefix handoff: %#v", input)
|
|
}
|
|
wantTask := "task"
|
|
if bCalls == 2 {
|
|
wantTask = "corrected"
|
|
}
|
|
if bCalls == 3 {
|
|
wantTask = "corrected-again"
|
|
}
|
|
if input["task"] != wantTask {
|
|
t.Errorf("selected retry memory input = %#v, want task %q", input["task"], wantTask)
|
|
}
|
|
if bCalls == 2 && input["correction"] != "use-new-plan" {
|
|
t.Errorf("selected retry lost input override: %#v", input)
|
|
}
|
|
if bCalls == 3 && input["correction"] != "use-new-plan-again" {
|
|
t.Errorf("transitive selected retry lost input override: %#v", input)
|
|
}
|
|
if bCalls >= 2 {
|
|
// The model reads input.md, not the persisted step input. A retry
|
|
// correction that lives only in the step row is invisible.
|
|
data, err := os.ReadFile(def.Turn.Input)
|
|
if err != nil {
|
|
t.Fatalf("read retry input.md: %v", err)
|
|
}
|
|
want := "use-new-plan"
|
|
if bCalls == 3 {
|
|
want = "use-new-plan-again"
|
|
}
|
|
if !strings.Contains(string(data), want) {
|
|
t.Errorf("retry input.md missing override %q:\n%s", want, data)
|
|
}
|
|
}
|
|
if bCalls <= 2 {
|
|
return stage.Result{}, fmt.Errorf("deliberate failure")
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil
|
|
case "c":
|
|
cCalls++
|
|
if input["handoff"] != "from-b" {
|
|
t.Errorf("C received wrong handoff: %#v", input)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
|
default:
|
|
t.Fatalf("unexpected stage %q", def.Key)
|
|
return stage.Result{}, nil
|
|
}
|
|
}
|
|
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
original, err = store.GetRun(ctx, original.ID)
|
|
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
|
|
t.Fatalf("initial run did not stop at B: %+v %v", original, err)
|
|
}
|
|
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key", RetryStage: "b", RetryInput: map[string]any{"task": "corrected", "correction": "use-new-plan"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
retried, err = store.GetRun(ctx, retried.ID)
|
|
if err != nil || retried.Status != "failed" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
|
|
t.Fatalf("first resume did not preserve failed provenance: %+v %v", retried, err)
|
|
}
|
|
transitive, err := r.retry(protocol.Message{RunID: retried.ID, RequestKey: "transitive-resume-key", RetryStage: "b", RetryInput: map[string]any{"task": "corrected-again", "correction": "use-new-plan-again"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
transitive, err = store.GetRun(ctx, transitive.ID)
|
|
if err != nil || transitive.Status != "succeeded" || transitive.RetryMode != "resume" || transitive.RetrySource != retried.ID {
|
|
t.Fatalf("transitive resume did not complete with provenance: %+v %v", transitive, err)
|
|
}
|
|
if aCalls != 1 || bCalls != 3 || cCalls != 1 {
|
|
t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls)
|
|
}
|
|
if len(transitive.Steps) != 2 || transitive.Steps[0].Stage != "b" || transitive.Steps[1].Stage != "c" {
|
|
t.Fatalf("retry persisted the wrong steps: %+v", transitive.Steps)
|
|
}
|
|
if transitive.Steps[0].Input["task"] != "corrected-again" || transitive.Steps[0].Input["correction"] != "use-new-plan-again" {
|
|
t.Fatalf("retry persisted the wrong selected input: %#v", transitive.Steps[0].Input)
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = pipeline.Open(filepath.Join(dir, "resume.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r2, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
replayed, err := r2.retry(protocol.Message{RunID: retried.ID, RequestKey: "transitive-resume-key"})
|
|
if err != nil || replayed.ID != transitive.ID {
|
|
t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err)
|
|
}
|
|
}
|
|
|
|
func TestRetryResumeWithNativeToolsThroughHandlerAndCLI(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
dir := t.TempDir()
|
|
executable := filepath.Join(dir, "opencode")
|
|
script := `#!/bin/sh
|
|
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
|
case "$3" in
|
|
/api/session/active) echo '{"data":{}}' ;;
|
|
/api/session)
|
|
n=0; if [ -f "@STATE@/sessions" ]; then n=$(cat "@STATE@/sessions"); fi
|
|
n=$((n+1)); echo "$n" > "@STATE@/sessions"
|
|
echo '{"data":{"id":"ses_native_'$n'"}}' ;;
|
|
*/prompt) echo '{}' ;;
|
|
*/wait) echo '{}' ;;
|
|
*/message*)
|
|
case "$3" in
|
|
*/ses_native_2/*) printf '%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\ninvalid\n"}]}]}' ;;
|
|
*/ses_native_3/*) echo '{}' ;;
|
|
*/ses_native_3/*) printf '%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n\n# Handoff\n\nfrom-b\n"}]}]}' ;;
|
|
*) printf '%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n\n# Handoff\n\nfrom-a\n"}]}]}' ;;
|
|
esac ;;
|
|
*) echo '{}' ;;
|
|
esac
|
|
`
|
|
if err := os.WriteFile(executable, []byte(strings.ReplaceAll(script, "@STATE@", dir)), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PIPELINES_OPENCODE", executable)
|
|
localExportService(t, "http://127.0.0.1:1")
|
|
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
source := "# stage a\n\n```output\n{ handoff, summary }\n```\n\n```prompt\nDo A.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage b\n\n```output\n{ handoff }\n```\n\n```prompt\nDo B.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage c\n\n```output\n{ handoff, summary }\n```\n\n```prompt\nDo C.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n"
|
|
if _, err := store.Submit(ctx, "native-resume", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
handler := Handler(ctx, store)
|
|
handler.(*runHandler).runner.exportSession = validGateExport
|
|
defer func() { cancel(); WaitRuns(handler) }()
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
|
workspace := t.TempDir()
|
|
runCLI := func(args ...string) (protocol.Message, error) {
|
|
var out bytes.Buffer
|
|
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
|
err := app.Run(ctx, append([]string{"--url", url}, args...))
|
|
var result protocol.Message
|
|
if len(out.Bytes()) > 0 {
|
|
_ = json.Unmarshal(out.Bytes(), &result)
|
|
}
|
|
return result, err
|
|
}
|
|
first, err := runCLI("run", fmt.Sprint(items[0].Name), "task", "--yes", "--json", "--wait", "--timeout", "10s", "--workspace", workspace, "--request-key", "native-original")
|
|
if err == nil || first.RunDetail == nil || first.RunDetail.Status != "failed" || len(first.RunDetail.Steps) != 2 {
|
|
t.Fatalf("initial native run did not fail at B: %v %#v", err, first.RunDetail)
|
|
}
|
|
firstInput := filepath.Join(dir, "first-input.json")
|
|
if err := os.WriteFile(firstInput, []byte(`{"task":"corrected-first","correction":"first"}`), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
retried, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-resume-key", "--mode", "resume", "--stage", "b", "--input-file", firstInput)
|
|
if err == nil || retried.RunDetail == nil || retried.RunDetail.Status != "failed" {
|
|
t.Fatalf("first selected native resume did not fail at B: %v %#v", err, retried.RunDetail)
|
|
}
|
|
if retried.RunDetail.RetryOf != first.RunDetail.ID || len(retried.RunDetail.Steps) != 1 || retried.RunDetail.Steps[0].Stage != "b" {
|
|
t.Fatalf("native resume lost provenance or replayed prefix: %#v", retried.RunDetail)
|
|
}
|
|
secondInput := filepath.Join(dir, "second-input.json")
|
|
if err := os.WriteFile(secondInput, []byte(`{"task":"corrected-second","correction":"second"}`), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
transitive, err := runCLI("retry", fmt.Sprint(retried.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-transitive-key", "--mode", "resume", "--stage", "b", "--input-file", secondInput)
|
|
if err != nil || transitive.RunDetail == nil || transitive.RunDetail.Status != "succeeded" {
|
|
t.Fatalf("transitive selected native resume failed: %v %#v", err, transitive.RunDetail)
|
|
}
|
|
if transitive.RunDetail.RetryOf != retried.RunDetail.ID || len(transitive.RunDetail.Steps) != 2 || transitive.RunDetail.Steps[0].Stage != "b" || transitive.RunDetail.Steps[1].Stage != "c" {
|
|
t.Fatalf("transitive native resume lost provenance or replayed prefix: %#v", transitive.RunDetail)
|
|
}
|
|
if transitive.RunDetail.Steps[0].Input["task"] != "corrected-second" || transitive.RunDetail.Steps[0].Input["correction"] != "second" {
|
|
t.Fatalf("transitive native resume persisted wrong selected input: %#v", transitive.RunDetail.Steps[0].Input)
|
|
}
|
|
sessions, err := os.ReadFile(filepath.Join(dir, "sessions"))
|
|
if err != nil || strings.TrimSpace(string(sessions)) != "5" {
|
|
t.Fatalf("native stage execution count = %q, want 5 (A once, B three times, C once): %v", sessions, err)
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = pipeline.Open(filepath.Join(dir, "runs.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reopened, err := store.GetRun(ctx, transitive.RunDetail.ID)
|
|
if err != nil || reopened.Status != "succeeded" || reopened.RetryOf != retried.RunDetail.ID {
|
|
t.Fatalf("transitive retry did not survive database reopen: %#v %v", reopened, err)
|
|
}
|
|
}
|
|
|
|
// TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves,
|
|
// through the real runner and CLI fixture, that a read-only stage retries a
|
|
// post-admission transport (final-readback) failure with a fresh session while
|
|
// an edit stage never replays the admitted prompt, and that the failed step's
|
|
// diagnostic retains the underlying transport cause.
|
|
func TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
dir := t.TempDir()
|
|
executable := filepath.Join(dir, "opencode")
|
|
t.Setenv("PIPELINES_OPENCODE", executable)
|
|
localExportService(t, "http://127.0.0.1:1")
|
|
scriptTemplate := `#!/bin/sh
|
|
if [ "$1" = "--help" ]; then echo ' api'; exit 0; fi
|
|
printf '%s %s\n' "$2" "$3" >> "@STATE@/calls"
|
|
case "$3" in
|
|
/api/session)
|
|
n=0; if [ -f "@STATE@/sessions" ]; then n=$(cat "@STATE@/sessions"); fi
|
|
n=$((n+1)); echo "$n" > "@STATE@/sessions"
|
|
printf '{"data":{"id":"ses_bounded_%s"}}' "$n" ;;
|
|
*/prompt) echo '{}' ;;
|
|
/api/session/active) echo '{"data":{}}' ;;
|
|
*/message*)
|
|
r=0; if [ -f "@STATE@/readbacks" ]; then r=$(cat "@STATE@/readbacks"); fi
|
|
r=$((r+1)); echo "$r" > "@STATE@/readbacks"
|
|
if [ "$r" = "1" ]; then
|
|
echo '{"data":[{"finish":"stop","content":'
|
|
else
|
|
printf '%s\n' '{"data":[{"finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess\n"}]}]}'
|
|
fi ;;
|
|
*/interrupt) echo '{}' ;;
|
|
*) exit 99 ;;
|
|
esac
|
|
`
|
|
store, err := pipeline.Open(filepath.Join(dir, "runs.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.wg.Wait() }()
|
|
toolsBlock := func(tools string) string {
|
|
return "# stage worker\n\n```output\nResult.\n```\n\n```prompt\nDo the work.\n```\n\n```tools\n" + tools + "\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=1\ntimeout=30s\n```\n"
|
|
}
|
|
r.exportSession = validGateExport
|
|
runStage := func(name, tools string) (status, sessions, diagnostic string) {
|
|
t.Helper()
|
|
workspace := t.TempDir()
|
|
script := strings.ReplaceAll(scriptTemplate, "@STATE@", workspace)
|
|
if err := os.WriteFile(executable, []byte(script), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, name, toolsBlock(tools)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
var id int64
|
|
for _, item := range items {
|
|
if item.Name == name {
|
|
id = item.ID
|
|
}
|
|
}
|
|
started, err := r.start(protocol.Message{PipelineID: id, Version: 1, RequestKey: name, Workspace: workspace, Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
detail, err := store.GetRun(ctx, started.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(detail.Steps) > 0 {
|
|
diagnostic = detail.Steps[0].Diagnostic
|
|
}
|
|
if data, err := os.ReadFile(filepath.Join(workspace, "sessions")); err == nil {
|
|
sessions = strings.TrimSpace(string(data))
|
|
}
|
|
return detail.Status, sessions, diagnostic
|
|
}
|
|
|
|
status, sessions, diagnostic := runStage("readonly", "read")
|
|
if status != "succeeded" || sessions != "2" || diagnostic != "" {
|
|
t.Fatalf("read-only stage did not retry the transport readback: status=%q sessions=%q diagnostic=%q", status, sessions, diagnostic)
|
|
}
|
|
status, sessions, diagnostic = runStage("edit", "edit")
|
|
if status != "failed" || sessions != "1" {
|
|
t.Fatalf("edit stage replayed admitted work: status=%q sessions=%q diagnostic=%q", status, sessions, diagnostic)
|
|
}
|
|
if !strings.Contains(diagnostic, "HARNESS TRANSPORT") || !strings.Contains(diagnostic, "malformed or incomplete") {
|
|
t.Fatalf("step diagnostic lost the underlying transport cause: %q", diagnostic)
|
|
}
|
|
}
|