feat: resume retries from failed stage

This commit is contained in:
2026-10-18 20:38:03 +01:00
parent b79d0401f3
commit b8beeb250b
14 changed files with 336 additions and 19 deletions
+1 -1
View File
@@ -270,7 +270,7 @@ func TestHumanStageCancelRetryAndRebuild(t *testing.T) {
t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary)
}
r3.execute = successExecute
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human"})
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"})
if err != nil {
t.Fatal(err)
}
+2 -2
View File
@@ -43,14 +43,14 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) {
if err != nil || !runs[0].Executing {
t.Fatal("list advertised stopped run during cleanup", runs, err)
}
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err == nil || !strings.Contains(err.Error(), "still executing") {
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err == nil || !strings.Contains(err.Error(), "still executing") {
t.Fatal("retry accepted during execution cleanup", err)
}
delete(r.active, id)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
}
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err != nil {
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err != nil {
t.Fatal("retry rejected after cleanup", err)
}
r.wg.Wait()
+91 -2
View File
@@ -183,7 +183,7 @@ func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
if err != nil || len(beforeRetry) != 1 {
t.Fatal("unconfirmed retry created a run", err)
}
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key")
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key", "--mode", "restart")
if err != nil {
t.Fatal(err)
}
@@ -236,7 +236,7 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
t.Fatal(err)
}
}
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status})
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status, RetryMode: "restart"})
if err == nil {
t.Fatal("invalid retry accepted", status)
}
@@ -249,6 +249,95 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
}
}
func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir := t.TempDir()
store, err := pipeline.Open(filepath.Join(dir, "resume.db"))
if err != nil {
t.Fatal(err)
}
source := "# stage a\n\n```output\nA\n```\n\n```prompt\nA\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage b\n\n```output\nB\n```\n\n```prompt\nB\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage c\n\n```output\nC\n```\n\n```prompt\nC\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n"
if _, err := store.Submit(ctx, "resume", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
workspace := t.TempDir()
var aCalls, bCalls, cCalls int
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
switch def.Key {
case "a":
aCalls++
if input["prompt"] != "task" {
t.Errorf("entry received altered input: %#v", input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-a"}}, nil
case "b":
bCalls++
if input["handoff"] != "from-a" {
t.Errorf("resume lost exact prefix handoff: %#v", input)
}
if bCalls == 1 {
return stage.Result{}, fmt.Errorf("deliberate failure")
}
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil
case "c":
cCalls++
if input["handoff"] != "from-b" {
t.Errorf("C received wrong handoff: %#v", input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
default:
t.Fatalf("unexpected stage %q", def.Key)
return stage.Result{}, nil
}
}
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
original, err = store.GetRun(ctx, original.ID)
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
t.Fatalf("initial run did not stop at B: %+v %v", original, err)
}
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
retried, err = store.GetRun(ctx, retried.ID)
if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
t.Fatalf("resume did not complete with provenance: %+v %v", retried, err)
}
if aCalls != 1 || bCalls != 2 || cCalls != 1 {
t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls)
}
if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" {
t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = pipeline.Open(filepath.Join(dir, "resume.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r2, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
if err != nil || replayed.ID != retried.ID {
t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err)
}
}
// TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves,
// through the real runner and CLI fixture, that a read-only stage retries a
// post-admission transport (final-readback) failure with a fresh session while
+145 -1
View File
@@ -285,6 +285,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
}
if created {
if request.RetrySource > 0 && request.RetryMode != "restart" {
checkpoint, checkpointErr := r.retrySourceCheckpoint(request.RetrySource, d)
if checkpointErr != nil {
return nil, checkpointErr
}
if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil {
return nil, err
}
}
// Emit the created delta before launching: the queued frame reaches
// subscribers before the running/step-started transition, so a client
// sees the run appear and then advance.
@@ -404,6 +413,65 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
return items, err
}
type retryCheckpoint struct {
detail *protocol.RunDetail
failed protocol.RunStep
handoff map[string]any
}
func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) {
source, err := r.store.GetRun(r.ctx, sourceID)
if err != nil {
return retryCheckpoint{}, err
}
if source.Status != "failed" || source.FinishedAt == "" {
return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID)
}
if source.PipelineID != pinned.ID || source.Version != pinned.Version || source.ContentHash != pinned.ContentHash {
return retryCheckpoint{}, fmt.Errorf("retry source %d is pinned to a different pipeline version", sourceID)
}
if len(source.Steps) == 0 {
return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID)
}
failed := source.Steps[len(source.Steps)-1]
if (failed.Status != "completed" && failed.Status != "failed") || failed.Target != protocol.FailureEnd {
return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID)
}
var handoff map[string]any
if failed.Input == nil {
return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID)
}
handoff = map[string]any{"input": failed.Input, "sequence": failed.Sequence, "stage": failed.Stage}
prefix := map[string]any{}
for _, step := range source.Steps {
if step.Sequence >= failed.Sequence || step.Status != "completed" {
continue
}
prefix[step.Stage] = step.Output
}
handoff["prefix"] = prefix
for _, declared := range pinned.Graph.Stages {
if declared.Key != failed.Stage {
continue
}
typ := stage.EffectiveType(declared.Type)
if typ == stage.HumanType || typ == stage.NtfyType {
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage)
}
if typ == stage.ModelType {
tools, explicit, toolErr := stage.ToolNames(pipeline.StageDefinition(declared))
if toolErr != nil {
return retryCheckpoint{}, toolErr
}
if explicit && len(tools) > 0 {
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot safely replay admitted native stage %s", sourceID, failed.Stage)
}
}
return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil
}
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID)
}
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("retry requires a positive run ID")
@@ -418,7 +486,26 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if original.Status != "failed" || original.FinishedAt == "" {
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
if request.RequestKey == "" {
return nil, fmt.Errorf("retry requires an idempotency request key")
}
mode := request.RetryMode
if mode == "" {
mode = "resume"
}
if mode != "resume" && mode != "restart" {
return nil, fmt.Errorf("retry mode must be resume or restart")
}
d, err := r.store.Get(r.ctx, original.PipelineID, original.Version)
if err != nil {
return nil, err
}
if mode == "resume" {
if _, err := r.retrySourceCheckpoint(original.ID, d); err != nil {
return nil, err
}
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
}
func (r *runner) cancel(id int64) error {
@@ -520,6 +607,35 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
}
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
if request.RetrySource > 0 && request.RetryMode != "restart" {
source, err := r.store.GetRun(ctx, request.RetrySource)
if err != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
return
}
checkpoint, err := r.retrySourceCheckpoint(request.RetrySource, d)
if err != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
return
}
storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id)
if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil {
if checkpointErr == nil {
checkpointErr = fmt.Errorf("retry checkpoint is incomplete")
}
r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil)
return
}
if input, ok := storedHandoff["input"].(map[string]any); ok {
checkpoint.failed.Input = input
} else {
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
return
}
state := rebuildRetryState(source, checkpoint.failed)
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
return
}
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
@@ -629,6 +745,34 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
return state
}
// rebuildRetryState restores only the completed prefix. The failed visit is
// deliberately not counted and its step/evidence is not copied into the new
// attempt, so retries cannot duplicate execution or usage aggregates.
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState {
state := newRunState(failed.Stage, stage.Values(failed.Input))
state.nextSequence = failed.Sequence
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
state.memory["input"] = memoryValue{output: original}
}
for _, step := range detail.Steps {
if step.Sequence >= failed.Sequence {
break
}
state.visits[step.Stage]++
if step.Status == "completed" {
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
state.completed++
}
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
}
}
return state
}
// artifactAccess derives exact visit directories; it never consumes a path from
// a definition or model result. An unreadable or symlinked root fails closed.
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {