From 461b13841331a50cbdb935f6d38dc62e3cf92078 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 18 Oct 2026 23:57:28 +0100 Subject: [PATCH] fix retry-of-retry memory prefix reconstruction --- docs/notes/retry-transitive-memory-prefix.md | 6 + internal/server/retry_test.go | 90 ++++++++--- internal/server/runner.go | 153 +++++++++++++++---- 3 files changed, 202 insertions(+), 47 deletions(-) create mode 100644 docs/notes/retry-transitive-memory-prefix.md diff --git a/docs/notes/retry-transitive-memory-prefix.md b/docs/notes/retry-transitive-memory-prefix.md new file mode 100644 index 00000000..ec03aa5f --- /dev/null +++ b/docs/notes/retry-transitive-memory-prefix.md @@ -0,0 +1,6 @@ +# Transitive retry memory prefixes + +On 2026-10-18, resume checkpoints retain prefix provenance across selected-stage +retries. A retry-of-retry restores the original completed memory prefix from +the immutable checkpoint, then applies the new selected-stage override once; +the source attempt's steps remain evidence and are never copied or rerun. diff --git a/internal/server/retry_test.go b/internal/server/retry_test.go index c92b7470..3d79613b 100644 --- a/internal/server/retry_test.go +++ b/internal/server/retry_test.go @@ -317,13 +317,19 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T if bCalls == 2 { wantTask = "corrected" } + if bCalls == 3 { + wantTask = "corrected-again" + } if input["task"] != wantTask { t.Errorf("selected retry memory input = %#v, want task %q", input["task"], wantTask) } if bCalls == 2 && input["correction"] != "use-new-plan" { t.Errorf("selected retry lost input override: %#v", input) } - if bCalls == 1 { + if bCalls == 3 && input["correction"] != "use-new-plan-again" { + t.Errorf("transitive selected retry lost input override: %#v", input) + } + if bCalls <= 2 { return stage.Result{}, fmt.Errorf("deliberate failure") } return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil @@ -353,17 +359,29 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T } r.wg.Wait() retried, err = store.GetRun(ctx, retried.ID) - if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID { - t.Fatalf("resume did not complete with provenance: %+v %v", retried, err) + if err != nil || retried.Status != "failed" || retried.RetryMode != "resume" || retried.RetrySource != original.ID { + t.Fatalf("first resume did not preserve failed provenance: %+v %v", retried, err) } - if aCalls != 1 || bCalls != 2 || cCalls != 1 { + transitive, err := r.retry(protocol.Message{RunID: retried.ID, RequestKey: "transitive-resume-key", RetryStage: "b", RetryInput: map[string]any{"task": "corrected-again", "correction": "use-new-plan-again"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + transitive, err = store.GetRun(ctx, transitive.ID) + if err != nil || transitive.Status != "succeeded" || transitive.RetryMode != "resume" || transitive.RetrySource != retried.ID { + t.Fatalf("transitive resume did not complete with provenance: %+v %v", transitive, err) + } + if aCalls != 1 || bCalls != 3 || cCalls != 1 { t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls) } - if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" { - t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps) + if len(transitive.Steps) != 2 || transitive.Steps[0].Stage != "b" || transitive.Steps[1].Stage != "c" { + t.Fatalf("retry persisted the wrong steps: %+v", transitive.Steps) } - if retried.Steps[0].Input["task"] != "corrected" || retried.Steps[0].Input["correction"] != "use-new-plan" { - t.Fatalf("retry persisted the wrong selected input: %#v", retried.Steps[0].Input) + if transitive.Steps[0].Input["task"] != "corrected-again" || transitive.Steps[0].Input["correction"] != "use-new-plan-again" { + t.Fatalf("retry persisted the wrong selected input: %#v", transitive.Steps[0].Input) + } + if transitive.Steps[0].Control == nil || transitive.Steps[0].Control.Memory["handoff"].Sequence != 1 { + t.Fatalf("transitive retry lost prefix memory provenance: %#v", transitive.Steps[0].Control) } if err := store.Close(); err != nil { t.Fatal(err) @@ -377,8 +395,8 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T if err != nil { t.Fatal(err) } - replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"}) - if err != nil || replayed.ID != retried.ID { + replayed, err := r2.retry(protocol.Message{RunID: retried.ID, RequestKey: "transitive-resume-key"}) + if err != nil || replayed.ID != transitive.ID { t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err) } } @@ -399,8 +417,9 @@ case "$3" in */prompt) echo '{}' ;; */wait) echo '{}' ;; */message*) - case "$3" in + case "$3" in */ses_native_2/*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"invalid\\",\\"output\\":{\\"wrong\\":\\"format\\"}}"}]}]}' ;; + */ses_native_3/*) echo '{}' ;; */ses_native_3/*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"success\\",\\"output\\":{\\"handoff\\":\\"from-b\\"}}"}]}]}' ;; *) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"success\\",\\"output\\":{\\"handoff\\":\\"from-a\\",\\"summary\\":\\"done\\"}}"}]}]}' ;; esac ;; @@ -417,6 +436,7 @@ esac } defer store.Close() source := "# stage a\n\n```output\n{ handoff, summary }\n```\n\n```prompt\nDo A.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage b\n\n```output\n{ handoff }\n```\n\n```prompt\nDo B.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage c\n\n```output\n{ handoff, summary }\n```\n\n```prompt\nDo C.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n" + source = strings.Replace(source, "# stage b\n\n```output", "# stage b\n\n```memory\n{\"task\":\"input.prompt\",\"handoff\":\"a.handoff\"}\n```\n\n```output", 1) if _, err := store.Submit(ctx, "native-resume", source); err != nil { t.Fatal(err) } @@ -444,20 +464,48 @@ esac if err == nil || first.RunDetail == nil || first.RunDetail.Status != "failed" || len(first.RunDetail.Steps) != 2 { t.Fatalf("initial native run did not fail at B: %v %#v", err, first.RunDetail) } - retried, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-resume-key", "--mode", "resume") - if err != nil || retried.RunDetail == nil || retried.RunDetail.Status != "succeeded" { - t.Fatalf("native resume failed: %v %#v", err, retried.RunDetail) + firstInput := filepath.Join(dir, "first-input.json") + if err := os.WriteFile(firstInput, []byte(`{"task":"corrected-first","correction":"first"}`), 0600); err != nil { + t.Fatal(err) } - if retried.RunDetail.RetryOf != first.RunDetail.ID || len(retried.RunDetail.Steps) != 2 || retried.RunDetail.Steps[0].Stage != "b" || retried.RunDetail.Steps[1].Stage != "c" { + retried, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-resume-key", "--mode", "resume", "--stage", "b", "--input-file", firstInput) + if err == nil || retried.RunDetail == nil || retried.RunDetail.Status != "failed" { + t.Fatalf("first selected native resume did not fail at B: %v %#v", err, retried.RunDetail) + } + if retried.RunDetail.RetryOf != first.RunDetail.ID || len(retried.RunDetail.Steps) != 1 || retried.RunDetail.Steps[0].Stage != "b" { t.Fatalf("native resume lost provenance or replayed prefix: %#v", retried.RunDetail) } - sessions, err := os.ReadFile(filepath.Join(dir, "sessions")) - if err != nil || strings.TrimSpace(string(sessions)) != "4" { - t.Fatalf("native stage execution count = %q, want 4 (A once, B twice, C once): %v", sessions, err) + secondInput := filepath.Join(dir, "second-input.json") + if err := os.WriteFile(secondInput, []byte(`{"task":"corrected-second","correction":"second"}`), 0600); err != nil { + t.Fatal(err) } - again, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--request-key", "native-resume-key") - if err != nil || again.RunDetail == nil || again.RunDetail.ID != retried.RunDetail.ID { - t.Fatalf("same-key replay was not idempotent: %v %#v", err, again.RunDetail) + transitive, err := runCLI("retry", fmt.Sprint(retried.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-transitive-key", "--mode", "resume", "--stage", "b", "--input-file", secondInput) + if err != nil || transitive.RunDetail == nil || transitive.RunDetail.Status != "succeeded" { + t.Fatalf("transitive selected native resume failed: %v %#v", err, transitive.RunDetail) + } + if transitive.RunDetail.RetryOf != retried.RunDetail.ID || len(transitive.RunDetail.Steps) != 2 || transitive.RunDetail.Steps[0].Stage != "b" || transitive.RunDetail.Steps[1].Stage != "c" { + t.Fatalf("transitive native resume lost provenance or replayed prefix: %#v", transitive.RunDetail) + } + if transitive.RunDetail.Steps[0].Input["task"] != "corrected-second" || transitive.RunDetail.Steps[0].Input["correction"] != "second" { + t.Fatalf("transitive native resume persisted wrong selected input: %#v", transitive.RunDetail.Steps[0].Input) + } + if transitive.RunDetail.Steps[0].Control == nil || transitive.RunDetail.Steps[0].Control.Memory["handoff"].Sequence != 1 { + t.Fatalf("transitive native resume lost prefix provenance: %#v", transitive.RunDetail.Steps[0].Control) + } + sessions, err := os.ReadFile(filepath.Join(dir, "sessions")) + if err != nil || strings.TrimSpace(string(sessions)) != "5" { + t.Fatalf("native stage execution count = %q, want 5 (A once, B three times, C once): %v", sessions, err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + store, err = pipeline.Open(filepath.Join(dir, "runs.db")) + if err != nil { + t.Fatal(err) + } + reopened, err := store.GetRun(ctx, transitive.RunDetail.ID) + if err != nil || reopened.Status != "succeeded" || reopened.RetryOf != retried.RunDetail.ID { + t.Fatalf("transitive retry did not survive database reopen: %#v %v", reopened, err) } } diff --git a/internal/server/runner.go b/internal/server/runner.go index 90fb7985..d8c836c6 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -420,6 +420,11 @@ type retryCheckpoint struct { handoff map[string]any } +type retryPrefixRecord struct { + stage string + sequence int +} + const maxRetryInputBytes = 64 * 1024 func retryInputOverride(base map[string]any, override map[string]any) (map[string]any, error) { @@ -526,14 +531,16 @@ func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol. // stage's memory bindings must be resolved from the original prefix before // this override is applied at invocation time. handoff = map[string]any{"input": failed.Input, "override": override, "sequence": failed.Sequence, "stage": failed.Stage, "visit": failed.Sequence} - prefix := map[string]any{} - for _, step := range source.Steps { - if step.Sequence >= failed.Sequence || step.Status != "completed" { - continue - } - prefix[step.Stage] = step.Output + prefix, prefixRecords, prefixErr := r.retryPrefix(sourceID, source, failed.Sequence) + if prefixErr != nil { + return retryCheckpoint{}, prefixErr } handoff["prefix"] = prefix + serializedPrefix := make([]map[string]any, 0, len(prefixRecords)) + for _, record := range prefixRecords { + serializedPrefix = append(serializedPrefix, map[string]any{"stage": record.stage, "sequence": record.sequence}) + } + handoff["prefix_provenance"] = serializedPrefix // A newer explicitly selected version may continue only when every recorded // prefix stage still exists with the same kind. Its outputs are evidence from // the source attempt and cannot be safely handed to a changed stage contract. @@ -541,20 +548,21 @@ func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol. for _, declared := range source.Graph.Stages { sourceStages[declared.Key] = declared } - for _, step := range source.Steps { - if step.Sequence >= failed.Sequence { - break + for _, record := range prefixRecords { + _, found := retryPrefixStep(source, record) + if !found { + continue } var pinnedStage *protocol.Stage for i := range pinned.Graph.Stages { - if pinned.Graph.Stages[i].Key == step.Stage { + if pinned.Graph.Stages[i].Key == record.stage { pinnedStage = &pinned.Graph.Stages[i] break } } - previous, ok := sourceStages[step.Stage] + previous, ok := sourceStages[record.stage] if pinnedStage == nil || !ok || !retryStageContractEqual(previous, *pinnedStage) { - return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, step.Stage) + return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, record.stage) } } var sourceSelected protocol.Stage @@ -775,7 +783,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil) return } - state := rebuildRetryState(source, checkpoint.failed) + state := rebuildRetryState(source, checkpoint.failed, storedHandoff) if override, ok := storedHandoff["override"].(map[string]any); ok { state.retryOverride = stage.Values(override) } @@ -897,31 +905,124 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS // rebuildRetryState restores only the completed prefix. The failed visit is // deliberately not counted and its step/evidence is not copied into the new // attempt, so retries cannot duplicate execution or usage aggregates. -func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState { +func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep, handoff map[string]any) *runState { state := newRunState(failed.Stage, stage.Values(failed.Input)) state.nextSequence = failed.Sequence if original, err := copyValues(stage.Values(detail.Input)); err == nil { state.memory["input"] = memoryValue{output: original} } - for _, step := range detail.Steps { - if step.Sequence >= failed.Sequence { - break - } - state.visits[step.Stage]++ - if step.Status == "completed" { - if values, err := copyValues(stage.Values(step.Output)); err == nil { - state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} + _, hasCheckpointPrefix := handoff["prefix"] + if !hasCheckpointPrefix { + for _, step := range detail.Steps { + if step.Sequence >= failed.Sequence { + break + } + state.visits[step.Stage]++ + if step.Status == "completed" { + if values, err := copyValues(stage.Values(step.Output)); err == nil { + state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} + } + state.completed++ + } + state.recordChanges(step.FileChanges, step.Artifacts) + if step.Status == "completed" { + state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) } - state.completed++ } - state.recordChanges(step.FileChanges, step.Artifacts) - if step.Status == "completed" { - state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) + } + // A selected-stage retry has no persisted rows for its inherited prefix. + // Restore it from the immutable checkpoint instead of guessing from the + // retry source's immediate steps (which is what made retry-of-retry fail + // closed with "memory source ... has not completed"). + if prefix, ok := handoff["prefix"].(map[string]any); ok { + for _, record := range retryPrefixRecords(handoff["prefix_provenance"]) { + output, exists := prefix[record.stage].(map[string]any) + if !exists { + continue + } + values, err := copyValues(stage.Values(output)) + if err != nil { + continue + } + state.memory[record.stage] = memoryValue{output: values, sequence: record.sequence} + state.visits[record.stage]++ + state.completed++ + state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(record.sequence))) } } return state } +// retryPrefix returns the completed prefix from the source attempt plus the +// immutable prefix carried by that attempt's checkpoint. A retry attempt only +// persists the selected stage onward, so immediate source steps are not a +// complete representation of its memory. +func (r *runner) retryPrefix(sourceID int64, source *protocol.RunDetail, failedSequence int) (map[string]any, []retryPrefixRecord, error) { + prefix := map[string]any{} + records := []retryPrefixRecord{} + if source.RetryMode == "resume" && source.RetrySource > 0 { + checkpointSource, mode, _, inherited, err := r.store.RetryCheckpoint(r.ctx, sourceID) + if err != nil { + return nil, nil, fmt.Errorf("retry source %d has no immutable prefix checkpoint: %w", sourceID, err) + } + if checkpointSource != source.RetrySource || mode != "resume" { + return nil, nil, fmt.Errorf("retry source %d has an invalid prefix checkpoint", sourceID) + } + if values, ok := inherited["prefix"].(map[string]any); ok { + for key, value := range values { + prefix[key] = value + } + } + for _, record := range retryPrefixRecords(inherited["prefix_provenance"]) { + records = append(records, record) + } + } + for _, step := range source.Steps { + if step.Sequence >= failedSequence || step.Status != "completed" { + continue + } + prefix[step.Stage] = step.Output + records = append(records, retryPrefixRecord{stage: step.Stage, sequence: step.Sequence}) + } + return prefix, records, nil +} + +func retryPrefixRecords(raw any) []retryPrefixRecord { + items, ok := raw.([]any) + if !ok { + return nil + } + result := make([]retryPrefixRecord, 0, len(items)) + for _, item := range items { + entry, ok := item.(map[string]any) + if !ok { + continue + } + stageName, ok := entry["stage"].(string) + if !ok || stageName == "" { + continue + } + sequence, ok := entry["sequence"].(float64) + if !ok || sequence < 1 { + continue + } + result = append(result, retryPrefixRecord{stage: stageName, sequence: int(sequence)}) + } + return result +} + +func retryPrefixStep(detail *protocol.RunDetail, record retryPrefixRecord) (protocol.RunStep, bool) { + for _, step := range detail.Steps { + if step.Stage == record.stage && step.Sequence == record.sequence { + return step, true + } + } + // Inherited rows intentionally are not present in a selected retry. The + // contract check still uses the stage key; this marker keeps that check + // independent from model-claimed output. + return protocol.RunStep{Stage: record.stage, Sequence: record.sequence}, true +} + // artifactAccess derives exact visit directories; it never consumes a path from // a definition or model result. An unreadable or symlinked root fails closed. func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {