fix retry-of-retry memory prefix reconstruction

This commit is contained in:
2026-10-18 23:57:28 +01:00
parent e3d6aa0cce
commit 461b138413
3 changed files with 202 additions and 47 deletions
@@ -0,0 +1,6 @@
# Transitive retry memory prefixes
On 2026-10-18, resume checkpoints retain prefix provenance across selected-stage
retries. A retry-of-retry restores the original completed memory prefix from
the immutable checkpoint, then applies the new selected-stage override once;
the source attempt's steps remain evidence and are never copied or rerun.
+69 -21
View File
@@ -317,13 +317,19 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T
if bCalls == 2 {
wantTask = "corrected"
}
if bCalls == 3 {
wantTask = "corrected-again"
}
if input["task"] != wantTask {
t.Errorf("selected retry memory input = %#v, want task %q", input["task"], wantTask)
}
if bCalls == 2 && input["correction"] != "use-new-plan" {
t.Errorf("selected retry lost input override: %#v", input)
}
if bCalls == 1 {
if bCalls == 3 && input["correction"] != "use-new-plan-again" {
t.Errorf("transitive selected retry lost input override: %#v", input)
}
if bCalls <= 2 {
return stage.Result{}, fmt.Errorf("deliberate failure")
}
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil
@@ -353,17 +359,29 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T
}
r.wg.Wait()
retried, err = store.GetRun(ctx, retried.ID)
if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
t.Fatalf("resume did not complete with provenance: %+v %v", retried, err)
if err != nil || retried.Status != "failed" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
t.Fatalf("first resume did not preserve failed provenance: %+v %v", retried, err)
}
if aCalls != 1 || bCalls != 2 || cCalls != 1 {
transitive, err := r.retry(protocol.Message{RunID: retried.ID, RequestKey: "transitive-resume-key", RetryStage: "b", RetryInput: map[string]any{"task": "corrected-again", "correction": "use-new-plan-again"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
transitive, err = store.GetRun(ctx, transitive.ID)
if err != nil || transitive.Status != "succeeded" || transitive.RetryMode != "resume" || transitive.RetrySource != retried.ID {
t.Fatalf("transitive resume did not complete with provenance: %+v %v", transitive, err)
}
if aCalls != 1 || bCalls != 3 || cCalls != 1 {
t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls)
}
if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" {
t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps)
if len(transitive.Steps) != 2 || transitive.Steps[0].Stage != "b" || transitive.Steps[1].Stage != "c" {
t.Fatalf("retry persisted the wrong steps: %+v", transitive.Steps)
}
if retried.Steps[0].Input["task"] != "corrected" || retried.Steps[0].Input["correction"] != "use-new-plan" {
t.Fatalf("retry persisted the wrong selected input: %#v", retried.Steps[0].Input)
if transitive.Steps[0].Input["task"] != "corrected-again" || transitive.Steps[0].Input["correction"] != "use-new-plan-again" {
t.Fatalf("retry persisted the wrong selected input: %#v", transitive.Steps[0].Input)
}
if transitive.Steps[0].Control == nil || transitive.Steps[0].Control.Memory["handoff"].Sequence != 1 {
t.Fatalf("transitive retry lost prefix memory provenance: %#v", transitive.Steps[0].Control)
}
if err := store.Close(); err != nil {
t.Fatal(err)
@@ -377,8 +395,8 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T
if err != nil {
t.Fatal(err)
}
replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
if err != nil || replayed.ID != retried.ID {
replayed, err := r2.retry(protocol.Message{RunID: retried.ID, RequestKey: "transitive-resume-key"})
if err != nil || replayed.ID != transitive.ID {
t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err)
}
}
@@ -399,8 +417,9 @@ case "$3" in
*/prompt) echo '{}' ;;
*/wait) echo '{}' ;;
*/message*)
case "$3" in
case "$3" in
*/ses_native_2/*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"invalid\\",\\"output\\":{\\"wrong\\":\\"format\\"}}"}]}]}' ;;
*/ses_native_3/*) echo '{}' ;;
*/ses_native_3/*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"success\\",\\"output\\":{\\"handoff\\":\\"from-b\\"}}"}]}]}' ;;
*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"success\\",\\"output\\":{\\"handoff\\":\\"from-a\\",\\"summary\\":\\"done\\"}}"}]}]}' ;;
esac ;;
@@ -417,6 +436,7 @@ esac
}
defer store.Close()
source := "# stage a\n\n```output\n{ handoff, summary }\n```\n\n```prompt\nDo A.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage b\n\n```output\n{ handoff }\n```\n\n```prompt\nDo B.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage c\n\n```output\n{ handoff, summary }\n```\n\n```prompt\nDo C.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n"
source = strings.Replace(source, "# stage b\n\n```output", "# stage b\n\n```memory\n{\"task\":\"input.prompt\",\"handoff\":\"a.handoff\"}\n```\n\n```output", 1)
if _, err := store.Submit(ctx, "native-resume", source); err != nil {
t.Fatal(err)
}
@@ -444,20 +464,48 @@ esac
if err == nil || first.RunDetail == nil || first.RunDetail.Status != "failed" || len(first.RunDetail.Steps) != 2 {
t.Fatalf("initial native run did not fail at B: %v %#v", err, first.RunDetail)
}
retried, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-resume-key", "--mode", "resume")
if err != nil || retried.RunDetail == nil || retried.RunDetail.Status != "succeeded" {
t.Fatalf("native resume failed: %v %#v", err, retried.RunDetail)
firstInput := filepath.Join(dir, "first-input.json")
if err := os.WriteFile(firstInput, []byte(`{"task":"corrected-first","correction":"first"}`), 0600); err != nil {
t.Fatal(err)
}
if retried.RunDetail.RetryOf != first.RunDetail.ID || len(retried.RunDetail.Steps) != 2 || retried.RunDetail.Steps[0].Stage != "b" || retried.RunDetail.Steps[1].Stage != "c" {
retried, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-resume-key", "--mode", "resume", "--stage", "b", "--input-file", firstInput)
if err == nil || retried.RunDetail == nil || retried.RunDetail.Status != "failed" {
t.Fatalf("first selected native resume did not fail at B: %v %#v", err, retried.RunDetail)
}
if retried.RunDetail.RetryOf != first.RunDetail.ID || len(retried.RunDetail.Steps) != 1 || retried.RunDetail.Steps[0].Stage != "b" {
t.Fatalf("native resume lost provenance or replayed prefix: %#v", retried.RunDetail)
}
sessions, err := os.ReadFile(filepath.Join(dir, "sessions"))
if err != nil || strings.TrimSpace(string(sessions)) != "4" {
t.Fatalf("native stage execution count = %q, want 4 (A once, B twice, C once): %v", sessions, err)
secondInput := filepath.Join(dir, "second-input.json")
if err := os.WriteFile(secondInput, []byte(`{"task":"corrected-second","correction":"second"}`), 0600); err != nil {
t.Fatal(err)
}
again, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--request-key", "native-resume-key")
if err != nil || again.RunDetail == nil || again.RunDetail.ID != retried.RunDetail.ID {
t.Fatalf("same-key replay was not idempotent: %v %#v", err, again.RunDetail)
transitive, err := runCLI("retry", fmt.Sprint(retried.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-transitive-key", "--mode", "resume", "--stage", "b", "--input-file", secondInput)
if err != nil || transitive.RunDetail == nil || transitive.RunDetail.Status != "succeeded" {
t.Fatalf("transitive selected native resume failed: %v %#v", err, transitive.RunDetail)
}
if transitive.RunDetail.RetryOf != retried.RunDetail.ID || len(transitive.RunDetail.Steps) != 2 || transitive.RunDetail.Steps[0].Stage != "b" || transitive.RunDetail.Steps[1].Stage != "c" {
t.Fatalf("transitive native resume lost provenance or replayed prefix: %#v", transitive.RunDetail)
}
if transitive.RunDetail.Steps[0].Input["task"] != "corrected-second" || transitive.RunDetail.Steps[0].Input["correction"] != "second" {
t.Fatalf("transitive native resume persisted wrong selected input: %#v", transitive.RunDetail.Steps[0].Input)
}
if transitive.RunDetail.Steps[0].Control == nil || transitive.RunDetail.Steps[0].Control.Memory["handoff"].Sequence != 1 {
t.Fatalf("transitive native resume lost prefix provenance: %#v", transitive.RunDetail.Steps[0].Control)
}
sessions, err := os.ReadFile(filepath.Join(dir, "sessions"))
if err != nil || strings.TrimSpace(string(sessions)) != "5" {
t.Fatalf("native stage execution count = %q, want 5 (A once, B three times, C once): %v", sessions, err)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = pipeline.Open(filepath.Join(dir, "runs.db"))
if err != nil {
t.Fatal(err)
}
reopened, err := store.GetRun(ctx, transitive.RunDetail.ID)
if err != nil || reopened.Status != "succeeded" || reopened.RetryOf != retried.RunDetail.ID {
t.Fatalf("transitive retry did not survive database reopen: %#v %v", reopened, err)
}
}
+127 -26
View File
@@ -420,6 +420,11 @@ type retryCheckpoint struct {
handoff map[string]any
}
type retryPrefixRecord struct {
stage string
sequence int
}
const maxRetryInputBytes = 64 * 1024
func retryInputOverride(base map[string]any, override map[string]any) (map[string]any, error) {
@@ -526,14 +531,16 @@ func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.
// stage's memory bindings must be resolved from the original prefix before
// this override is applied at invocation time.
handoff = map[string]any{"input": failed.Input, "override": override, "sequence": failed.Sequence, "stage": failed.Stage, "visit": failed.Sequence}
prefix := map[string]any{}
for _, step := range source.Steps {
if step.Sequence >= failed.Sequence || step.Status != "completed" {
continue
}
prefix[step.Stage] = step.Output
prefix, prefixRecords, prefixErr := r.retryPrefix(sourceID, source, failed.Sequence)
if prefixErr != nil {
return retryCheckpoint{}, prefixErr
}
handoff["prefix"] = prefix
serializedPrefix := make([]map[string]any, 0, len(prefixRecords))
for _, record := range prefixRecords {
serializedPrefix = append(serializedPrefix, map[string]any{"stage": record.stage, "sequence": record.sequence})
}
handoff["prefix_provenance"] = serializedPrefix
// A newer explicitly selected version may continue only when every recorded
// prefix stage still exists with the same kind. Its outputs are evidence from
// the source attempt and cannot be safely handed to a changed stage contract.
@@ -541,20 +548,21 @@ func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.
for _, declared := range source.Graph.Stages {
sourceStages[declared.Key] = declared
}
for _, step := range source.Steps {
if step.Sequence >= failed.Sequence {
break
for _, record := range prefixRecords {
_, found := retryPrefixStep(source, record)
if !found {
continue
}
var pinnedStage *protocol.Stage
for i := range pinned.Graph.Stages {
if pinned.Graph.Stages[i].Key == step.Stage {
if pinned.Graph.Stages[i].Key == record.stage {
pinnedStage = &pinned.Graph.Stages[i]
break
}
}
previous, ok := sourceStages[step.Stage]
previous, ok := sourceStages[record.stage]
if pinnedStage == nil || !ok || !retryStageContractEqual(previous, *pinnedStage) {
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, step.Stage)
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, record.stage)
}
}
var sourceSelected protocol.Stage
@@ -775,7 +783,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
return
}
state := rebuildRetryState(source, checkpoint.failed)
state := rebuildRetryState(source, checkpoint.failed, storedHandoff)
if override, ok := storedHandoff["override"].(map[string]any); ok {
state.retryOverride = stage.Values(override)
}
@@ -897,31 +905,124 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
// rebuildRetryState restores only the completed prefix. The failed visit is
// deliberately not counted and its step/evidence is not copied into the new
// attempt, so retries cannot duplicate execution or usage aggregates.
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState {
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep, handoff map[string]any) *runState {
state := newRunState(failed.Stage, stage.Values(failed.Input))
state.nextSequence = failed.Sequence
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
state.memory["input"] = memoryValue{output: original}
}
for _, step := range detail.Steps {
if step.Sequence >= failed.Sequence {
break
}
state.visits[step.Stage]++
if step.Status == "completed" {
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
_, hasCheckpointPrefix := handoff["prefix"]
if !hasCheckpointPrefix {
for _, step := range detail.Steps {
if step.Sequence >= failed.Sequence {
break
}
state.visits[step.Stage]++
if step.Status == "completed" {
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
state.completed++
}
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
}
state.completed++
}
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
}
// A selected-stage retry has no persisted rows for its inherited prefix.
// Restore it from the immutable checkpoint instead of guessing from the
// retry source's immediate steps (which is what made retry-of-retry fail
// closed with "memory source ... has not completed").
if prefix, ok := handoff["prefix"].(map[string]any); ok {
for _, record := range retryPrefixRecords(handoff["prefix_provenance"]) {
output, exists := prefix[record.stage].(map[string]any)
if !exists {
continue
}
values, err := copyValues(stage.Values(output))
if err != nil {
continue
}
state.memory[record.stage] = memoryValue{output: values, sequence: record.sequence}
state.visits[record.stage]++
state.completed++
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(record.sequence)))
}
}
return state
}
// retryPrefix returns the completed prefix from the source attempt plus the
// immutable prefix carried by that attempt's checkpoint. A retry attempt only
// persists the selected stage onward, so immediate source steps are not a
// complete representation of its memory.
func (r *runner) retryPrefix(sourceID int64, source *protocol.RunDetail, failedSequence int) (map[string]any, []retryPrefixRecord, error) {
prefix := map[string]any{}
records := []retryPrefixRecord{}
if source.RetryMode == "resume" && source.RetrySource > 0 {
checkpointSource, mode, _, inherited, err := r.store.RetryCheckpoint(r.ctx, sourceID)
if err != nil {
return nil, nil, fmt.Errorf("retry source %d has no immutable prefix checkpoint: %w", sourceID, err)
}
if checkpointSource != source.RetrySource || mode != "resume" {
return nil, nil, fmt.Errorf("retry source %d has an invalid prefix checkpoint", sourceID)
}
if values, ok := inherited["prefix"].(map[string]any); ok {
for key, value := range values {
prefix[key] = value
}
}
for _, record := range retryPrefixRecords(inherited["prefix_provenance"]) {
records = append(records, record)
}
}
for _, step := range source.Steps {
if step.Sequence >= failedSequence || step.Status != "completed" {
continue
}
prefix[step.Stage] = step.Output
records = append(records, retryPrefixRecord{stage: step.Stage, sequence: step.Sequence})
}
return prefix, records, nil
}
func retryPrefixRecords(raw any) []retryPrefixRecord {
items, ok := raw.([]any)
if !ok {
return nil
}
result := make([]retryPrefixRecord, 0, len(items))
for _, item := range items {
entry, ok := item.(map[string]any)
if !ok {
continue
}
stageName, ok := entry["stage"].(string)
if !ok || stageName == "" {
continue
}
sequence, ok := entry["sequence"].(float64)
if !ok || sequence < 1 {
continue
}
result = append(result, retryPrefixRecord{stage: stageName, sequence: int(sequence)})
}
return result
}
func retryPrefixStep(detail *protocol.RunDetail, record retryPrefixRecord) (protocol.RunStep, bool) {
for _, step := range detail.Steps {
if step.Stage == record.stage && step.Sequence == record.sequence {
return step, true
}
}
// Inherited rows intentionally are not present in a selected retry. The
// contract check still uses the stage key; this marker keeps that check
// independent from model-claimed output.
return protocol.RunStep{Stage: record.stage, Sequence: record.sequence}, true
}
// artifactAccess derives exact visit directories; it never consumes a path from
// a definition or model result. An unreadable or symlinked root fails closed.
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {