fix retry-of-retry memory prefix reconstruction
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
# Transitive retry memory prefixes
|
||||
|
||||
On 2026-10-18, resume checkpoints retain prefix provenance across selected-stage
|
||||
retries. A retry-of-retry restores the original completed memory prefix from
|
||||
the immutable checkpoint, then applies the new selected-stage override once;
|
||||
the source attempt's steps remain evidence and are never copied or rerun.
|
||||
@@ -317,13 +317,19 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T
|
||||
if bCalls == 2 {
|
||||
wantTask = "corrected"
|
||||
}
|
||||
if bCalls == 3 {
|
||||
wantTask = "corrected-again"
|
||||
}
|
||||
if input["task"] != wantTask {
|
||||
t.Errorf("selected retry memory input = %#v, want task %q", input["task"], wantTask)
|
||||
}
|
||||
if bCalls == 2 && input["correction"] != "use-new-plan" {
|
||||
t.Errorf("selected retry lost input override: %#v", input)
|
||||
}
|
||||
if bCalls == 1 {
|
||||
if bCalls == 3 && input["correction"] != "use-new-plan-again" {
|
||||
t.Errorf("transitive selected retry lost input override: %#v", input)
|
||||
}
|
||||
if bCalls <= 2 {
|
||||
return stage.Result{}, fmt.Errorf("deliberate failure")
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil
|
||||
@@ -353,17 +359,29 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T
|
||||
}
|
||||
r.wg.Wait()
|
||||
retried, err = store.GetRun(ctx, retried.ID)
|
||||
if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
|
||||
t.Fatalf("resume did not complete with provenance: %+v %v", retried, err)
|
||||
if err != nil || retried.Status != "failed" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
|
||||
t.Fatalf("first resume did not preserve failed provenance: %+v %v", retried, err)
|
||||
}
|
||||
if aCalls != 1 || bCalls != 2 || cCalls != 1 {
|
||||
transitive, err := r.retry(protocol.Message{RunID: retried.ID, RequestKey: "transitive-resume-key", RetryStage: "b", RetryInput: map[string]any{"task": "corrected-again", "correction": "use-new-plan-again"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
transitive, err = store.GetRun(ctx, transitive.ID)
|
||||
if err != nil || transitive.Status != "succeeded" || transitive.RetryMode != "resume" || transitive.RetrySource != retried.ID {
|
||||
t.Fatalf("transitive resume did not complete with provenance: %+v %v", transitive, err)
|
||||
}
|
||||
if aCalls != 1 || bCalls != 3 || cCalls != 1 {
|
||||
t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls)
|
||||
}
|
||||
if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" {
|
||||
t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps)
|
||||
if len(transitive.Steps) != 2 || transitive.Steps[0].Stage != "b" || transitive.Steps[1].Stage != "c" {
|
||||
t.Fatalf("retry persisted the wrong steps: %+v", transitive.Steps)
|
||||
}
|
||||
if retried.Steps[0].Input["task"] != "corrected" || retried.Steps[0].Input["correction"] != "use-new-plan" {
|
||||
t.Fatalf("retry persisted the wrong selected input: %#v", retried.Steps[0].Input)
|
||||
if transitive.Steps[0].Input["task"] != "corrected-again" || transitive.Steps[0].Input["correction"] != "use-new-plan-again" {
|
||||
t.Fatalf("retry persisted the wrong selected input: %#v", transitive.Steps[0].Input)
|
||||
}
|
||||
if transitive.Steps[0].Control == nil || transitive.Steps[0].Control.Memory["handoff"].Sequence != 1 {
|
||||
t.Fatalf("transitive retry lost prefix memory provenance: %#v", transitive.Steps[0].Control)
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -377,8 +395,8 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
|
||||
if err != nil || replayed.ID != retried.ID {
|
||||
replayed, err := r2.retry(protocol.Message{RunID: retried.ID, RequestKey: "transitive-resume-key"})
|
||||
if err != nil || replayed.ID != transitive.ID {
|
||||
t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err)
|
||||
}
|
||||
}
|
||||
@@ -399,8 +417,9 @@ case "$3" in
|
||||
*/prompt) echo '{}' ;;
|
||||
*/wait) echo '{}' ;;
|
||||
*/message*)
|
||||
case "$3" in
|
||||
case "$3" in
|
||||
*/ses_native_2/*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"invalid\\",\\"output\\":{\\"wrong\\":\\"format\\"}}"}]}]}' ;;
|
||||
*/ses_native_3/*) echo '{}' ;;
|
||||
*/ses_native_3/*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"success\\",\\"output\\":{\\"handoff\\":\\"from-b\\"}}"}]}]}' ;;
|
||||
*) echo '{"data":[{"finish":"stop","content":[{"type":"text","text":"{\\"outcome\\":\\"success\\",\\"output\\":{\\"handoff\\":\\"from-a\\",\\"summary\\":\\"done\\"}}"}]}]}' ;;
|
||||
esac ;;
|
||||
@@ -417,6 +436,7 @@ esac
|
||||
}
|
||||
defer store.Close()
|
||||
source := "# stage a\n\n```output\n{ handoff, summary }\n```\n\n```prompt\nDo A.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage b\n\n```output\n{ handoff }\n```\n\n```prompt\nDo B.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n\n# stage c\n\n```output\n{ handoff, summary }\n```\n\n```prompt\nDo C.\n```\n\n```tools\nread\nglob\ngrep\nshell\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=0\n```\n"
|
||||
source = strings.Replace(source, "# stage b\n\n```output", "# stage b\n\n```memory\n{\"task\":\"input.prompt\",\"handoff\":\"a.handoff\"}\n```\n\n```output", 1)
|
||||
if _, err := store.Submit(ctx, "native-resume", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -444,20 +464,48 @@ esac
|
||||
if err == nil || first.RunDetail == nil || first.RunDetail.Status != "failed" || len(first.RunDetail.Steps) != 2 {
|
||||
t.Fatalf("initial native run did not fail at B: %v %#v", err, first.RunDetail)
|
||||
}
|
||||
retried, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-resume-key", "--mode", "resume")
|
||||
if err != nil || retried.RunDetail == nil || retried.RunDetail.Status != "succeeded" {
|
||||
t.Fatalf("native resume failed: %v %#v", err, retried.RunDetail)
|
||||
firstInput := filepath.Join(dir, "first-input.json")
|
||||
if err := os.WriteFile(firstInput, []byte(`{"task":"corrected-first","correction":"first"}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retried.RunDetail.RetryOf != first.RunDetail.ID || len(retried.RunDetail.Steps) != 2 || retried.RunDetail.Steps[0].Stage != "b" || retried.RunDetail.Steps[1].Stage != "c" {
|
||||
retried, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-resume-key", "--mode", "resume", "--stage", "b", "--input-file", firstInput)
|
||||
if err == nil || retried.RunDetail == nil || retried.RunDetail.Status != "failed" {
|
||||
t.Fatalf("first selected native resume did not fail at B: %v %#v", err, retried.RunDetail)
|
||||
}
|
||||
if retried.RunDetail.RetryOf != first.RunDetail.ID || len(retried.RunDetail.Steps) != 1 || retried.RunDetail.Steps[0].Stage != "b" {
|
||||
t.Fatalf("native resume lost provenance or replayed prefix: %#v", retried.RunDetail)
|
||||
}
|
||||
sessions, err := os.ReadFile(filepath.Join(dir, "sessions"))
|
||||
if err != nil || strings.TrimSpace(string(sessions)) != "4" {
|
||||
t.Fatalf("native stage execution count = %q, want 4 (A once, B twice, C once): %v", sessions, err)
|
||||
secondInput := filepath.Join(dir, "second-input.json")
|
||||
if err := os.WriteFile(secondInput, []byte(`{"task":"corrected-second","correction":"second"}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := runCLI("retry", fmt.Sprint(first.RunDetail.ID), "--yes", "--json", "--request-key", "native-resume-key")
|
||||
if err != nil || again.RunDetail == nil || again.RunDetail.ID != retried.RunDetail.ID {
|
||||
t.Fatalf("same-key replay was not idempotent: %v %#v", err, again.RunDetail)
|
||||
transitive, err := runCLI("retry", fmt.Sprint(retried.RunDetail.ID), "--yes", "--json", "--wait", "--timeout", "10s", "--request-key", "native-transitive-key", "--mode", "resume", "--stage", "b", "--input-file", secondInput)
|
||||
if err != nil || transitive.RunDetail == nil || transitive.RunDetail.Status != "succeeded" {
|
||||
t.Fatalf("transitive selected native resume failed: %v %#v", err, transitive.RunDetail)
|
||||
}
|
||||
if transitive.RunDetail.RetryOf != retried.RunDetail.ID || len(transitive.RunDetail.Steps) != 2 || transitive.RunDetail.Steps[0].Stage != "b" || transitive.RunDetail.Steps[1].Stage != "c" {
|
||||
t.Fatalf("transitive native resume lost provenance or replayed prefix: %#v", transitive.RunDetail)
|
||||
}
|
||||
if transitive.RunDetail.Steps[0].Input["task"] != "corrected-second" || transitive.RunDetail.Steps[0].Input["correction"] != "second" {
|
||||
t.Fatalf("transitive native resume persisted wrong selected input: %#v", transitive.RunDetail.Steps[0].Input)
|
||||
}
|
||||
if transitive.RunDetail.Steps[0].Control == nil || transitive.RunDetail.Steps[0].Control.Memory["handoff"].Sequence != 1 {
|
||||
t.Fatalf("transitive native resume lost prefix provenance: %#v", transitive.RunDetail.Steps[0].Control)
|
||||
}
|
||||
sessions, err := os.ReadFile(filepath.Join(dir, "sessions"))
|
||||
if err != nil || strings.TrimSpace(string(sessions)) != "5" {
|
||||
t.Fatalf("native stage execution count = %q, want 5 (A once, B three times, C once): %v", sessions, err)
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err = pipeline.Open(filepath.Join(dir, "runs.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := store.GetRun(ctx, transitive.RunDetail.ID)
|
||||
if err != nil || reopened.Status != "succeeded" || reopened.RetryOf != retried.RunDetail.ID {
|
||||
t.Fatalf("transitive retry did not survive database reopen: %#v %v", reopened, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+127
-26
@@ -420,6 +420,11 @@ type retryCheckpoint struct {
|
||||
handoff map[string]any
|
||||
}
|
||||
|
||||
type retryPrefixRecord struct {
|
||||
stage string
|
||||
sequence int
|
||||
}
|
||||
|
||||
const maxRetryInputBytes = 64 * 1024
|
||||
|
||||
func retryInputOverride(base map[string]any, override map[string]any) (map[string]any, error) {
|
||||
@@ -526,14 +531,16 @@ func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.
|
||||
// stage's memory bindings must be resolved from the original prefix before
|
||||
// this override is applied at invocation time.
|
||||
handoff = map[string]any{"input": failed.Input, "override": override, "sequence": failed.Sequence, "stage": failed.Stage, "visit": failed.Sequence}
|
||||
prefix := map[string]any{}
|
||||
for _, step := range source.Steps {
|
||||
if step.Sequence >= failed.Sequence || step.Status != "completed" {
|
||||
continue
|
||||
}
|
||||
prefix[step.Stage] = step.Output
|
||||
prefix, prefixRecords, prefixErr := r.retryPrefix(sourceID, source, failed.Sequence)
|
||||
if prefixErr != nil {
|
||||
return retryCheckpoint{}, prefixErr
|
||||
}
|
||||
handoff["prefix"] = prefix
|
||||
serializedPrefix := make([]map[string]any, 0, len(prefixRecords))
|
||||
for _, record := range prefixRecords {
|
||||
serializedPrefix = append(serializedPrefix, map[string]any{"stage": record.stage, "sequence": record.sequence})
|
||||
}
|
||||
handoff["prefix_provenance"] = serializedPrefix
|
||||
// A newer explicitly selected version may continue only when every recorded
|
||||
// prefix stage still exists with the same kind. Its outputs are evidence from
|
||||
// the source attempt and cannot be safely handed to a changed stage contract.
|
||||
@@ -541,20 +548,21 @@ func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.
|
||||
for _, declared := range source.Graph.Stages {
|
||||
sourceStages[declared.Key] = declared
|
||||
}
|
||||
for _, step := range source.Steps {
|
||||
if step.Sequence >= failed.Sequence {
|
||||
break
|
||||
for _, record := range prefixRecords {
|
||||
_, found := retryPrefixStep(source, record)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
var pinnedStage *protocol.Stage
|
||||
for i := range pinned.Graph.Stages {
|
||||
if pinned.Graph.Stages[i].Key == step.Stage {
|
||||
if pinned.Graph.Stages[i].Key == record.stage {
|
||||
pinnedStage = &pinned.Graph.Stages[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
previous, ok := sourceStages[step.Stage]
|
||||
previous, ok := sourceStages[record.stage]
|
||||
if pinnedStage == nil || !ok || !retryStageContractEqual(previous, *pinnedStage) {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, step.Stage)
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, record.stage)
|
||||
}
|
||||
}
|
||||
var sourceSelected protocol.Stage
|
||||
@@ -775,7 +783,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
|
||||
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
|
||||
return
|
||||
}
|
||||
state := rebuildRetryState(source, checkpoint.failed)
|
||||
state := rebuildRetryState(source, checkpoint.failed, storedHandoff)
|
||||
if override, ok := storedHandoff["override"].(map[string]any); ok {
|
||||
state.retryOverride = stage.Values(override)
|
||||
}
|
||||
@@ -897,31 +905,124 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
|
||||
// rebuildRetryState restores only the completed prefix. The failed visit is
|
||||
// deliberately not counted and its step/evidence is not copied into the new
|
||||
// attempt, so retries cannot duplicate execution or usage aggregates.
|
||||
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState {
|
||||
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep, handoff map[string]any) *runState {
|
||||
state := newRunState(failed.Stage, stage.Values(failed.Input))
|
||||
state.nextSequence = failed.Sequence
|
||||
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
||||
state.memory["input"] = memoryValue{output: original}
|
||||
}
|
||||
for _, step := range detail.Steps {
|
||||
if step.Sequence >= failed.Sequence {
|
||||
break
|
||||
}
|
||||
state.visits[step.Stage]++
|
||||
if step.Status == "completed" {
|
||||
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
||||
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
||||
_, hasCheckpointPrefix := handoff["prefix"]
|
||||
if !hasCheckpointPrefix {
|
||||
for _, step := range detail.Steps {
|
||||
if step.Sequence >= failed.Sequence {
|
||||
break
|
||||
}
|
||||
state.visits[step.Stage]++
|
||||
if step.Status == "completed" {
|
||||
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
||||
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
||||
}
|
||||
state.completed++
|
||||
}
|
||||
state.recordChanges(step.FileChanges, step.Artifacts)
|
||||
if step.Status == "completed" {
|
||||
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
||||
}
|
||||
state.completed++
|
||||
}
|
||||
state.recordChanges(step.FileChanges, step.Artifacts)
|
||||
if step.Status == "completed" {
|
||||
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
||||
}
|
||||
// A selected-stage retry has no persisted rows for its inherited prefix.
|
||||
// Restore it from the immutable checkpoint instead of guessing from the
|
||||
// retry source's immediate steps (which is what made retry-of-retry fail
|
||||
// closed with "memory source ... has not completed").
|
||||
if prefix, ok := handoff["prefix"].(map[string]any); ok {
|
||||
for _, record := range retryPrefixRecords(handoff["prefix_provenance"]) {
|
||||
output, exists := prefix[record.stage].(map[string]any)
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
values, err := copyValues(stage.Values(output))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
state.memory[record.stage] = memoryValue{output: values, sequence: record.sequence}
|
||||
state.visits[record.stage]++
|
||||
state.completed++
|
||||
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(record.sequence)))
|
||||
}
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// retryPrefix returns the completed prefix from the source attempt plus the
|
||||
// immutable prefix carried by that attempt's checkpoint. A retry attempt only
|
||||
// persists the selected stage onward, so immediate source steps are not a
|
||||
// complete representation of its memory.
|
||||
func (r *runner) retryPrefix(sourceID int64, source *protocol.RunDetail, failedSequence int) (map[string]any, []retryPrefixRecord, error) {
|
||||
prefix := map[string]any{}
|
||||
records := []retryPrefixRecord{}
|
||||
if source.RetryMode == "resume" && source.RetrySource > 0 {
|
||||
checkpointSource, mode, _, inherited, err := r.store.RetryCheckpoint(r.ctx, sourceID)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("retry source %d has no immutable prefix checkpoint: %w", sourceID, err)
|
||||
}
|
||||
if checkpointSource != source.RetrySource || mode != "resume" {
|
||||
return nil, nil, fmt.Errorf("retry source %d has an invalid prefix checkpoint", sourceID)
|
||||
}
|
||||
if values, ok := inherited["prefix"].(map[string]any); ok {
|
||||
for key, value := range values {
|
||||
prefix[key] = value
|
||||
}
|
||||
}
|
||||
for _, record := range retryPrefixRecords(inherited["prefix_provenance"]) {
|
||||
records = append(records, record)
|
||||
}
|
||||
}
|
||||
for _, step := range source.Steps {
|
||||
if step.Sequence >= failedSequence || step.Status != "completed" {
|
||||
continue
|
||||
}
|
||||
prefix[step.Stage] = step.Output
|
||||
records = append(records, retryPrefixRecord{stage: step.Stage, sequence: step.Sequence})
|
||||
}
|
||||
return prefix, records, nil
|
||||
}
|
||||
|
||||
func retryPrefixRecords(raw any) []retryPrefixRecord {
|
||||
items, ok := raw.([]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
result := make([]retryPrefixRecord, 0, len(items))
|
||||
for _, item := range items {
|
||||
entry, ok := item.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
stageName, ok := entry["stage"].(string)
|
||||
if !ok || stageName == "" {
|
||||
continue
|
||||
}
|
||||
sequence, ok := entry["sequence"].(float64)
|
||||
if !ok || sequence < 1 {
|
||||
continue
|
||||
}
|
||||
result = append(result, retryPrefixRecord{stage: stageName, sequence: int(sequence)})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func retryPrefixStep(detail *protocol.RunDetail, record retryPrefixRecord) (protocol.RunStep, bool) {
|
||||
for _, step := range detail.Steps {
|
||||
if step.Stage == record.stage && step.Sequence == record.sequence {
|
||||
return step, true
|
||||
}
|
||||
}
|
||||
// Inherited rows intentionally are not present in a selected retry. The
|
||||
// contract check still uses the stage key; this marker keeps that check
|
||||
// independent from model-claimed output.
|
||||
return protocol.RunStep{Stage: record.stage, Sequence: record.sequence}, true
|
||||
}
|
||||
|
||||
// artifactAccess derives exact visit directories; it never consumes a path from
|
||||
// a definition or model result. An unreadable or symlinked root fails closed.
|
||||
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
|
||||
|
||||
Reference in New Issue
Block a user