feat: resume retries from failed stage

This commit is contained in:
2026-10-18 20:38:03 +01:00
parent b79d0401f3
commit b8beeb250b
14 changed files with 336 additions and 19 deletions
+11
View File
@@ -0,0 +1,11 @@
# Retry from failed stage
As of 2026-10-18, retry requests default to `resume`: the server records a
durable checkpoint for the terminal failed visit and starts the new attempt at
that visit with its completed-prefix handoff context. `--mode=restart` retains
the older full-restart behavior. Older clients that omit the mode therefore
resume rather than restart.
The CLI and protocol now expose the mode and retry provenance. The shared CLI
documentation and packaged skill references should be folded from this note by
the manual `docs-fold` pipeline.
+6 -1
View File
@@ -37,6 +37,7 @@ type App struct {
type options struct {
harness string
url, name, prompt, title, template, workspace, paths, key, opencode, model string
retryMode string
option, feedback, evidence, actor string
output string
conflict string
@@ -162,6 +163,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)")
f.BoolVar(&o.wait, "wait", false, "Wait for the new run to finish")
f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait (0 waits until the run ends); independent of --timeout")
f.StringVar(&o.retryMode, "mode", "resume", "Retry mode: resume failed stage or restart from entry")
case "cancel", "draft-delete":
f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt")
case "retention":
@@ -983,11 +985,14 @@ func (a App) Run(ctx context.Context, args []string) error {
}
typeName = protocol.RetryRun
key = o.key
if o.retryMode != "resume" && o.retryMode != "restart" {
return fmt.Errorf("--mode must be resume or restart")
}
if key == "" {
key = fmt.Sprintf("retry-%d-%d", os.Getpid(), time.Now().UnixNano())
}
}
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key})
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key, RetryMode: o.retryMode})
if err != nil {
return err
}
+2 -2
View File
@@ -23,7 +23,7 @@ var usage = map[string]string{
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
"cancel": "cancel <run-id> [--json]",
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
"retry": "retry <run-id> [--mode resume|restart] [--json [--wait [--wait-timeout DURATION]]]",
"title": "title <run-id> \"<title>\" [--json]",
"debug": "debug <run-id> [--json | --no-open]",
"chat-adopt": "chat-adopt <session-id> [--json]",
@@ -51,7 +51,7 @@ var notes = map[string]string{
"validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.",
"submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
"retry": "Start a new attempt of a failed run with the same initial input, directory, pipeline version, and edit scope. Original history is retained. Already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only with `inspect <run-id> --json`, or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
"retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. Original history is retained and already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
"title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.",
"wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).",
"decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.",
+1 -1
View File
@@ -29,7 +29,7 @@ EXECUTE
wait <run-id> ... Block until runs finish, then report each (read-only)
cancel <run-id> Cancel a run; changes are NOT rolled back
decide <run-id> [stage] Choose an option for a run waiting on a person
retry <run-id> Restart a failed run with its original input and directory
retry <run-id> Resume a failed stage (use --mode=restart for a full restart)
title <run-id> "<title>" Rename a run after the fact; input/steps unchanged
debug <run-id> Investigate a run with a configured harness or nvim
chat-adopt <session-id> Adopt an existing OpenCode session into chat_sessions
+52
View File
@@ -0,0 +1,52 @@
package pipeline
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// SaveRetryCheckpoint records the immutable source and the completed-prefix
// handoff before a retry is launched. It intentionally stores no step or usage
// rows: those remain evidence belonging only to the source attempt.
func (s *Store) SaveRetryCheckpoint(ctx context.Context, runID, sourceID int64, mode string, sequence int, source *protocol.RunDetail, handoff map[string]any) error {
if mode == "" {
mode = "resume"
}
data, err := json.Marshal(handoff)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
var oldSource int64
var oldMode, oldHandoff, oldHash string
var oldSeq, oldVersion int
err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&oldSource, &oldMode, &oldSeq, &oldHandoff, &oldVersion, &oldHash)
if err == nil {
if oldSource != sourceID || oldMode != mode || oldSeq != sequence || oldHandoff != string(data) || oldVersion != source.Version || oldHash != source.ContentHash {
return fmt.Errorf("retry checkpoint conflicts with the existing request")
}
return nil
}
if err != sql.ErrNoRows {
return err
}
_, err = s.db.ExecContext(ctx, `INSERT INTO retry_checkpoints(run_id,source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash) VALUES(?,?,?,?,?,?,?)`, runID, sourceID, mode, sequence, string(data), source.Version, source.ContentHash)
return err
}
func (s *Store) RetryCheckpoint(ctx context.Context, runID int64) (sourceID int64, mode string, sequence int, handoff map[string]any, err error) {
s.mu.Lock()
defer s.mu.Unlock()
var data string
err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&sourceID, &mode, &sequence, &data)
if err != nil {
return 0, "", 0, nil, err
}
err = json.Unmarshal([]byte(data), &handoff)
return
}
+1 -1
View File
@@ -100,7 +100,7 @@ func (s *Store) GetRun(ctx context.Context, id int64) (*protocol.RunDetail, erro
if err := json.Unmarshal([]byte(graphJSON), &r.Graph); err != nil {
return nil, err
}
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.TitleSource, &r.TitleModel); err != nil {
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT mode FROM retry_checkpoints WHERE run_id=pipeline_runs.id),CASE WHEN EXISTS(SELECT 1 FROM run_retries WHERE run_id=pipeline_runs.id) THEN 'restart' ELSE '' END),COALESCE((SELECT source_run_id FROM retry_checkpoints WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.RetryMode, &r.RetrySource, &r.TitleSource, &r.TitleModel); err != nil {
return nil, err
}
if err := decodeValues(input, &r.Input); err != nil {
+4
View File
@@ -52,6 +52,10 @@ run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, refer
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
CREATE TABLE IF NOT EXISTS run_retries (
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
CREATE TABLE IF NOT EXISTS retry_checkpoints (
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL,
source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
CREATE TABLE IF NOT EXISTS resource_owners (
resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL,
+4
View File
@@ -159,6 +159,10 @@ type Message struct {
// them; the same fields are echoed on the run record.
Model string `json:"model,omitempty"`
StageModels map[string]string `json:"stage_models,omitempty"`
// Empty (or "resume") retries the failed visit; "restart" starts at the
// entry stage. Older clients omit this field and retain resume behavior.
RetryMode string `json:"retry_mode,omitempty"`
RetrySource int64 `json:"-"`
// Search is the broad, read-only search traffic over pipelines, versions,
// runs, steps, field text and artifacts. See search.go for the per-type
// payloads and the ranking/kind/bound vocabulary.
+10 -8
View File
@@ -370,14 +370,16 @@ type NotificationReceipt struct {
type RunDetail struct {
RunSummary
Graph Graph `json:"graph"`
RetryOf int64 `json:"retry_of,omitempty"`
Workspace string `json:"workspace"`
AllowTools bool `json:"allow_tools"`
WritePaths []string `json:"write_paths"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
Graph Graph `json:"graph"`
RetryOf int64 `json:"retry_of,omitempty"`
RetryMode string `json:"retry_mode,omitempty"`
RetrySource int64 `json:"retry_source,omitempty"`
Workspace string `json:"workspace"`
AllowTools bool `json:"allow_tools"`
WritePaths []string `json:"write_paths"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Steps []RunStep `json:"steps"`
// Usage is the totalled reported usage of the run's model stages, summed in
// one pass over Steps. It is absent for a run with no model usage recorded
// (including historical runs).
+1 -1
View File
@@ -270,7 +270,7 @@ func TestHumanStageCancelRetryAndRebuild(t *testing.T) {
t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary)
}
r3.execute = successExecute
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human"})
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"})
if err != nil {
t.Fatal(err)
}
+2 -2
View File
@@ -43,14 +43,14 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) {
if err != nil || !runs[0].Executing {
t.Fatal("list advertised stopped run during cleanup", runs, err)
}
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err == nil || !strings.Contains(err.Error(), "still executing") {
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err == nil || !strings.Contains(err.Error(), "still executing") {
t.Fatal("retry accepted during execution cleanup", err)
}
delete(r.active, id)
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
}
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err != nil {
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err != nil {
t.Fatal("retry rejected after cleanup", err)
}
r.wg.Wait()
+91 -2
View File
@@ -183,7 +183,7 @@ func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
if err != nil || len(beforeRetry) != 1 {
t.Fatal("unconfirmed retry created a run", err)
}
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key")
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key", "--mode", "restart")
if err != nil {
t.Fatal(err)
}
@@ -236,7 +236,7 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
t.Fatal(err)
}
}
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status})
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status, RetryMode: "restart"})
if err == nil {
t.Fatal("invalid retry accepted", status)
}
@@ -249,6 +249,95 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
}
}
func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir := t.TempDir()
store, err := pipeline.Open(filepath.Join(dir, "resume.db"))
if err != nil {
t.Fatal(err)
}
source := "# stage a\n\n```output\nA\n```\n\n```prompt\nA\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage b\n\n```output\nB\n```\n\n```prompt\nB\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage c\n\n```output\nC\n```\n\n```prompt\nC\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n"
if _, err := store.Submit(ctx, "resume", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
workspace := t.TempDir()
var aCalls, bCalls, cCalls int
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
switch def.Key {
case "a":
aCalls++
if input["prompt"] != "task" {
t.Errorf("entry received altered input: %#v", input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-a"}}, nil
case "b":
bCalls++
if input["handoff"] != "from-a" {
t.Errorf("resume lost exact prefix handoff: %#v", input)
}
if bCalls == 1 {
return stage.Result{}, fmt.Errorf("deliberate failure")
}
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil
case "c":
cCalls++
if input["handoff"] != "from-b" {
t.Errorf("C received wrong handoff: %#v", input)
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
default:
t.Fatalf("unexpected stage %q", def.Key)
return stage.Result{}, nil
}
}
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
original, err = store.GetRun(ctx, original.ID)
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
t.Fatalf("initial run did not stop at B: %+v %v", original, err)
}
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
retried, err = store.GetRun(ctx, retried.ID)
if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
t.Fatalf("resume did not complete with provenance: %+v %v", retried, err)
}
if aCalls != 1 || bCalls != 2 || cCalls != 1 {
t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls)
}
if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" {
t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = pipeline.Open(filepath.Join(dir, "resume.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
r2, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
if err != nil || replayed.ID != retried.ID {
t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err)
}
}
// TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves,
// through the real runner and CLI fixture, that a read-only stage retries a
// post-admission transport (final-readback) failure with a fresh session while
+145 -1
View File
@@ -285,6 +285,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
}
if created {
if request.RetrySource > 0 && request.RetryMode != "restart" {
checkpoint, checkpointErr := r.retrySourceCheckpoint(request.RetrySource, d)
if checkpointErr != nil {
return nil, checkpointErr
}
if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil {
return nil, err
}
}
// Emit the created delta before launching: the queued frame reaches
// subscribers before the running/step-started transition, so a client
// sees the run appear and then advance.
@@ -404,6 +413,65 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
return items, err
}
type retryCheckpoint struct {
detail *protocol.RunDetail
failed protocol.RunStep
handoff map[string]any
}
func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) {
source, err := r.store.GetRun(r.ctx, sourceID)
if err != nil {
return retryCheckpoint{}, err
}
if source.Status != "failed" || source.FinishedAt == "" {
return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID)
}
if source.PipelineID != pinned.ID || source.Version != pinned.Version || source.ContentHash != pinned.ContentHash {
return retryCheckpoint{}, fmt.Errorf("retry source %d is pinned to a different pipeline version", sourceID)
}
if len(source.Steps) == 0 {
return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID)
}
failed := source.Steps[len(source.Steps)-1]
if (failed.Status != "completed" && failed.Status != "failed") || failed.Target != protocol.FailureEnd {
return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID)
}
var handoff map[string]any
if failed.Input == nil {
return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID)
}
handoff = map[string]any{"input": failed.Input, "sequence": failed.Sequence, "stage": failed.Stage}
prefix := map[string]any{}
for _, step := range source.Steps {
if step.Sequence >= failed.Sequence || step.Status != "completed" {
continue
}
prefix[step.Stage] = step.Output
}
handoff["prefix"] = prefix
for _, declared := range pinned.Graph.Stages {
if declared.Key != failed.Stage {
continue
}
typ := stage.EffectiveType(declared.Type)
if typ == stage.HumanType || typ == stage.NtfyType {
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage)
}
if typ == stage.ModelType {
tools, explicit, toolErr := stage.ToolNames(pipeline.StageDefinition(declared))
if toolErr != nil {
return retryCheckpoint{}, toolErr
}
if explicit && len(tools) > 0 {
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot safely replay admitted native stage %s", sourceID, failed.Stage)
}
}
return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil
}
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID)
}
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("retry requires a positive run ID")
@@ -418,7 +486,26 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if original.Status != "failed" || original.FinishedAt == "" {
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
if request.RequestKey == "" {
return nil, fmt.Errorf("retry requires an idempotency request key")
}
mode := request.RetryMode
if mode == "" {
mode = "resume"
}
if mode != "resume" && mode != "restart" {
return nil, fmt.Errorf("retry mode must be resume or restart")
}
d, err := r.store.Get(r.ctx, original.PipelineID, original.Version)
if err != nil {
return nil, err
}
if mode == "resume" {
if _, err := r.retrySourceCheckpoint(original.ID, d); err != nil {
return nil, err
}
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
}
func (r *runner) cancel(id int64) error {
@@ -520,6 +607,35 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
}
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
if request.RetrySource > 0 && request.RetryMode != "restart" {
source, err := r.store.GetRun(ctx, request.RetrySource)
if err != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
return
}
checkpoint, err := r.retrySourceCheckpoint(request.RetrySource, d)
if err != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
return
}
storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id)
if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil {
if checkpointErr == nil {
checkpointErr = fmt.Errorf("retry checkpoint is incomplete")
}
r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil)
return
}
if input, ok := storedHandoff["input"].(map[string]any); ok {
checkpoint.failed.Input = input
} else {
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
return
}
state := rebuildRetryState(source, checkpoint.failed)
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
return
}
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
@@ -629,6 +745,34 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
return state
}
// rebuildRetryState restores only the completed prefix. The failed visit is
// deliberately not counted and its step/evidence is not copied into the new
// attempt, so retries cannot duplicate execution or usage aggregates.
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState {
state := newRunState(failed.Stage, stage.Values(failed.Input))
state.nextSequence = failed.Sequence
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
state.memory["input"] = memoryValue{output: original}
}
for _, step := range detail.Steps {
if step.Sequence >= failed.Sequence {
break
}
state.visits[step.Stage]++
if step.Status == "completed" {
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
state.completed++
}
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
}
}
return state
}
// artifactAccess derives exact visit directories; it never consumes a path from
// a definition or model result. An unreadable or symlinked root fails closed.
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
+6
View File
@@ -32,6 +32,12 @@ func TestRetryShortcutAndPaletteSendServerRetryWithoutReplacingDirectory(t *test
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "original task"}); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "failed", Outcome: "error", Target: protocol.FailureEnd, Input: map[string]any{"prompt": "original task"}}); err != nil {
t.Fatal(err)
}
if err := store.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil {
t.Fatal(err)
}