diff --git a/docs/notes/retry-from-failed-stage.md b/docs/notes/retry-from-failed-stage.md new file mode 100644 index 00000000..57dd006d --- /dev/null +++ b/docs/notes/retry-from-failed-stage.md @@ -0,0 +1,11 @@ +# Retry from failed stage + +As of 2026-10-18, retry requests default to `resume`: the server records a +durable checkpoint for the terminal failed visit and starts the new attempt at +that visit with its completed-prefix handoff context. `--mode=restart` retains +the older full-restart behavior. Older clients that omit the mode therefore +resume rather than restart. + +The CLI and protocol now expose the mode and retry provenance. The shared CLI +documentation and packaged skill references should be folded from this note by +the manual `docs-fold` pipeline. diff --git a/internal/cli/app.go b/internal/cli/app.go index e25c493c..cadada39 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -37,6 +37,7 @@ type App struct { type options struct { harness string url, name, prompt, title, template, workspace, paths, key, opencode, model string + retryMode string option, feedback, evidence, actor string output string conflict string @@ -162,6 +163,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) { f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)") f.BoolVar(&o.wait, "wait", false, "Wait for the new run to finish") f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait (0 waits until the run ends); independent of --timeout") + f.StringVar(&o.retryMode, "mode", "resume", "Retry mode: resume failed stage or restart from entry") case "cancel", "draft-delete": f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt") case "retention": @@ -983,11 +985,14 @@ func (a App) Run(ctx context.Context, args []string) error { } typeName = protocol.RetryRun key = o.key + if o.retryMode != "resume" && o.retryMode != "restart" { + return fmt.Errorf("--mode must be resume or restart") + } if key == "" { key = fmt.Sprintf("retry-%d-%d", os.Getpid(), time.Now().UnixNano()) } } - r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key}) + r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key, RetryMode: o.retryMode}) if err != nil { return err } diff --git a/internal/cli/commands.go b/internal/cli/commands.go index a78a8d2d..f1e7f9fb 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -23,7 +23,7 @@ var usage = map[string]string{ "wait": "wait [run-id ...] [--timeout DURATION] [--json]", "cancel": "cancel [--json]", "decide": "decide [stage] --option KEY [--feedback TEXT] [--json]", - "retry": "retry [--json [--wait [--wait-timeout DURATION]]]", + "retry": "retry [--mode resume|restart] [--json [--wait [--wait-timeout DURATION]]]", "title": "title \"\" [--json]", "debug": "debug <run-id> [--json | --no-open]", "chat-adopt": "chat-adopt <session-id> [--json]", @@ -51,7 +51,7 @@ var notes = map[string]string{ "validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.", "submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.", "run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).", - "retry": "Start a new attempt of a failed run with the same initial input, directory, pipeline version, and edit scope. Original history is retained. Already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only with `inspect <run-id> --json`, or reattach with --wait and the original --request-key instead of starting a duplicate retry.", + "retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. Original history is retained and already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only or reattach with --wait and the original --request-key instead of starting a duplicate retry.", "title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.", "wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).", "decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.", diff --git a/internal/cli/help.go b/internal/cli/help.go index dd3585ef..51ddf8d7 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -29,7 +29,7 @@ EXECUTE wait <run-id> ... Block until runs finish, then report each (read-only) cancel <run-id> Cancel a run; changes are NOT rolled back decide <run-id> [stage] Choose an option for a run waiting on a person - retry <run-id> Restart a failed run with its original input and directory + retry <run-id> Resume a failed stage (use --mode=restart for a full restart) title <run-id> "<title>" Rename a run after the fact; input/steps unchanged debug <run-id> Investigate a run with a configured harness or nvim chat-adopt <session-id> Adopt an existing OpenCode session into chat_sessions diff --git a/internal/pipeline/retry_checkpoints.go b/internal/pipeline/retry_checkpoints.go new file mode 100644 index 00000000..af8de1f7 --- /dev/null +++ b/internal/pipeline/retry_checkpoints.go @@ -0,0 +1,52 @@ +package pipeline + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// SaveRetryCheckpoint records the immutable source and the completed-prefix +// handoff before a retry is launched. It intentionally stores no step or usage +// rows: those remain evidence belonging only to the source attempt. +func (s *Store) SaveRetryCheckpoint(ctx context.Context, runID, sourceID int64, mode string, sequence int, source *protocol.RunDetail, handoff map[string]any) error { + if mode == "" { + mode = "resume" + } + data, err := json.Marshal(handoff) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + var oldSource int64 + var oldMode, oldHandoff, oldHash string + var oldSeq, oldVersion int + err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&oldSource, &oldMode, &oldSeq, &oldHandoff, &oldVersion, &oldHash) + if err == nil { + if oldSource != sourceID || oldMode != mode || oldSeq != sequence || oldHandoff != string(data) || oldVersion != source.Version || oldHash != source.ContentHash { + return fmt.Errorf("retry checkpoint conflicts with the existing request") + } + return nil + } + if err != sql.ErrNoRows { + return err + } + _, err = s.db.ExecContext(ctx, `INSERT INTO retry_checkpoints(run_id,source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash) VALUES(?,?,?,?,?,?,?)`, runID, sourceID, mode, sequence, string(data), source.Version, source.ContentHash) + return err +} + +func (s *Store) RetryCheckpoint(ctx context.Context, runID int64) (sourceID int64, mode string, sequence int, handoff map[string]any, err error) { + s.mu.Lock() + defer s.mu.Unlock() + var data string + err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&sourceID, &mode, &sequence, &data) + if err != nil { + return 0, "", 0, nil, err + } + err = json.Unmarshal([]byte(data), &handoff) + return +} diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 76e1de11..0ff3577e 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -100,7 +100,7 @@ func (s *Store) GetRun(ctx context.Context, id int64) (*protocol.RunDetail, erro if err := json.Unmarshal([]byte(graphJSON), &r.Graph); err != nil { return nil, err } - if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.TitleSource, &r.TitleModel); err != nil { + if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT mode FROM retry_checkpoints WHERE run_id=pipeline_runs.id),CASE WHEN EXISTS(SELECT 1 FROM run_retries WHERE run_id=pipeline_runs.id) THEN 'restart' ELSE '' END),COALESCE((SELECT source_run_id FROM retry_checkpoints WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.RetryMode, &r.RetrySource, &r.TitleSource, &r.TitleModel); err != nil { return nil, err } if err := decodeValues(input, &r.Input); err != nil { diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index 2c9414ef..ff766442 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -52,6 +52,10 @@ run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, refer PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence)); CREATE TABLE IF NOT EXISTS run_retries ( run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id)); +CREATE TABLE IF NOT EXISTS retry_checkpoints ( + run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), + mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL, + source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); CREATE TABLE IF NOT EXISTS resource_owners ( resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id), stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL, diff --git a/internal/protocol/message.go b/internal/protocol/message.go index b3b8e12f..0d7eb45d 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -159,6 +159,10 @@ type Message struct { // them; the same fields are echoed on the run record. Model string `json:"model,omitempty"` StageModels map[string]string `json:"stage_models,omitempty"` + // Empty (or "resume") retries the failed visit; "restart" starts at the + // entry stage. Older clients omit this field and retain resume behavior. + RetryMode string `json:"retry_mode,omitempty"` + RetrySource int64 `json:"-"` // Search is the broad, read-only search traffic over pipelines, versions, // runs, steps, field text and artifacts. See search.go for the per-type // payloads and the ranking/kind/bound vocabulary. diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index fb3f2830..ae9b98ea 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -370,14 +370,16 @@ type NotificationReceipt struct { type RunDetail struct { RunSummary - Graph Graph `json:"graph"` - RetryOf int64 `json:"retry_of,omitempty"` - Workspace string `json:"workspace"` - AllowTools bool `json:"allow_tools"` - WritePaths []string `json:"write_paths"` - Input map[string]any `json:"input"` - Output map[string]any `json:"output"` - Steps []RunStep `json:"steps"` + Graph Graph `json:"graph"` + RetryOf int64 `json:"retry_of,omitempty"` + RetryMode string `json:"retry_mode,omitempty"` + RetrySource int64 `json:"retry_source,omitempty"` + Workspace string `json:"workspace"` + AllowTools bool `json:"allow_tools"` + WritePaths []string `json:"write_paths"` + Input map[string]any `json:"input"` + Output map[string]any `json:"output"` + Steps []RunStep `json:"steps"` // Usage is the totalled reported usage of the run's model stages, summed in // one pass over Steps. It is absent for a run with no model usage recorded // (including historical runs). diff --git a/internal/server/human_test.go b/internal/server/human_test.go index c5ac1dfd..30d9307c 100644 --- a/internal/server/human_test.go +++ b/internal/server/human_test.go @@ -270,7 +270,7 @@ func TestHumanStageCancelRetryAndRebuild(t *testing.T) { t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary) } r3.execute = successExecute - retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human"}) + retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"}) if err != nil { t.Fatal(err) } diff --git a/internal/server/retry_cleanup_test.go b/internal/server/retry_cleanup_test.go index ffe00230..a82a4f45 100644 --- a/internal/server/retry_cleanup_test.go +++ b/internal/server/retry_cleanup_test.go @@ -43,14 +43,14 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) { if err != nil || !runs[0].Executing { t.Fatal("list advertised stopped run during cleanup", runs, err) } - if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err == nil || !strings.Contains(err.Error(), "still executing") { + if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err == nil || !strings.Contains(err.Error(), "still executing") { t.Fatal("retry accepted during execution cleanup", err) } delete(r.active, id) r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil } - if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err != nil { + if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err != nil { t.Fatal("retry rejected after cleanup", err) } r.wg.Wait() diff --git a/internal/server/retry_test.go b/internal/server/retry_test.go index 152201ce..93f5f279 100644 --- a/internal/server/retry_test.go +++ b/internal/server/retry_test.go @@ -183,7 +183,7 @@ func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) { if err != nil || len(beforeRetry) != 1 { t.Fatal("unconfirmed retry created a run", err) } - result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key") + result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key", "--mode", "restart") if err != nil { t.Fatal(err) } @@ -236,7 +236,7 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T) t.Fatal(err) } } - _, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status}) + _, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status, RetryMode: "restart"}) if err == nil { t.Fatal("invalid retry accepted", status) } @@ -249,6 +249,95 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T) } } +func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dir := t.TempDir() + store, err := pipeline.Open(filepath.Join(dir, "resume.db")) + if err != nil { + t.Fatal(err) + } + source := "# stage a\n\n```output\nA\n```\n\n```prompt\nA\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage b\n\n```output\nB\n```\n\n```prompt\nB\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage c\n\n```output\nC\n```\n\n```prompt\nC\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n" + if _, err := store.Submit(ctx, "resume", source); err != nil { + t.Fatal(err) + } + items, _ := store.List(ctx) + workspace := t.TempDir() + var aCalls, bCalls, cCalls int + r, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) { + switch def.Key { + case "a": + aCalls++ + if input["prompt"] != "task" { + t.Errorf("entry received altered input: %#v", input) + } + return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-a"}}, nil + case "b": + bCalls++ + if input["handoff"] != "from-a" { + t.Errorf("resume lost exact prefix handoff: %#v", input) + } + if bCalls == 1 { + return stage.Result{}, fmt.Errorf("deliberate failure") + } + return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil + case "c": + cCalls++ + if input["handoff"] != "from-b" { + t.Errorf("C received wrong handoff: %#v", input) + } + return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil + default: + t.Fatalf("unexpected stage %q", def.Key) + return stage.Result{}, nil + } + } + original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, Input: map[string]any{"prompt": "task"}}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + original, err = store.GetRun(ctx, original.ID) + if err != nil || original.Status != "failed" || len(original.Steps) != 2 { + t.Fatalf("initial run did not stop at B: %+v %v", original, err) + } + retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"}) + if err != nil { + t.Fatal(err) + } + r.wg.Wait() + retried, err = store.GetRun(ctx, retried.ID) + if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID { + t.Fatalf("resume did not complete with provenance: %+v %v", retried, err) + } + if aCalls != 1 || bCalls != 2 || cCalls != 1 { + t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls) + } + if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" { + t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + store, err = pipeline.Open(filepath.Join(dir, "resume.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + r2, err := newRunner(ctx, store) + if err != nil { + t.Fatal(err) + } + replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"}) + if err != nil || replayed.ID != retried.ID { + t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err) + } +} + // TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves, // through the real runner and CLI fixture, that a read-only stage retries a // post-admission transport (final-readback) failure with a fresh session while diff --git a/internal/server/runner.go b/internal/server/runner.go index 260397fe..4c722216 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -285,6 +285,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) } if created { + if request.RetrySource > 0 && request.RetryMode != "restart" { + checkpoint, checkpointErr := r.retrySourceCheckpoint(request.RetrySource, d) + if checkpointErr != nil { + return nil, checkpointErr + } + if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil { + return nil, err + } + } // Emit the created delta before launching: the queued frame reaches // subscribers before the running/step-started transition, so a client // sees the run appear and then advance. @@ -404,6 +413,65 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) { return items, err } +type retryCheckpoint struct { + detail *protocol.RunDetail + failed protocol.RunStep + handoff map[string]any +} + +func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) { + source, err := r.store.GetRun(r.ctx, sourceID) + if err != nil { + return retryCheckpoint{}, err + } + if source.Status != "failed" || source.FinishedAt == "" { + return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID) + } + if source.PipelineID != pinned.ID || source.Version != pinned.Version || source.ContentHash != pinned.ContentHash { + return retryCheckpoint{}, fmt.Errorf("retry source %d is pinned to a different pipeline version", sourceID) + } + if len(source.Steps) == 0 { + return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID) + } + failed := source.Steps[len(source.Steps)-1] + if (failed.Status != "completed" && failed.Status != "failed") || failed.Target != protocol.FailureEnd { + return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID) + } + var handoff map[string]any + if failed.Input == nil { + return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID) + } + handoff = map[string]any{"input": failed.Input, "sequence": failed.Sequence, "stage": failed.Stage} + prefix := map[string]any{} + for _, step := range source.Steps { + if step.Sequence >= failed.Sequence || step.Status != "completed" { + continue + } + prefix[step.Stage] = step.Output + } + handoff["prefix"] = prefix + for _, declared := range pinned.Graph.Stages { + if declared.Key != failed.Stage { + continue + } + typ := stage.EffectiveType(declared.Type) + if typ == stage.HumanType || typ == stage.NtfyType { + return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage) + } + if typ == stage.ModelType { + tools, explicit, toolErr := stage.ToolNames(pipeline.StageDefinition(declared)) + if toolErr != nil { + return retryCheckpoint{}, toolErr + } + if explicit && len(tools) > 0 { + return retryCheckpoint{}, fmt.Errorf("retry source %d cannot safely replay admitted native stage %s", sourceID, failed.Stage) + } + } + return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil + } + return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID) +} + func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("retry requires a positive run ID") @@ -418,7 +486,26 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if original.Status != "failed" || original.FinishedAt == "" { return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status) } - return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID) + if request.RequestKey == "" { + return nil, fmt.Errorf("retry requires an idempotency request key") + } + mode := request.RetryMode + if mode == "" { + mode = "resume" + } + if mode != "resume" && mode != "restart" { + return nil, fmt.Errorf("retry mode must be resume or restart") + } + d, err := r.store.Get(r.ctx, original.PipelineID, original.Version) + if err != nil { + return nil, err + } + if mode == "resume" { + if _, err := r.retrySourceCheckpoint(original.ID, d); err != nil { + return nil, err + } + } + return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID) } func (r *runner) cancel(id int64) error { @@ -520,6 +607,35 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { } func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) { + if request.RetrySource > 0 && request.RetryMode != "restart" { + source, err := r.store.GetRun(ctx, request.RetrySource) + if err != nil { + r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) + return + } + checkpoint, err := r.retrySourceCheckpoint(request.RetrySource, d) + if err != nil { + r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) + return + } + storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id) + if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil { + if checkpointErr == nil { + checkpointErr = fmt.Errorf("retry checkpoint is incomplete") + } + r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil) + return + } + if input, ok := storedHandoff["input"].(map[string]any); ok { + checkpoint.failed.Input = input + } else { + r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil) + return + } + state := rebuildRetryState(source, checkpoint.failed) + r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state) + return + } original, copyErr := copyValues(stage.Values(request.Input)) if copyErr != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input)) @@ -629,6 +745,34 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS return state } +// rebuildRetryState restores only the completed prefix. The failed visit is +// deliberately not counted and its step/evidence is not copied into the new +// attempt, so retries cannot duplicate execution or usage aggregates. +func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState { + state := newRunState(failed.Stage, stage.Values(failed.Input)) + state.nextSequence = failed.Sequence + if original, err := copyValues(stage.Values(detail.Input)); err == nil { + state.memory["input"] = memoryValue{output: original} + } + for _, step := range detail.Steps { + if step.Sequence >= failed.Sequence { + break + } + state.visits[step.Stage]++ + if step.Status == "completed" { + if values, err := copyValues(stage.Values(step.Output)); err == nil { + state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} + } + state.completed++ + } + state.recordChanges(step.FileChanges, step.Artifacts) + if step.Status == "completed" { + state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) + } + } + return state +} + // artifactAccess derives exact visit directories; it never consumes a path from // a definition or model result. An unreadable or symlinked root fails closed. func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess { diff --git a/internal/tui/retry_test.go b/internal/tui/retry_test.go index 904e9a13..85d3a3d0 100644 --- a/internal/tui/retry_test.go +++ b/internal/tui/retry_test.go @@ -32,6 +32,12 @@ func TestRetryShortcutAndPaletteSendServerRetryWithoutReplacingDirectory(t *test if err != nil { t.Fatal(err) } + if err := store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "original task"}); err != nil { + t.Fatal(err) + } + if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "failed", Outcome: "error", Target: protocol.FailureEnd, Input: map[string]any{"prompt": "original task"}}); err != nil { + t.Fatal(err) + } if err := store.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil { t.Fatal(err) }