feat: resume retries from failed stage
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
# Retry from failed stage
|
||||
|
||||
As of 2026-10-18, retry requests default to `resume`: the server records a
|
||||
durable checkpoint for the terminal failed visit and starts the new attempt at
|
||||
that visit with its completed-prefix handoff context. `--mode=restart` retains
|
||||
the older full-restart behavior. Older clients that omit the mode therefore
|
||||
resume rather than restart.
|
||||
|
||||
The CLI and protocol now expose the mode and retry provenance. The shared CLI
|
||||
documentation and packaged skill references should be folded from this note by
|
||||
the manual `docs-fold` pipeline.
|
||||
+6
-1
@@ -37,6 +37,7 @@ type App struct {
|
||||
type options struct {
|
||||
harness string
|
||||
url, name, prompt, title, template, workspace, paths, key, opencode, model string
|
||||
retryMode string
|
||||
option, feedback, evidence, actor string
|
||||
output string
|
||||
conflict string
|
||||
@@ -162,6 +163,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)")
|
||||
f.BoolVar(&o.wait, "wait", false, "Wait for the new run to finish")
|
||||
f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait (0 waits until the run ends); independent of --timeout")
|
||||
f.StringVar(&o.retryMode, "mode", "resume", "Retry mode: resume failed stage or restart from entry")
|
||||
case "cancel", "draft-delete":
|
||||
f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt")
|
||||
case "retention":
|
||||
@@ -983,11 +985,14 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
}
|
||||
typeName = protocol.RetryRun
|
||||
key = o.key
|
||||
if o.retryMode != "resume" && o.retryMode != "restart" {
|
||||
return fmt.Errorf("--mode must be resume or restart")
|
||||
}
|
||||
if key == "" {
|
||||
key = fmt.Sprintf("retry-%d-%d", os.Getpid(), time.Now().UnixNano())
|
||||
}
|
||||
}
|
||||
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key})
|
||||
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key, RetryMode: o.retryMode})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ var usage = map[string]string{
|
||||
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
|
||||
"cancel": "cancel <run-id> [--json]",
|
||||
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
|
||||
"retry": "retry <run-id> [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"retry": "retry <run-id> [--mode resume|restart] [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"title": "title <run-id> \"<title>\" [--json]",
|
||||
"debug": "debug <run-id> [--json | --no-open]",
|
||||
"chat-adopt": "chat-adopt <session-id> [--json]",
|
||||
@@ -51,7 +51,7 @@ var notes = map[string]string{
|
||||
"validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.",
|
||||
"submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
|
||||
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
|
||||
"retry": "Start a new attempt of a failed run with the same initial input, directory, pipeline version, and edit scope. Original history is retained. Already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only with `inspect <run-id> --json`, or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
|
||||
"retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. Original history is retained and already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
|
||||
"title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.",
|
||||
"wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).",
|
||||
"decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.",
|
||||
|
||||
@@ -29,7 +29,7 @@ EXECUTE
|
||||
wait <run-id> ... Block until runs finish, then report each (read-only)
|
||||
cancel <run-id> Cancel a run; changes are NOT rolled back
|
||||
decide <run-id> [stage] Choose an option for a run waiting on a person
|
||||
retry <run-id> Restart a failed run with its original input and directory
|
||||
retry <run-id> Resume a failed stage (use --mode=restart for a full restart)
|
||||
title <run-id> "<title>" Rename a run after the fact; input/steps unchanged
|
||||
debug <run-id> Investigate a run with a configured harness or nvim
|
||||
chat-adopt <session-id> Adopt an existing OpenCode session into chat_sessions
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// SaveRetryCheckpoint records the immutable source and the completed-prefix
|
||||
// handoff before a retry is launched. It intentionally stores no step or usage
|
||||
// rows: those remain evidence belonging only to the source attempt.
|
||||
func (s *Store) SaveRetryCheckpoint(ctx context.Context, runID, sourceID int64, mode string, sequence int, source *protocol.RunDetail, handoff map[string]any) error {
|
||||
if mode == "" {
|
||||
mode = "resume"
|
||||
}
|
||||
data, err := json.Marshal(handoff)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var oldSource int64
|
||||
var oldMode, oldHandoff, oldHash string
|
||||
var oldSeq, oldVersion int
|
||||
err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&oldSource, &oldMode, &oldSeq, &oldHandoff, &oldVersion, &oldHash)
|
||||
if err == nil {
|
||||
if oldSource != sourceID || oldMode != mode || oldSeq != sequence || oldHandoff != string(data) || oldVersion != source.Version || oldHash != source.ContentHash {
|
||||
return fmt.Errorf("retry checkpoint conflicts with the existing request")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err != sql.ErrNoRows {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, `INSERT INTO retry_checkpoints(run_id,source_run_id,mode,failed_sequence,handoff_json,source_version,source_hash) VALUES(?,?,?,?,?,?,?)`, runID, sourceID, mode, sequence, string(data), source.Version, source.ContentHash)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) RetryCheckpoint(ctx context.Context, runID int64) (sourceID int64, mode string, sequence int, handoff map[string]any, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var data string
|
||||
err = s.db.QueryRowContext(ctx, `SELECT source_run_id,mode,failed_sequence,handoff_json FROM retry_checkpoints WHERE run_id=?`, runID).Scan(&sourceID, &mode, &sequence, &data)
|
||||
if err != nil {
|
||||
return 0, "", 0, nil, err
|
||||
}
|
||||
err = json.Unmarshal([]byte(data), &handoff)
|
||||
return
|
||||
}
|
||||
@@ -100,7 +100,7 @@ func (s *Store) GetRun(ctx context.Context, id int64) (*protocol.RunDetail, erro
|
||||
if err := json.Unmarshal([]byte(graphJSON), &r.Graph); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.TitleSource, &r.TitleModel); err != nil {
|
||||
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT mode FROM retry_checkpoints WHERE run_id=pipeline_runs.id),CASE WHEN EXISTS(SELECT 1 FROM run_retries WHERE run_id=pipeline_runs.id) THEN 'restart' ELSE '' END),COALESCE((SELECT source_run_id FROM retry_checkpoints WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.RetryMode, &r.RetrySource, &r.TitleSource, &r.TitleModel); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := decodeValues(input, &r.Input); err != nil {
|
||||
|
||||
@@ -52,6 +52,10 @@ run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, refer
|
||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
||||
CREATE TABLE IF NOT EXISTS run_retries (
|
||||
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));
|
||||
CREATE TABLE IF NOT EXISTS retry_checkpoints (
|
||||
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), source_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
|
||||
mode TEXT NOT NULL, failed_sequence INTEGER NOT NULL, handoff_json TEXT NOT NULL,
|
||||
source_version INTEGER NOT NULL, source_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
|
||||
CREATE TABLE IF NOT EXISTS resource_owners (
|
||||
resource_key TEXT PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES pipeline_runs(id),
|
||||
stage_key TEXT NOT NULL, sequence INTEGER NOT NULL, token TEXT NOT NULL,
|
||||
|
||||
@@ -159,6 +159,10 @@ type Message struct {
|
||||
// them; the same fields are echoed on the run record.
|
||||
Model string `json:"model,omitempty"`
|
||||
StageModels map[string]string `json:"stage_models,omitempty"`
|
||||
// Empty (or "resume") retries the failed visit; "restart" starts at the
|
||||
// entry stage. Older clients omit this field and retain resume behavior.
|
||||
RetryMode string `json:"retry_mode,omitempty"`
|
||||
RetrySource int64 `json:"-"`
|
||||
// Search is the broad, read-only search traffic over pipelines, versions,
|
||||
// runs, steps, field text and artifacts. See search.go for the per-type
|
||||
// payloads and the ranking/kind/bound vocabulary.
|
||||
|
||||
@@ -370,14 +370,16 @@ type NotificationReceipt struct {
|
||||
|
||||
type RunDetail struct {
|
||||
RunSummary
|
||||
Graph Graph `json:"graph"`
|
||||
RetryOf int64 `json:"retry_of,omitempty"`
|
||||
Workspace string `json:"workspace"`
|
||||
AllowTools bool `json:"allow_tools"`
|
||||
WritePaths []string `json:"write_paths"`
|
||||
Input map[string]any `json:"input"`
|
||||
Output map[string]any `json:"output"`
|
||||
Steps []RunStep `json:"steps"`
|
||||
Graph Graph `json:"graph"`
|
||||
RetryOf int64 `json:"retry_of,omitempty"`
|
||||
RetryMode string `json:"retry_mode,omitempty"`
|
||||
RetrySource int64 `json:"retry_source,omitempty"`
|
||||
Workspace string `json:"workspace"`
|
||||
AllowTools bool `json:"allow_tools"`
|
||||
WritePaths []string `json:"write_paths"`
|
||||
Input map[string]any `json:"input"`
|
||||
Output map[string]any `json:"output"`
|
||||
Steps []RunStep `json:"steps"`
|
||||
// Usage is the totalled reported usage of the run's model stages, summed in
|
||||
// one pass over Steps. It is absent for a run with no model usage recorded
|
||||
// (including historical runs).
|
||||
|
||||
@@ -270,7 +270,7 @@ func TestHumanStageCancelRetryAndRebuild(t *testing.T) {
|
||||
t.Fatalf("post-decision failure did not fail the run: %+v", failed.RunSummary)
|
||||
}
|
||||
r3.execute = successExecute
|
||||
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human"})
|
||||
retried, err := r3.retry(protocol.Message{RunID: failing.ID, RequestKey: "retry-human", RetryMode: "restart"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -43,14 +43,14 @@ func TestRetryWaitsForExecutionCleanupEvenAfterTerminalStatus(t *testing.T) {
|
||||
if err != nil || !runs[0].Executing {
|
||||
t.Fatal("list advertised stopped run during cleanup", runs, err)
|
||||
}
|
||||
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err == nil || !strings.Contains(err.Error(), "still executing") {
|
||||
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err == nil || !strings.Contains(err.Error(), "still executing") {
|
||||
t.Fatal("retry accepted during execution cleanup", err)
|
||||
}
|
||||
delete(r.active, id)
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
||||
}
|
||||
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry"}); err != nil {
|
||||
if _, err := r.retry(protocol.Message{RunID: id, RequestKey: "retry", RetryMode: "restart"}); err != nil {
|
||||
t.Fatal("retry rejected after cleanup", err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
|
||||
@@ -183,7 +183,7 @@ func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
|
||||
if err != nil || len(beforeRetry) != 1 {
|
||||
t.Fatal("unconfirmed retry created a run", err)
|
||||
}
|
||||
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key")
|
||||
result, err := runCLI("retry", fmt.Sprint(original.ID), "--yes", "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key", "--mode", "restart")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -236,7 +236,7 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status})
|
||||
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status, RetryMode: "restart"})
|
||||
if err == nil {
|
||||
t.Fatal("invalid retry accepted", status)
|
||||
}
|
||||
@@ -249,6 +249,95 @@ func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dir := t.TempDir()
|
||||
store, err := pipeline.Open(filepath.Join(dir, "resume.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source := "# stage a\n\n```output\nA\n```\n\n```prompt\nA\n```\n\n```edges\nsuccess -> b\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage b\n\n```output\nB\n```\n\n```prompt\nB\n```\n\n```edges\nsuccess -> c\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n\n# stage c\n\n```output\nC\n```\n\n```prompt\nC\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=test\nmodel=test\n```\n"
|
||||
if _, err := store.Submit(ctx, "resume", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
workspace := t.TempDir()
|
||||
var aCalls, bCalls, cCalls int
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
switch def.Key {
|
||||
case "a":
|
||||
aCalls++
|
||||
if input["prompt"] != "task" {
|
||||
t.Errorf("entry received altered input: %#v", input)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-a"}}, nil
|
||||
case "b":
|
||||
bCalls++
|
||||
if input["handoff"] != "from-a" {
|
||||
t.Errorf("resume lost exact prefix handoff: %#v", input)
|
||||
}
|
||||
if bCalls == 1 {
|
||||
return stage.Result{}, fmt.Errorf("deliberate failure")
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "from-b"}}, nil
|
||||
case "c":
|
||||
cCalls++
|
||||
if input["handoff"] != "from-b" {
|
||||
t.Errorf("C received wrong handoff: %#v", input)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
||||
default:
|
||||
t.Fatalf("unexpected stage %q", def.Key)
|
||||
return stage.Result{}, nil
|
||||
}
|
||||
}
|
||||
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
original, err = store.GetRun(ctx, original.ID)
|
||||
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
|
||||
t.Fatalf("initial run did not stop at B: %+v %v", original, err)
|
||||
}
|
||||
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
retried, err = store.GetRun(ctx, retried.ID)
|
||||
if err != nil || retried.Status != "succeeded" || retried.RetryMode != "resume" || retried.RetrySource != original.ID {
|
||||
t.Fatalf("resume did not complete with provenance: %+v %v", retried, err)
|
||||
}
|
||||
if aCalls != 1 || bCalls != 2 || cCalls != 1 {
|
||||
t.Fatalf("retry duplicated prefix or suffix: A=%d B=%d C=%d", aCalls, bCalls, cCalls)
|
||||
}
|
||||
if len(retried.Steps) != 2 || retried.Steps[0].Stage != "b" || retried.Steps[1].Stage != "c" {
|
||||
t.Fatalf("retry persisted the wrong steps: %+v", retried.Steps)
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err = pipeline.Open(filepath.Join(dir, "resume.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
r2, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replayed, err := r2.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
|
||||
if err != nil || replayed.ID != retried.ID {
|
||||
t.Fatalf("persisted request-key replay lost its run: %+v %v", replayed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadOnlyStageRetriesPostAdmissionTransportButMutatingStageDoesNot proves,
|
||||
// through the real runner and CLI fixture, that a read-only stage retries a
|
||||
// post-admission transport (final-readback) failure with a fresh session while
|
||||
|
||||
+145
-1
@@ -285,6 +285,15 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
|
||||
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
||||
}
|
||||
if created {
|
||||
if request.RetrySource > 0 && request.RetryMode != "restart" {
|
||||
checkpoint, checkpointErr := r.retrySourceCheckpoint(request.RetrySource, d)
|
||||
if checkpointErr != nil {
|
||||
return nil, checkpointErr
|
||||
}
|
||||
if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// Emit the created delta before launching: the queued frame reaches
|
||||
// subscribers before the running/step-started transition, so a client
|
||||
// sees the run appear and then advance.
|
||||
@@ -404,6 +413,65 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
||||
return items, err
|
||||
}
|
||||
|
||||
type retryCheckpoint struct {
|
||||
detail *protocol.RunDetail
|
||||
failed protocol.RunStep
|
||||
handoff map[string]any
|
||||
}
|
||||
|
||||
func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) {
|
||||
source, err := r.store.GetRun(r.ctx, sourceID)
|
||||
if err != nil {
|
||||
return retryCheckpoint{}, err
|
||||
}
|
||||
if source.Status != "failed" || source.FinishedAt == "" {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID)
|
||||
}
|
||||
if source.PipelineID != pinned.ID || source.Version != pinned.Version || source.ContentHash != pinned.ContentHash {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d is pinned to a different pipeline version", sourceID)
|
||||
}
|
||||
if len(source.Steps) == 0 {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID)
|
||||
}
|
||||
failed := source.Steps[len(source.Steps)-1]
|
||||
if (failed.Status != "completed" && failed.Status != "failed") || failed.Target != protocol.FailureEnd {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID)
|
||||
}
|
||||
var handoff map[string]any
|
||||
if failed.Input == nil {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID)
|
||||
}
|
||||
handoff = map[string]any{"input": failed.Input, "sequence": failed.Sequence, "stage": failed.Stage}
|
||||
prefix := map[string]any{}
|
||||
for _, step := range source.Steps {
|
||||
if step.Sequence >= failed.Sequence || step.Status != "completed" {
|
||||
continue
|
||||
}
|
||||
prefix[step.Stage] = step.Output
|
||||
}
|
||||
handoff["prefix"] = prefix
|
||||
for _, declared := range pinned.Graph.Stages {
|
||||
if declared.Key != failed.Stage {
|
||||
continue
|
||||
}
|
||||
typ := stage.EffectiveType(declared.Type)
|
||||
if typ == stage.HumanType || typ == stage.NtfyType {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage)
|
||||
}
|
||||
if typ == stage.ModelType {
|
||||
tools, explicit, toolErr := stage.ToolNames(pipeline.StageDefinition(declared))
|
||||
if toolErr != nil {
|
||||
return retryCheckpoint{}, toolErr
|
||||
}
|
||||
if explicit && len(tools) > 0 {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot safely replay admitted native stage %s", sourceID, failed.Stage)
|
||||
}
|
||||
}
|
||||
return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil
|
||||
}
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID)
|
||||
}
|
||||
|
||||
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
if request.RunID <= 0 {
|
||||
return nil, fmt.Errorf("retry requires a positive run ID")
|
||||
@@ -418,7 +486,26 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
if original.Status != "failed" || original.FinishedAt == "" {
|
||||
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
||||
}
|
||||
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
|
||||
if request.RequestKey == "" {
|
||||
return nil, fmt.Errorf("retry requires an idempotency request key")
|
||||
}
|
||||
mode := request.RetryMode
|
||||
if mode == "" {
|
||||
mode = "resume"
|
||||
}
|
||||
if mode != "resume" && mode != "restart" {
|
||||
return nil, fmt.Errorf("retry mode must be resume or restart")
|
||||
}
|
||||
d, err := r.store.Get(r.ctx, original.PipelineID, original.Version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if mode == "resume" {
|
||||
if _, err := r.retrySourceCheckpoint(original.ID, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
|
||||
}
|
||||
|
||||
func (r *runner) cancel(id int64) error {
|
||||
@@ -520,6 +607,35 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
}
|
||||
|
||||
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
|
||||
if request.RetrySource > 0 && request.RetryMode != "restart" {
|
||||
source, err := r.store.GetRun(ctx, request.RetrySource)
|
||||
if err != nil {
|
||||
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
||||
return
|
||||
}
|
||||
checkpoint, err := r.retrySourceCheckpoint(request.RetrySource, d)
|
||||
if err != nil {
|
||||
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
||||
return
|
||||
}
|
||||
storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id)
|
||||
if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil {
|
||||
if checkpointErr == nil {
|
||||
checkpointErr = fmt.Errorf("retry checkpoint is incomplete")
|
||||
}
|
||||
r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil)
|
||||
return
|
||||
}
|
||||
if input, ok := storedHandoff["input"].(map[string]any); ok {
|
||||
checkpoint.failed.Input = input
|
||||
} else {
|
||||
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
|
||||
return
|
||||
}
|
||||
state := rebuildRetryState(source, checkpoint.failed)
|
||||
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
|
||||
return
|
||||
}
|
||||
original, copyErr := copyValues(stage.Values(request.Input))
|
||||
if copyErr != nil {
|
||||
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
|
||||
@@ -629,6 +745,34 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
|
||||
return state
|
||||
}
|
||||
|
||||
// rebuildRetryState restores only the completed prefix. The failed visit is
|
||||
// deliberately not counted and its step/evidence is not copied into the new
|
||||
// attempt, so retries cannot duplicate execution or usage aggregates.
|
||||
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState {
|
||||
state := newRunState(failed.Stage, stage.Values(failed.Input))
|
||||
state.nextSequence = failed.Sequence
|
||||
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
||||
state.memory["input"] = memoryValue{output: original}
|
||||
}
|
||||
for _, step := range detail.Steps {
|
||||
if step.Sequence >= failed.Sequence {
|
||||
break
|
||||
}
|
||||
state.visits[step.Stage]++
|
||||
if step.Status == "completed" {
|
||||
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
||||
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
||||
}
|
||||
state.completed++
|
||||
}
|
||||
state.recordChanges(step.FileChanges, step.Artifacts)
|
||||
if step.Status == "completed" {
|
||||
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
||||
}
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// artifactAccess derives exact visit directories; it never consumes a path from
|
||||
// a definition or model result. An unreadable or symlinked root fails closed.
|
||||
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
|
||||
|
||||
@@ -32,6 +32,12 @@ func TestRetryShortcutAndPaletteSendServerRetryWithoutReplacingDirectory(t *test
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "original task"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "failed", Outcome: "error", Target: protocol.FailureEnd, Input: map[string]any{"prompt": "original task"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user