feat: retry selected stage with input overrides
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
# Retry selected stage and input overrides
|
||||
|
||||
As of 2026-10-18, `retry` can select a stage recorded in a terminal failed run
|
||||
with `--stage` and, when that stage was visited repeatedly, `--visit`. The
|
||||
default remains the last failed visit. `--version` may select an explicitly
|
||||
published version of the same pipeline; the server rejects a changed or missing
|
||||
completed prefix.
|
||||
|
||||
`--input-file FILE` (or `-` for stdin) accepts a bounded JSON object. Its keys
|
||||
are shallow-merged into the reconstructed selected-stage input, replacing values
|
||||
with the same key while preserving all other handoff fields. The override is
|
||||
stored in the retry checkpoint and downstream stages receive the selected
|
||||
stage's output normally. Runtime, ownership, pipeline, workspace and scope
|
||||
fields are reserved. The selected stage may repeat its partial side effects.
|
||||
|
||||
The shared documentation and packaged OpenCode skill reference for retry options
|
||||
are now stale and should be folded from this note by the manual `docs-fold`
|
||||
pipeline.
|
||||
+44
-5
@@ -37,7 +37,7 @@ type App struct {
|
||||
type options struct {
|
||||
harness string
|
||||
url, name, prompt, title, template, workspace, paths, key, opencode, model string
|
||||
retryMode string
|
||||
retryMode, retryStage, retryInputFile string
|
||||
option, feedback, evidence, actor string
|
||||
output string
|
||||
conflict string
|
||||
@@ -47,7 +47,7 @@ type options struct {
|
||||
keepSet bool
|
||||
all bool
|
||||
json, source, noOpen, yes, allow, createOnly, force, wait bool
|
||||
version, expected, limit, recoverySequence int
|
||||
version, expected, limit, recoverySequence, retryVisit int
|
||||
recoveryRun int64
|
||||
recoveryStage, resource string
|
||||
timeout, waitTimeout time.Duration
|
||||
@@ -164,6 +164,10 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
f.BoolVar(&o.wait, "wait", false, "Wait for the new run to finish")
|
||||
f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait (0 waits until the run ends); independent of --timeout")
|
||||
f.StringVar(&o.retryMode, "mode", "resume", "Retry mode: resume failed stage or restart from entry")
|
||||
f.StringVar(&o.retryStage, "stage", "", "Retry this recorded stage (required when it was visited more than once)")
|
||||
f.IntVar(&o.retryVisit, "visit", 0, "1-based occurrence of --stage in the recorded execution")
|
||||
f.IntVar(&o.version, "version", 0, "Explicit published version of the same pipeline (default: source version)")
|
||||
f.StringVar(&o.retryInputFile, "input-file", "", "JSON object merged into the selected stage input; '-' reads stdin")
|
||||
case "cancel", "draft-delete":
|
||||
f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt")
|
||||
case "retention":
|
||||
@@ -991,14 +995,49 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
if key == "" {
|
||||
key = fmt.Sprintf("retry-%d-%d", os.Getpid(), time.Now().UnixNano())
|
||||
}
|
||||
var retryInput map[string]any
|
||||
if o.retryInputFile != "" {
|
||||
var data []byte
|
||||
if o.retryInputFile == "-" {
|
||||
input := a.In
|
||||
if input == nil {
|
||||
input = os.Stdin
|
||||
}
|
||||
data, err = io.ReadAll(io.LimitReader(input, 64*1024+1))
|
||||
} else {
|
||||
var file *os.File
|
||||
file, err = os.Open(o.retryInputFile)
|
||||
if err == nil {
|
||||
data, err = io.ReadAll(io.LimitReader(file, 64*1024+1))
|
||||
_ = file.Close()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(data) > 64*1024 {
|
||||
return fmt.Errorf("retry input exceeds 65536 bytes")
|
||||
}
|
||||
if err := json.Unmarshal(data, &retryInput); err != nil {
|
||||
return fmt.Errorf("invalid retry input JSON: %w", err)
|
||||
}
|
||||
if retryInput == nil {
|
||||
return fmt.Errorf("retry input must be a JSON object")
|
||||
}
|
||||
}
|
||||
if o.retryVisit < 0 {
|
||||
return fmt.Errorf("--visit must be positive")
|
||||
}
|
||||
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key, RetryMode: o.retryMode, RetryStage: o.retryStage, RetryVisit: o.retryVisit, RetryVersion: o.version, RetryInput: retryInput})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return a.finishRun(ctx, r, o, call)
|
||||
}
|
||||
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key, RetryMode: o.retryMode})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if command == "retry" {
|
||||
return a.finishRun(ctx, r, o, call)
|
||||
}
|
||||
return a.output(r.RunDetail)
|
||||
case "decide":
|
||||
if len(pos) < 1 || len(pos) > 2 {
|
||||
|
||||
@@ -23,7 +23,7 @@ var usage = map[string]string{
|
||||
"wait": "wait <run-id> [run-id ...] [--timeout DURATION] [--json]",
|
||||
"cancel": "cancel <run-id> [--json]",
|
||||
"decide": "decide <run-id> [stage] --option KEY [--feedback TEXT] [--json]",
|
||||
"retry": "retry <run-id> [--mode resume|restart] [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"retry": "retry <run-id> [--mode resume|restart] [--stage KEY [--visit N]] [--version N] [--input-file FILE|-] [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"title": "title <run-id> \"<title>\" [--json]",
|
||||
"debug": "debug <run-id> [--json | --no-open]",
|
||||
"chat-adopt": "chat-adopt <session-id> [--json]",
|
||||
@@ -51,7 +51,7 @@ var notes = map[string]string{
|
||||
"validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.",
|
||||
"submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
|
||||
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
|
||||
"retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. Original history is retained and already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
|
||||
"retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. --stage selects a recorded stage (use --visit for repeated stages). --input-file accepts a bounded JSON object, merged with replacement at the selected stage; '-' reads stdin. Original history and pinned identity are retained; --version selects an explicit published version of the same pipeline only. --wait-timeout leaves an active retry for later read-only inspection or reattachment with the original --request-key, rather than starting a duplicate retry.",
|
||||
"title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.",
|
||||
"wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).",
|
||||
"decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.",
|
||||
|
||||
@@ -100,7 +100,7 @@ func (s *Store) GetRun(ctx context.Context, id int64) (*protocol.RunDetail, erro
|
||||
if err := json.Unmarshal([]byte(graphJSON), &r.Graph); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT mode FROM retry_checkpoints WHERE run_id=pipeline_runs.id),CASE WHEN EXISTS(SELECT 1 FROM run_retries WHERE run_id=pipeline_runs.id) THEN 'restart' ELSE '' END),COALESCE((SELECT source_run_id FROM retry_checkpoints WHERE run_id=pipeline_runs.id),0),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.RetryMode, &r.RetrySource, &r.TitleSource, &r.TitleModel); err != nil {
|
||||
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT mode FROM retry_checkpoints WHERE run_id=pipeline_runs.id),CASE WHEN EXISTS(SELECT 1 FROM run_retries WHERE run_id=pipeline_runs.id) THEN 'restart' ELSE '' END),COALESCE((SELECT source_run_id FROM retry_checkpoints WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT source_version FROM retry_checkpoints WHERE run_id=pipeline_runs.id),0),COALESCE((SELECT source_hash FROM retry_checkpoints WHERE run_id=pipeline_runs.id),''),title_source,title_model FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf, &r.RetryMode, &r.RetrySource, &r.RetrySourceVersion, &r.RetrySourceHash, &r.TitleSource, &r.TitleModel); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := decodeValues(input, &r.Input); err != nil {
|
||||
|
||||
@@ -161,8 +161,12 @@ type Message struct {
|
||||
StageModels map[string]string `json:"stage_models,omitempty"`
|
||||
// Empty (or "resume") retries the failed visit; "restart" starts at the
|
||||
// entry stage. Older clients omit this field and retain resume behavior.
|
||||
RetryMode string `json:"retry_mode,omitempty"`
|
||||
RetrySource int64 `json:"-"`
|
||||
RetryMode string `json:"retry_mode,omitempty"`
|
||||
RetrySource int64 `json:"-"`
|
||||
RetryStage string `json:"retry_stage,omitempty"`
|
||||
RetryVisit int `json:"retry_visit,omitempty"`
|
||||
RetryVersion int `json:"retry_version,omitempty"`
|
||||
RetryInput map[string]any `json:"retry_input,omitempty"`
|
||||
// Search is the broad, read-only search traffic over pipelines, versions,
|
||||
// runs, steps, field text and artifacts. See search.go for the per-type
|
||||
// payloads and the ranking/kind/bound vocabulary.
|
||||
|
||||
+12
-10
@@ -370,16 +370,18 @@ type NotificationReceipt struct {
|
||||
|
||||
type RunDetail struct {
|
||||
RunSummary
|
||||
Graph Graph `json:"graph"`
|
||||
RetryOf int64 `json:"retry_of,omitempty"`
|
||||
RetryMode string `json:"retry_mode,omitempty"`
|
||||
RetrySource int64 `json:"retry_source,omitempty"`
|
||||
Workspace string `json:"workspace"`
|
||||
AllowTools bool `json:"allow_tools"`
|
||||
WritePaths []string `json:"write_paths"`
|
||||
Input map[string]any `json:"input"`
|
||||
Output map[string]any `json:"output"`
|
||||
Steps []RunStep `json:"steps"`
|
||||
Graph Graph `json:"graph"`
|
||||
RetryOf int64 `json:"retry_of,omitempty"`
|
||||
RetryMode string `json:"retry_mode,omitempty"`
|
||||
RetrySource int64 `json:"retry_source,omitempty"`
|
||||
RetrySourceVersion int `json:"retry_source_version,omitempty"`
|
||||
RetrySourceHash string `json:"retry_source_hash,omitempty"`
|
||||
Workspace string `json:"workspace"`
|
||||
AllowTools bool `json:"allow_tools"`
|
||||
WritePaths []string `json:"write_paths"`
|
||||
Input map[string]any `json:"input"`
|
||||
Output map[string]any `json:"output"`
|
||||
Steps []RunStep `json:"steps"`
|
||||
// Usage is the totalled reported usage of the run's model stages, summed in
|
||||
// one pass over Steps. It is absent for a run with no model usage recorded
|
||||
// (including historical runs).
|
||||
|
||||
@@ -18,6 +18,37 @@ import (
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestRetryInputOverrideRejectsReservedAndOversizedValues(t *testing.T) {
|
||||
base := map[string]any{"handoff": "retained"}
|
||||
if _, err := retryInputOverride(base, map[string]any{"workspace": "/escape"}); err == nil {
|
||||
t.Fatal("reserved retry input was accepted")
|
||||
}
|
||||
if _, err := retryInputOverride(base, map[string]any{"large": strings.Repeat("x", maxRetryInputBytes)}); err == nil {
|
||||
t.Fatal("oversized retry input was accepted")
|
||||
}
|
||||
merged, err := retryInputOverride(base, map[string]any{"correction": "new"})
|
||||
if err != nil || merged["handoff"] != "retained" || merged["correction"] != "new" {
|
||||
t.Fatalf("retry input merge lost retained context: %#v, %v", merged, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryStageContractIncludesPromptConfigAndEdges(t *testing.T) {
|
||||
original := protocol.Stage{Key: "b", Type: "model", Prompt: "old", Configs: map[string]string{"model": "one"}, Edges: map[string]string{"success": "c"}}
|
||||
if !retryStageContractEqual(original, original) {
|
||||
t.Fatal("identical stage contracts differ")
|
||||
}
|
||||
changed := original
|
||||
changed.Prompt = "corrected"
|
||||
if retryStageContractEqual(original, changed) {
|
||||
t.Fatal("changed stage prompt was treated as compatible")
|
||||
}
|
||||
changed = original
|
||||
changed.Edges = map[string]string{"success": "other"}
|
||||
if retryStageContractEqual(original, changed) {
|
||||
t.Fatal("changed stage edge was treated as compatible")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryCreatesFreshOpenCodeSessionAndRetainsFailedSession(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -281,6 +312,9 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T
|
||||
if input["handoff"] != "from-a" {
|
||||
t.Errorf("resume lost exact prefix handoff: %#v", input)
|
||||
}
|
||||
if bCalls == 2 && input["correction"] != "use-new-plan" {
|
||||
t.Errorf("selected retry lost input override: %#v", input)
|
||||
}
|
||||
if bCalls == 1 {
|
||||
return stage.Result{}, fmt.Errorf("deliberate failure")
|
||||
}
|
||||
@@ -305,7 +339,7 @@ func TestRetryResumeReusesPrefixHandoffAndReplaysRequestAfterReopen(t *testing.T
|
||||
if err != nil || original.Status != "failed" || len(original.Steps) != 2 {
|
||||
t.Fatalf("initial run did not stop at B: %+v %v", original, err)
|
||||
}
|
||||
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key"})
|
||||
retried, err := r.retry(protocol.Message{RunID: original.ID, RequestKey: "resume-key", RetryStage: "b", RetryInput: map[string]any{"correction": "use-new-plan"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
+154
-9
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
@@ -286,7 +287,7 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
|
||||
}
|
||||
if created {
|
||||
if request.RetrySource > 0 && request.RetryMode != "restart" {
|
||||
checkpoint, checkpointErr := r.retrySourceCheckpoint(request.RetrySource, d)
|
||||
checkpoint, checkpointErr := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput)
|
||||
if checkpointErr != nil {
|
||||
return nil, checkpointErr
|
||||
}
|
||||
@@ -419,7 +420,51 @@ type retryCheckpoint struct {
|
||||
handoff map[string]any
|
||||
}
|
||||
|
||||
const maxRetryInputBytes = 64 * 1024
|
||||
|
||||
func retryInputOverride(base map[string]any, override map[string]any) (map[string]any, error) {
|
||||
if override == nil {
|
||||
return base, nil
|
||||
}
|
||||
encoded, err := json.Marshal(override)
|
||||
if err != nil || len(encoded) > maxRetryInputBytes {
|
||||
return nil, fmt.Errorf("retry input must be a JSON object no larger than %d bytes", maxRetryInputBytes)
|
||||
}
|
||||
reserved := map[string]bool{"pwd": true, "workspace": true, "write_paths": true, "allow_tools": true, "pipeline_id": true, "pipeline_version": true, "request_key": true, "retry_mode": true, "retry_source": true, "retry_stage": true, "retry_visit": true}
|
||||
merged := map[string]any{}
|
||||
for key, value := range base {
|
||||
merged[key] = value
|
||||
}
|
||||
for key, value := range override {
|
||||
if reserved[key] {
|
||||
return nil, fmt.Errorf("retry input field %q is reserved", key)
|
||||
}
|
||||
merged[key] = value
|
||||
}
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
// retryStageContractEqual compares the complete persisted stage contract. A
|
||||
// retry may carry completed output across a published version only when the
|
||||
// stages that produced that output are unchanged; comparing only effective
|
||||
// types would allow prompt, config, edge, resource, or limit changes through.
|
||||
func retryStageContractEqual(left, right protocol.Stage) bool {
|
||||
a, err := json.Marshal(left)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
b, err := json.Marshal(right)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return string(a) == string(b)
|
||||
}
|
||||
|
||||
func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) {
|
||||
return r.retrySourceCheckpointSelected(sourceID, pinned, "", 0, nil)
|
||||
}
|
||||
|
||||
func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.PipelineDetail, selectedStage string, selectedVisit int, override map[string]any) (retryCheckpoint, error) {
|
||||
source, err := r.store.GetRun(r.ctx, sourceID)
|
||||
if err != nil {
|
||||
return retryCheckpoint{}, err
|
||||
@@ -427,21 +472,58 @@ func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.Pipeline
|
||||
if source.Status != "failed" || source.FinishedAt == "" {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID)
|
||||
}
|
||||
if source.PipelineID != pinned.ID || source.Version != pinned.Version || source.ContentHash != pinned.ContentHash {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d is pinned to a different pipeline version", sourceID)
|
||||
if source.PipelineID != pinned.ID {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d belongs to a different pipeline", sourceID)
|
||||
}
|
||||
if len(source.Steps) == 0 {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID)
|
||||
}
|
||||
failed := source.Steps[len(source.Steps)-1]
|
||||
if (failed.Status != "completed" && failed.Status != "failed") || failed.Target != protocol.FailureEnd {
|
||||
if selectedStage != "" {
|
||||
failed = protocol.RunStep{}
|
||||
visits := 0
|
||||
for _, candidate := range source.Steps {
|
||||
if candidate.Stage != selectedStage {
|
||||
continue
|
||||
}
|
||||
visits++
|
||||
if selectedVisit == 0 && visits == 1 {
|
||||
failed = candidate
|
||||
}
|
||||
if selectedVisit > 0 && visits == selectedVisit {
|
||||
failed = candidate
|
||||
}
|
||||
}
|
||||
if failed.Stage == "" {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry stage %q was not recorded in run %d", selectedStage, sourceID)
|
||||
}
|
||||
if selectedVisit == 0 {
|
||||
count := 0
|
||||
for _, candidate := range source.Steps {
|
||||
if candidate.Stage == selectedStage {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count > 1 {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry stage %q was visited %d times; --visit is required", selectedStage, count)
|
||||
}
|
||||
}
|
||||
if selectedVisit > visits {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry stage %q has no visit %d", selectedStage, selectedVisit)
|
||||
}
|
||||
}
|
||||
if (failed.Status != "completed" && failed.Status != "failed") || (selectedStage == "" && failed.Target != protocol.FailureEnd) {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID)
|
||||
}
|
||||
var handoff map[string]any
|
||||
if failed.Input == nil {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID)
|
||||
}
|
||||
handoff = map[string]any{"input": failed.Input, "sequence": failed.Sequence, "stage": failed.Stage}
|
||||
selectedInput, err := retryInputOverride(failed.Input, override)
|
||||
if err != nil {
|
||||
return retryCheckpoint{}, err
|
||||
}
|
||||
handoff = map[string]any{"input": selectedInput, "sequence": failed.Sequence, "stage": failed.Stage, "visit": failed.Sequence}
|
||||
prefix := map[string]any{}
|
||||
for _, step := range source.Steps {
|
||||
if step.Sequence >= failed.Sequence || step.Status != "completed" {
|
||||
@@ -450,10 +532,58 @@ func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.Pipeline
|
||||
prefix[step.Stage] = step.Output
|
||||
}
|
||||
handoff["prefix"] = prefix
|
||||
// A newer explicitly selected version may continue only when every recorded
|
||||
// prefix stage still exists with the same kind. Its outputs are evidence from
|
||||
// the source attempt and cannot be safely handed to a changed stage contract.
|
||||
sourceStages := map[string]protocol.Stage{}
|
||||
for _, declared := range source.Graph.Stages {
|
||||
sourceStages[declared.Key] = declared
|
||||
}
|
||||
for _, step := range source.Steps {
|
||||
if step.Sequence >= failed.Sequence {
|
||||
break
|
||||
}
|
||||
var pinnedStage *protocol.Stage
|
||||
for i := range pinned.Graph.Stages {
|
||||
if pinned.Graph.Stages[i].Key == step.Stage {
|
||||
pinnedStage = &pinned.Graph.Stages[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
previous, ok := sourceStages[step.Stage]
|
||||
if pinnedStage == nil || !ok || !retryStageContractEqual(previous, *pinnedStage) {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, step.Stage)
|
||||
}
|
||||
}
|
||||
var sourceSelected protocol.Stage
|
||||
var sourceSelectedOK bool
|
||||
if sourceSelected, sourceSelectedOK = sourceStages[failed.Stage]; !sourceSelectedOK {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in its recorded graph", sourceID)
|
||||
}
|
||||
pinnedStages := map[string]protocol.Stage{}
|
||||
for _, declared := range pinned.Graph.Stages {
|
||||
pinnedStages[declared.Key] = declared
|
||||
}
|
||||
// The retry will continue through every stage that the source execution
|
||||
// already recorded after the selected checkpoint. Validate those stages
|
||||
// before admitting the new run as well: otherwise an explicitly selected
|
||||
// version can pass prefix/selected-stage validation and fail later in the
|
||||
// loop with the less useful "missing stage" error.
|
||||
for _, step := range source.Steps {
|
||||
if step.Sequence <= failed.Sequence {
|
||||
continue
|
||||
}
|
||||
if _, ok := pinnedStages[step.Stage]; !ok {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d selected version is missing later stage %q", sourceID, step.Stage)
|
||||
}
|
||||
}
|
||||
for _, declared := range pinned.Graph.Stages {
|
||||
if declared.Key != failed.Stage {
|
||||
continue
|
||||
}
|
||||
if stage.EffectiveType(sourceSelected.Type) != stage.EffectiveType(declared.Type) {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible selected stage %q", sourceID, failed.Stage)
|
||||
}
|
||||
typ := stage.EffectiveType(declared.Type)
|
||||
if typ == stage.HumanType || typ == stage.NtfyType {
|
||||
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage)
|
||||
@@ -485,6 +615,17 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
if request.RequestKey == "" {
|
||||
return nil, fmt.Errorf("retry requires an idempotency request key")
|
||||
}
|
||||
// Resolve an already admitted request before validating optional retry
|
||||
// selectors. This makes replay after a restart genuinely idempotent even
|
||||
// when the original request used an input override that the replay omits.
|
||||
if existing, err := r.store.RunByKey(r.ctx, request.RequestKey); err != nil {
|
||||
return nil, err
|
||||
} else if existing > 0 {
|
||||
return r.store.GetRun(r.ctx, existing)
|
||||
}
|
||||
if request.RetryVisit < 0 || (request.RetryVisit > 0 && request.RetryStage == "") {
|
||||
return nil, fmt.Errorf("retry visit requires a positive --stage visit selection")
|
||||
}
|
||||
mode := request.RetryMode
|
||||
if mode == "" {
|
||||
mode = "resume"
|
||||
@@ -492,16 +633,20 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
if mode != "resume" && mode != "restart" {
|
||||
return nil, fmt.Errorf("retry mode must be resume or restart")
|
||||
}
|
||||
d, err := r.store.Get(r.ctx, original.PipelineID, original.Version)
|
||||
version := original.Version
|
||||
if request.RetryVersion > 0 {
|
||||
version = request.RetryVersion
|
||||
}
|
||||
d, err := r.store.Get(r.ctx, original.PipelineID, version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if mode == "resume" {
|
||||
if _, err := r.retrySourceCheckpoint(original.ID, d); err != nil {
|
||||
if _, err := r.retrySourceCheckpointSelected(original.ID, d, request.RetryStage, request.RetryVisit, request.RetryInput); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
|
||||
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, RetryStage: request.RetryStage, RetryVisit: request.RetryVisit, RetryInput: request.RetryInput, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
|
||||
}
|
||||
|
||||
func (r *runner) cancel(id int64) error {
|
||||
@@ -609,7 +754,7 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
|
||||
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
||||
return
|
||||
}
|
||||
checkpoint, err := r.retrySourceCheckpoint(request.RetrySource, d)
|
||||
checkpoint, err := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput)
|
||||
if err != nil {
|
||||
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user