Make agent-stage retries safe and retain timeout failure evidence
Fix four stage failure-handling and observability defects (A2, A3, A6, A9).
- A2: add safeToReplay(def) as the single definition of retryable post-admission
work - only an explicit tools block that selects neither edit nor shell. The
agent/pi WorkStarted guards now break only when the stage is not safe to
replay, so read-only stages genuinely retry a post-admission timeout while
mutating and legacy-bundle stages still never replay admitted work.
- A6: re-emit a malformed final result on an explicit read-only stage up to the
retry budget, preserving format_repairs evidence; mutating stages keep the
immediate return and their session.
- A3: timed-out stages retain usable evidence. waitForSessionIdle records one
bounded sanitized OperationEvidence on deadline; the agent deferred cleanup
does a bounded best-effort latest-assistant readback and settles knowable V2
usage; RetainSession adds a retained_session pointer to failed step
transcripts so the session is visible beneath the BoundTranscript threshold.
- A9: a stage outcome routed to #failure now yields a non-empty run diagnostic
("stage <name> returned outcome <outcome>") while preferring an existing step
diagnostic.
Adds focused stage and server tests for retry safety, malformed-result
re-emission, timeout evidence retention and run-level diagnostics; updates the
packaged skill references and docs/cli.md, docs/model-results.md.
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// TestFailureRouteRunDiagnostic proves A9: a declared outcome routed to #failure
|
||||
// with an empty step diagnostic still yields a meaningful run diagnostic, while
|
||||
// an existing step diagnostic is preferred verbatim.
|
||||
func TestFailureRouteRunDiagnostic(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "runs.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := strings.Replace(pipeline.Template([]string{"worker"}, true), "success -> #success", "success -> #success\nblocked -> #failure", 1)
|
||||
if _, err := store.Submit(ctx, "diagnostic", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "blocked", Output: stage.Values{}}, nil
|
||||
}
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "blocked", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "failed" || detail.Outcome != "error" || !strings.Contains(detail.Diagnostic, `stage worker returned outcome "blocked"`) {
|
||||
t.Fatalf("empty run diagnostic not replaced: %+v", detail)
|
||||
}
|
||||
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{}, errors.New("fixture execution failed")
|
||||
}
|
||||
run, err = r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "execution", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err = store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "failed" || detail.Diagnostic != "fixture execution failed" {
|
||||
t.Fatalf("existing step diagnostic not preferred: %+v", detail)
|
||||
}
|
||||
}
|
||||
@@ -473,6 +473,11 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
|
||||
step.Outcome = "error"
|
||||
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
||||
}
|
||||
// A failed or timed-out step carries the session pointer even when the
|
||||
// bounded transcript would otherwise be only its stage_limits envelope.
|
||||
if err != nil && stageSession != "" {
|
||||
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
|
||||
}
|
||||
step.Target = def.Edges[step.Outcome]
|
||||
step.Control = &protocol.ControlEvidence{Visit: visits[key], MaxVisits: maxVisits, Memory: reads}
|
||||
if err == nil && selected.Limits != nil && visits[key] >= maxVisits {
|
||||
@@ -526,7 +531,11 @@ func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail,
|
||||
return
|
||||
}
|
||||
if step.Target == protocol.FailureEnd {
|
||||
finish("failed", "error", step.Diagnostic)
|
||||
diagnostic := step.Diagnostic
|
||||
if diagnostic == "" {
|
||||
diagnostic = fmt.Sprintf("stage %s returned outcome %q", step.Stage, step.Outcome)
|
||||
}
|
||||
finish("failed", "error", diagnostic)
|
||||
return
|
||||
}
|
||||
if step.Target == "" {
|
||||
|
||||
Reference in New Issue
Block a user