Files
pipelines/internal/server/runner.go
T
alan f57457aec9 Make agent-stage retries safe and retain timeout failure evidence
Fix four stage failure-handling and observability defects (A2, A3, A6, A9).

- A2: add safeToReplay(def) as the single definition of retryable post-admission
  work - only an explicit tools block that selects neither edit nor shell. The
  agent/pi WorkStarted guards now break only when the stage is not safe to
  replay, so read-only stages genuinely retry a post-admission timeout while
  mutating and legacy-bundle stages still never replay admitted work.
- A6: re-emit a malformed final result on an explicit read-only stage up to the
  retry budget, preserving format_repairs evidence; mutating stages keep the
  immediate return and their session.
- A3: timed-out stages retain usable evidence. waitForSessionIdle records one
  bounded sanitized OperationEvidence on deadline; the agent deferred cleanup
  does a bounded best-effort latest-assistant readback and settles knowable V2
  usage; RetainSession adds a retained_session pointer to failed step
  transcripts so the session is visible beneath the BoundTranscript threshold.
- A9: a stage outcome routed to #failure now yields a non-empty run diagnostic
  ("stage <name> returned outcome <outcome>") while preferring an existing step
  diagnostic.

Adds focused stage and server tests for retry safety, malformed-result
re-emission, timeout evidence retention and run-level diagnostics; updates the
packaged skill references and docs/cli.md, docs/model-results.md.
2026-10-15 18:27:22 +01:00

549 lines
18 KiB
Go

package server
import (
"context"
"errors"
"fmt"
"log"
"maps"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"git.a-wan.net/alan/pipelines/internal/filediff"
"git.a-wan.net/alan/pipelines/internal/harness"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// activeRun records one in-flight run's cancellation seam and the resolved
// workspace it is changing. A single map is the only active-run state, so the
// workspace cannot drift from the cancel function.
type activeRun struct {
cancel context.CancelFunc
workspace string
}
type runner struct {
ctx context.Context
store *pipeline.Store
artifactsRoot string
mu sync.Mutex
active map[int64]activeRun
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
wg sync.WaitGroup
schedulerWG sync.WaitGroup
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
sessions, err := s.ActiveSessionOwners(ctx)
if err != nil {
return nil, err
}
for _, owner := range sessions {
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
var err error
if owner.Backend == "pi" {
err = stage.InterruptPi(stop, owner.Reference)
} else {
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
}
cancel()
if err != nil {
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
}
}
if err := s.RecoverRuns(ctx); err != nil {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}}, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
return nil, fmt.Errorf("an explicit version and request key are required")
}
workspace, err := os.Getwd()
if err != nil {
return nil, err
}
workspace, err = filepath.EvalSymlinks(workspace)
if err != nil {
return nil, err
}
if request.Workspace != "" {
if !filepath.IsAbs(request.Workspace) {
return nil, fmt.Errorf("run workspace must be an absolute directory")
}
selected, err := filepath.EvalSymlinks(request.Workspace)
if err != nil {
return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err)
}
info, err := os.Stat(selected)
if err != nil || !info.IsDir() {
return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace)
}
workspace = selected
}
if len(request.WritePaths) > 20 {
return nil, fmt.Errorf("too many write paths")
}
for _, path := range request.WritePaths {
if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") {
return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths")
}
parent := filepath.Join(workspace, path)
for {
_, err := os.Lstat(parent)
if err == nil {
break
}
if !os.IsNotExist(err) {
return nil, err
}
next := filepath.Dir(parent)
if next == parent {
return nil, fmt.Errorf("write path is unavailable")
}
parent = next
}
resolved, err := filepath.EvalSymlinks(parent)
if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) {
return nil, fmt.Errorf("write path must stay within the run directory")
}
}
request.Input = maps.Clone(request.Input)
if request.Input == nil {
request.Input = map[string]any{}
}
request.Input["pwd"] = workspace
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
}
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
if err != nil {
return nil, err
}
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
return nil, err
}
for _, s := range d.Graph.Stages {
if stage.EffectiveType(s.Type) != stage.ModelType {
continue
}
if s.Configs["harness"] == "pi" && r.execute == nil {
configured, err := harness.Load()
if err != nil {
return nil, err
}
if _, ok := configured.Profiles["pi"]; !ok {
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
}
}
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
if err != nil {
return nil, err
}
if !explicit {
request.AllowTools = true
}
for _, tool := range tools {
if tool == "edit" || tool == "shell" {
request.AllowTools = true
}
}
}
r.mu.Lock()
defer r.mu.Unlock()
if r.ctx.Err() != nil {
return nil, r.ctx.Err()
}
owners, err := r.store.ActiveSessionOwners(r.ctx)
if err != nil {
return nil, err
}
// Refuse only an orphaned session: one whose owning run is not active now.
// A session owned by a currently active run (another run in another
// workspace) must not block new work.
for _, owner := range owners {
if _, ok := r.active[owner.RunID]; !ok {
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
}
}
// Idempotent repeats are accepted even while the original run is active.
if len(r.active) > 0 {
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
if err != nil {
return nil, err
}
if existing == 0 {
// The workspace is the only scope a run reserves: refuse a second
// run only when an active run is already changing this exact
// resolved directory. One active run per workspace keeps this scan
// finding at most one match, so the message is deterministic.
for id, entry := range r.active {
if entry.workspace == workspace {
return nil, fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
}
}
}
}
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...)
if err != nil {
return nil, err
}
if created {
ctx, cancel := context.WithCancel(r.ctx)
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
r.wg.Add(1)
go func() {
defer r.wg.Done()
defer cancel()
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
r.run(ctx, id, d, workspace, request)
}()
}
detail, err := r.store.GetRun(r.ctx, id)
if detail != nil {
_, detail.Executing = r.active[id]
}
return detail, err
}
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
r.mu.Lock()
defer r.mu.Unlock()
d, err := r.store.GetRun(r.ctx, id)
if d != nil {
_, d.Executing = r.active[id]
for _, s := range d.Steps {
if s.AgentActive {
d.Executing = true
}
}
}
return d, err
}
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
r.mu.Lock()
defer r.mu.Unlock()
items, err := r.store.ListRuns(r.ctx)
for i := range items {
_, items[i].Executing = r.active[items[i].ID]
if items[i].Status == "failed" && !items[i].Executing {
d, err := r.store.GetRun(r.ctx, items[i].ID)
if err != nil {
return nil, err
}
for _, step := range d.Steps {
if step.AgentActive {
items[i].Executing = true
}
}
}
}
return items, err
}
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("retry requires a positive run ID")
}
original, err := r.getRun(request.RunID)
if err != nil {
return nil, err
}
if original.Executing {
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
}
if original.Status != "failed" || original.FinishedAt == "" {
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID)
}
func (r *runner) cancel(id int64) error {
r.mu.Lock()
defer r.mu.Unlock()
entry, ok := r.active[id]
if !ok {
return fmt.Errorf("run is not active")
}
entry.cancel()
return nil
}
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
input := stage.Values(request.Input)
key := d.Graph.Stages[0].Key
visits := map[string]int{}
budgets := map[string]*stage.Budget{}
finish := func(status, outcome, diagnostic string) {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
log.Printf("finish run %d: %v", id, err)
}
}
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
finish("failed", "error", copyErr.Error())
return
}
memory := map[string]memoryValue{"input": {output: original}}
defer func() {
if recovered := recover(); recovered != nil {
finish("failed", "error", "stage execution panicked")
}
}()
for sequence := 1; sequence <= 30; sequence++ {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
visits[key]++
if visits[key] > 5 {
finish("failed", "error", "stage visit budget exceeded")
return
}
var def stage.Definition
var selected protocol.Stage
found := false
for _, s := range d.Graph.Stages {
if s.Key == key {
selected = s
def = pipeline.StageDefinition(s)
found = true
break
}
}
if !found {
finish("failed", "error", "missing stage")
return
}
maxVisits := 5
if selected.Limits != nil {
maxVisits = selected.Limits.MaxVisits
}
if visits[key] > maxVisits {
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
return
}
var reads map[string]protocol.MemoryRead
input, reads, copyErr = memoryInput(input, selected.Memory, memory)
if copyErr != nil {
finish("failed", "error", copyErr.Error())
return
}
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
finish("failed", "error", err.Error())
return
}
var tools []string
explicitTools := false
var toolErr error
if stage.EffectiveType(def.Type) == stage.ModelType {
tools, explicitTools, toolErr = stage.ToolNames(def)
}
if toolErr != nil {
finish("failed", "error", toolErr.Error())
return
}
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
// Every agent visit gets a per-run/step directory for the files it
// produces. Creation is best-effort: an unusable directory must never
// fail the run, and it is only attempted for model stages.
artifactsDir := ""
if stage.EffectiveType(def.Type) == stage.ModelType {
candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence))
if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil {
log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr)
} else {
artifactsDir = candidate
def.ArtifactsDir = candidate
}
}
operations := &stage.OperationLog{}
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools}
client.OnSession = func(session string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
}
var result stage.Result
var piClient *stage.PiClient
var err error
stageCtx := ctx
var modelConfig stage.ModelConfig
if stage.EffectiveType(def.Type) == stage.ModelType {
spec, specErr := (stage.ModelStage{}).Spec(def)
if specErr != nil {
finish("failed", "error", specErr.Error())
return
}
modelConfig = spec.Config
if budgets[key] == nil {
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
}
stageCtx = stage.WithBudget(ctx, budgets[key])
}
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
before := filediff.Capture(snapshotCtx, workspace)
snapshotCancel()
var stageSession string
panicked := false
func() {
defer func() {
if recover() != nil {
panicked = true
err = fmt.Errorf("stage execution panicked")
}
}()
if r.execute != nil {
timeout, _ := stage.StageTimeout(def)
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
defer cancel()
result, err = r.execute(executeCtx, def, input)
} else if modelConfig.Harness == "pi" {
configured, configErr := harness.Load()
if configErr != nil {
err = configErr
return
}
profile, ok := configured.Profiles["pi"]
if !ok {
err = fmt.Errorf("Pi is not configured on execution server")
return
}
piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir}
piClient.OnOwnership = func(session, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
}
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
stageSession = piClient.SessionID
client.Transcript = piClient.Transcript
} else if useAgent {
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
stageSession = client.SessionID
} else if stage.EffectiveType(def.Type) == stage.ModelType {
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if active {
stageSession = session
}
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
}
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
client.Transcript = plain.FormatAudit
} else {
result, err = stage.DefaultRegistry().Run(ctx, def, input)
}
}()
// Capture after cleanup, even when the stage's context was canceled.
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
after := filediff.Capture(snapshotCtx, workspace, before)
snapshotCancel()
changes := filediff.Compare(before, after)
if errors.Is(err, stage.ErrStageCleanup) {
changes.Status = "partial"
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
}
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir)
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
if budget := budgets[key]; budget != nil {
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
}
if err != nil {
step.Status = "failed"
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
}
// A failed or timed-out step carries the session pointer even when the
// bounded transcript would otherwise be only its stage_limits envelope.
if err != nil && stageSession != "" {
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
}
step.Target = def.Edges[step.Outcome]
step.Control = &protocol.ControlEvidence{Visit: visits[key], MaxVisits: maxVisits, Memory: reads}
if err == nil && selected.Limits != nil && visits[key] >= maxVisits {
for _, outcome := range selected.Limits.On {
if outcome == step.Outcome {
step.Control.LimitReached = true
step.Control.ModelTarget = step.Target
step.Target = protocol.FailureEnd
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
}
}
}
if panicked {
step.Target = protocol.FailureEnd
}
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
persistErr := r.store.FinishStep(save, id, step)
cancel()
if persistErr != nil {
finish("failed", "error", persistErr.Error())
return
}
if step.Status == "completed" {
values, err := copyValues(stage.Values(step.Output))
if err != nil {
finish("failed", "error", err.Error())
return
}
memory[key] = memoryValue{output: values, sequence: sequence}
}
for _, s := range d.Graph.Stages {
if s.Key == key {
r.notifyStep(id, d, workspace, s.Notify, step)
break
}
}
input = stage.Values(step.Output)
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
if checkErr != nil || len(sessions) > 0 {
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
return
}
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
if step.Target == protocol.SuccessEnd {
finish("succeeded", "success", "")
return
}
if step.Target == protocol.FailureEnd {
diagnostic := step.Diagnostic
if diagnostic == "" {
diagnostic = fmt.Sprintf("stage %s returned outcome %q", step.Stage, step.Outcome)
}
finish("failed", "error", diagnostic)
return
}
if step.Target == "" {
finish("failed", "error", "stage returned an undeclared outcome")
return
}
key = step.Target
}
finish("failed", "error", "total stage budget exceeded")
}