Files
pipelines/internal/server/runner.go
T

1575 lines
60 KiB
Go

package server
import (
"context"
"errors"
"fmt"
"log"
"maps"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"git.a-wan.net/alan/pipelines/internal/filediff"
"git.a-wan.net/alan/pipelines/internal/harness"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"git.a-wan.net/alan/pipelines/internal/summarise"
)
// activeRun records one in-flight run's cancellation seam and the resolved
// workspace it is changing. A single map is the only active-run state, so the
// workspace cannot drift from the cancel function.
type activeRun struct {
cancel context.CancelFunc
workspace string
}
type runner struct {
ctx context.Context
store *pipeline.Store
artifactsRoot string
mu sync.Mutex
active map[int64]activeRun
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
// title generates the operator-facing run title from the run prompt. When nil
// the store derives a fallback from the prompt with no model call, so tests
// and unconfigured servers stay deterministic.
title func(context.Context, string) summarise.Title
// verdict generates the bounded one-line reason for a completed stage visit
// from its output map. When nil the store derives the reason at read time with
// no model call. It is best-effort: any failure must never fail the stage.
verdict func(context.Context, map[string]any) string
// events pushes run state deltas to subscribed clients. When nil (tests that
// build a runner directly) emits are dropped, so nothing else changes.
events *runEventHub
// identMu guards idents: the pipeline identity and stored title (see
// runIdentity) that every emitted frame of a run carries, so a client with no
// cached row renders a brand-new run without re-reading the list. Populated
// where a run is created, decided or canceled; a run with no remembered
// identity keeps blank identity fields and the client resyncs once instead.
identMu sync.Mutex
idents map[int64]runIdentity
// exportSession writes a finished stage's session into its own artifacts
// directory. It defaults to stage.ExportSession; a failure is best-effort and
// never changes the stage's status, outcome, target or result validation.
exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error)
// verifyTarget is installed only by server wiring for a controlled fixture
// workflow. Leaving it nil fails closed for target-aware definitions.
verifyTarget stage.VerificationRunner
wg sync.WaitGroup
schedulerWG sync.WaitGroup
// attentionMu guards attentionSent, the per-run set of run-attention events
// already delivered — by a stage's own notify policy or by the run-level
// default — so neither republishes the same genuine event.
attentionMu sync.Mutex
attentionSent map[int64]map[string]bool
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
sessions, err := s.ActiveSessionOwners(ctx)
if err != nil {
return nil, err
}
for _, owner := range sessions {
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
var err error
if owner.Backend == "pi" {
err = stage.InterruptPi(stop, owner.Reference)
} else {
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID)
}
cancel()
if err != nil {
if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource {
log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID)
continue
}
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
}
if err := s.ConfirmRecoveredSession(ctx, owner); err != nil {
return nil, err
}
}
if err := s.RecoverRuns(ctx); err != nil {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, exportSession: stage.ExportSession}, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
return nil, fmt.Errorf("an explicit version and request key are required")
}
workspace, err := os.Getwd()
if err != nil {
return nil, err
}
workspace, err = filepath.EvalSymlinks(workspace)
if err != nil {
return nil, err
}
if request.Workspace != "" {
if !filepath.IsAbs(request.Workspace) {
return nil, fmt.Errorf("run workspace must be an absolute directory")
}
selected, err := filepath.EvalSymlinks(request.Workspace)
if err != nil {
return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err)
}
info, err := os.Stat(selected)
if err != nil || !info.IsDir() {
return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace)
}
workspace = selected
}
if len(request.WritePaths) > 20 {
return nil, fmt.Errorf("too many write paths")
}
for _, path := range request.WritePaths {
if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") {
return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths")
}
parent := filepath.Join(workspace, path)
for {
_, err := os.Lstat(parent)
if err == nil {
break
}
if !os.IsNotExist(err) {
return nil, err
}
next := filepath.Dir(parent)
if next == parent {
return nil, fmt.Errorf("write path is unavailable")
}
parent = next
}
resolved, err := filepath.EvalSymlinks(parent)
if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) {
return nil, fmt.Errorf("write path must stay within the run directory")
}
}
request.Input = maps.Clone(request.Input)
if request.Input == nil {
request.Input = map[string]any{}
}
request.Input["pwd"] = workspace
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
}
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
if err != nil {
return nil, err
}
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
return nil, err
}
// Validate the launch-time model overrides against the pinned graph before
// admission: an unknown stage key or a malformed provider/model must never
// insert a run row. The resolution is O(#overrides + #stages) with no model or
// network call, and the same helper is the authoritative precedence rule.
if _, err := pipeline.EffectiveModels(d.Graph, request.Model, request.StageModels); err != nil {
return nil, err
}
for _, s := range d.Graph.Stages {
if stage.EffectiveType(s.Type) != stage.ModelType {
continue
}
if s.Configs["harness"] == "pi" && r.execute == nil {
configured, err := harness.Load()
if err != nil {
return nil, err
}
if _, ok := configured.Profiles["pi"]; !ok {
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
}
}
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
if err != nil {
return nil, err
}
if !explicit {
request.AllowTools = true
}
for _, tool := range tools {
if tool == "edit" || tool == "shell" {
request.AllowTools = true
}
}
}
// Decide admission and idempotency under the lock, but generate the title
// only for a request that will actually start a run: a rejected or replayed
// request must not pay for a bounded model generation, and a live one must
// not hold the lock (stalling listings and cancellation) while a model is
// called. An unavailable or slow model falls back to the prompt's first line.
r.mu.Lock()
if r.ctx.Err() != nil {
r.mu.Unlock()
return nil, r.ctx.Err()
}
owners, err := r.store.ActiveSessionOwners(r.ctx)
if err != nil {
r.mu.Unlock()
return nil, err
}
// Refuse only an orphaned session: one whose owning run is not active now.
// A session owned by a currently active run (another run in another
// workspace) must not block new work.
for _, owner := range owners {
if _, ok := r.active[owner.RunID]; !ok {
if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace {
continue
}
r.mu.Unlock()
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
}
}
// Idempotent repeats are accepted even while the original run is active, and
// a replay reuses the stored title instead of generating a new one.
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
if err != nil {
r.mu.Unlock()
return nil, err
}
if existing == 0 {
if err := r.checkWorkspaceFree(workspace); err != nil {
r.mu.Unlock()
return nil, err
}
}
r.mu.Unlock()
var runTitle summarise.Title
if existing == 0 {
// An explicit operator title skips the model call; otherwise the
// configured generator names the run, falling back to the prompt's
// first line when it is unavailable or slow.
if explicit, ok := summarise.ExplicitTitle(request.Title); ok {
runTitle = explicit
} else if r.title != nil {
prompt, _ := request.Input["prompt"].(string)
runTitle = r.title(r.ctx, prompt)
}
}
r.mu.Lock()
defer r.mu.Unlock()
// Another run may have reserved this workspace while the title was
// generated, so re-check before reserving it; a race on the request key is
// handled by the store's idempotent insert.
if existing == 0 {
if err := r.checkWorkspaceFree(workspace); err != nil {
return nil, err
}
}
id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Model: request.Model, StageModels: request.StageModels}, runTitle, retryOf...)
if err != nil {
return nil, err
}
// The run row now exists: read it once for its resolved identity (pipeline
// id/name, the store-derived title, the pinned version), so the queued frame
// and every later frame carry what a client needs to render a brand-new run
// without re-reading the list. A failed read only costs that seed: the run is
// still launched below and its frames carry no identity, so the client falls
// back to its one resync.
detail, readErr := r.store.GetRun(r.ctx, id)
if readErr != nil {
detail = nil
} else if detail != nil {
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
}
if created {
if request.RetrySource > 0 && request.RetryMode != "restart" {
checkpoint, checkpointErr := r.retrySourceCheckpoint(request.RetrySource, d)
if checkpointErr != nil {
return nil, checkpointErr
}
if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil {
return nil, err
}
}
// Emit the created delta before launching: the queued frame reaches
// subscribers before the running/step-started transition, so a client
// sees the run appear and then advance.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusQueued,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}, false)
r.launch(id, workspace, func(ctx context.Context) {
r.run(ctx, id, d, workspace, request)
})
}
if detail != nil {
_, detail.Executing = r.active[id]
r.overlayEffectiveModels(detail)
}
if readErr != nil {
return nil, readErr
}
return detail, nil
}
// launch registers an active run's cancellation seam and workspace, adds it to
// the wait group and starts fn on a derived cancellable context that cleans up
// the registration when it returns. A fresh run and a resumed decision both go
// through it, so registration and teardown live once. Callers must hold r.mu.
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
ctx, cancel := context.WithCancel(r.ctx)
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
r.wg.Add(1)
go func() {
defer r.wg.Done()
defer cancel()
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
fn(ctx)
}()
}
// checkWorkspaceFree refuses a run when an active run is already changing the
// same resolved workspace. The workspace is the only scope a run reserves, and
// one active run per workspace keeps this scan finding at most one match, so the
// message is deterministic. Callers must hold r.mu.
func (r *runner) checkWorkspaceFree(workspace string) error {
for id, entry := range r.active {
if entry.workspace == workspace {
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
}
}
return nil
}
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
r.mu.Lock()
defer r.mu.Unlock()
d, err := r.store.GetRun(r.ctx, id)
if d != nil {
_, d.Executing = r.active[id]
for _, s := range d.Steps {
if s.AgentActive {
d.Executing = true
}
}
r.overlayEffectiveModels(d)
}
return d, err
}
// overlayEffectiveModels replaces each model stage's provider/model in a returned
// run's graph with the effective pair the run actually uses, so the app and TUI
// stage-definition surfaces show the override with no client change. It is
// presentation only: the persisted graph_json and the pinned source returned by
// `show` are never rewritten. A run with no overrides is left untouched.
func (r *runner) overlayEffectiveModels(d *protocol.RunDetail) {
if d == nil || (d.Model == "" && len(d.StageModels) == 0) {
return
}
effective, err := pipeline.EffectiveModels(d.Graph, d.Model, d.StageModels)
if err != nil {
return
}
for i := range d.Graph.Stages {
s := &d.Graph.Stages[i]
if stage.EffectiveType(s.Type) != stage.ModelType {
continue
}
override, ok := effective[s.Key]
if !ok {
continue
}
if s.Configs == nil {
s.Configs = map[string]string{}
}
s.Configs["provider"] = override.Provider
s.Configs["model"] = override.Model
}
}
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
r.mu.Lock()
defer r.mu.Unlock()
items, err := r.store.ListRuns(r.ctx)
for i := range items {
_, items[i].Executing = r.active[items[i].ID]
if items[i].Status == "failed" && !items[i].Executing {
d, err := r.store.GetRun(r.ctx, items[i].ID)
if err != nil {
return nil, err
}
for _, step := range d.Steps {
if step.AgentActive {
items[i].Executing = true
}
}
}
}
return items, err
}
type retryCheckpoint struct {
detail *protocol.RunDetail
failed protocol.RunStep
handoff map[string]any
}
func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) {
source, err := r.store.GetRun(r.ctx, sourceID)
if err != nil {
return retryCheckpoint{}, err
}
if source.Status != "failed" || source.FinishedAt == "" {
return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID)
}
if source.PipelineID != pinned.ID || source.Version != pinned.Version || source.ContentHash != pinned.ContentHash {
return retryCheckpoint{}, fmt.Errorf("retry source %d is pinned to a different pipeline version", sourceID)
}
if len(source.Steps) == 0 {
return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID)
}
failed := source.Steps[len(source.Steps)-1]
if (failed.Status != "completed" && failed.Status != "failed") || failed.Target != protocol.FailureEnd {
return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID)
}
var handoff map[string]any
if failed.Input == nil {
return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID)
}
handoff = map[string]any{"input": failed.Input, "sequence": failed.Sequence, "stage": failed.Stage}
prefix := map[string]any{}
for _, step := range source.Steps {
if step.Sequence >= failed.Sequence || step.Status != "completed" {
continue
}
prefix[step.Stage] = step.Output
}
handoff["prefix"] = prefix
for _, declared := range pinned.Graph.Stages {
if declared.Key != failed.Stage {
continue
}
typ := stage.EffectiveType(declared.Type)
if typ == stage.HumanType || typ == stage.NtfyType {
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage)
}
if typ == stage.ModelType {
tools, explicit, toolErr := stage.ToolNames(pipeline.StageDefinition(declared))
if toolErr != nil {
return retryCheckpoint{}, toolErr
}
if explicit && len(tools) > 0 {
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot safely replay admitted native stage %s", sourceID, failed.Stage)
}
}
return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil
}
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID)
}
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("retry requires a positive run ID")
}
original, err := r.getRun(request.RunID)
if err != nil {
return nil, err
}
if original.Executing {
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
}
if original.Status != "failed" || original.FinishedAt == "" {
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
}
if request.RequestKey == "" {
return nil, fmt.Errorf("retry requires an idempotency request key")
}
mode := request.RetryMode
if mode == "" {
mode = "resume"
}
if mode != "resume" && mode != "restart" {
return nil, fmt.Errorf("retry mode must be resume or restart")
}
d, err := r.store.Get(r.ctx, original.PipelineID, original.Version)
if err != nil {
return nil, err
}
if mode == "resume" {
if _, err := r.retrySourceCheckpoint(original.ID, d); err != nil {
return nil, err
}
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
}
func (r *runner) cancel(id int64) error {
r.mu.Lock()
entry, ok := r.active[id]
r.mu.Unlock()
if ok {
entry.cancel()
return nil
}
// A run that is waiting on a person has no live goroutine, so cancel it at
// the persistence layer instead. This keeps cancel working across a restart.
// The single follow-up read seeds the terminal delta; cancel is a cold
// operator action, not a per-transition hot path.
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
return err
}
// A canceled run is terminal: nothing more can notify for it.
r.forgetAttention(id)
if r.events != nil {
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: detail.Workspace,
Status: detail.Status,
Outcome: detail.Outcome,
CurrentStage: detail.CurrentStage,
CompletedSteps: detail.CompletedSteps,
FinishedAt: detail.FinishedAt,
}, true)
}
}
return nil
}
// decide records an operator's choice for a waiting human stage and resumes the
// run along the chosen edge. ResolveDecision owns validation and idempotency, so
// decide does no option parsing of its own. A resumed run is registered and
// launched exactly like a fresh one but starts from the persisted state.
func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("decide requires a positive run ID")
}
r.mu.Lock()
detail, err := r.store.GetRun(r.ctx, request.RunID)
if err != nil {
r.mu.Unlock()
return nil, err
}
if _, busy := r.active[detail.ID]; busy {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
}
if detail.Status == protocol.RunStatusWaiting {
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
r.mu.Unlock()
return nil, err
}
}
step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback)
if err != nil {
r.mu.Unlock()
return nil, err
}
if applied {
// Re-seed the identity so the resumed frames (this one and the resumed
// loop's) carry it even after a restart, when no creation was observed.
r.rememberRunIdentity(detail.ID, identityOfRunSummary(detail.RunSummary))
for i := range detail.Steps {
if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage {
detail.Steps[i] = step
break
}
}
// The decision closes the waiting step and returns the run to running;
// report that before the resumed loop emits its own deltas.
r.emitRun(protocol.RunEventFrame{
RunID: detail.ID,
Workspace: detail.Workspace,
Status: protocol.RunStatusRunning,
CurrentStage: step.Stage,
CompletedSteps: completedSteps(detail.Steps),
Step: &protocol.RunStepDelta{
Sequence: step.Sequence,
Stage: step.Stage,
Status: step.Status,
Outcome: step.Outcome,
Target: step.Target,
Diagnostic: step.Diagnostic,
},
}, false)
r.launch(detail.ID, detail.Workspace, func(ctx context.Context) {
r.resume(ctx, detail.ID, detail, detail.Workspace, step)
})
}
r.mu.Unlock()
return r.getRun(request.RunID)
}
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
if request.RetrySource > 0 && request.RetryMode != "restart" {
source, err := r.store.GetRun(ctx, request.RetrySource)
if err != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
return
}
checkpoint, err := r.retrySourceCheckpoint(request.RetrySource, d)
if err != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
return
}
storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id)
if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil {
if checkpointErr == nil {
checkpointErr = fmt.Errorf("retry checkpoint is incomplete")
}
r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil)
return
}
if input, ok := storedHandoff["input"].(map[string]any); ok {
checkpoint.failed.Input = input
} else {
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
return
}
state := rebuildRetryState(source, checkpoint.failed)
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
return
}
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
return
}
input := stage.Values(request.Input)
state := newRunState(d.Graph.Stages[0].Key, input)
state.memory["input"] = memoryValue{output: original}
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
}
// terminalFinish classifies a stage's resolved target. A terminal key yields the
// run's final status, outcome and diagnostic; an empty target means the stage
// named an undeclared outcome; any other value is a stage key and is not
// terminal. resume and the loop tail share it, so terminal routing lives once.
func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) {
switch target {
case protocol.SuccessEnd:
return "succeeded", "success", "", true
case protocol.FailureEnd:
if diagnostic == "" {
diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome)
}
return "failed", "error", diagnostic, true
case "":
return "failed", "error", "stage returned an undeclared outcome", true
}
return "", "", "", false
}
// resume continues a decided run from the state already persisted for its run
// detail: memory from the run input and the completed steps, visits counted per
// stage, and the next sequence above the highest recorded one. It then runs the
// shared loop from the decision's target; no full-table scan and no extra
// per-step query are introduced.
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
// A resumed run's frames carry the same identity as its creation, so re-seed
// it here too: resume is also reachable from a store read after a restart.
r.rememberRunIdentity(id, identityOfRunSummary(d.RunSummary))
// The decision may route straight to a terminal, which has no stage to run.
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
r.finishRunState(id, d.Name, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Sequence, step.Stage, stage.Values(step.Output))
return
}
state := rebuildRunState(d, step)
r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools, Model: d.Model, StageModels: d.StageModels}, state)
}
// runState is the loop's whole mutable state, extracted so a fresh run and a
// resumed run share one implementation. nextSequence is set before each visit so
// a wait that returns to the caller records where the run will continue.
type runState struct {
key string
nextSequence int
visits map[string]int
memory map[string]memoryValue
input stage.Values
// changedPaths retains a bounded sample of the run's recorded changed paths
// and changedCount the full total, so the skip guard reads only in-memory
// evidence. uiSurface is latched once any recorded change touched a UI
// surface. Both are folded in by recordChanges (ui_surface.go).
changedPaths []string
changedCount int
uiSurface bool
// capturedPaths is the full, deduplicated, first-seen-ordered set of the
// run's server-captured changed paths, distinct from the bounded
// changedPaths sample used only by the skip reason. capturedSeen is the
// single dedupe set. Both are folded in by recordChanges (ui_surface.go) and
// published to a selecting stage as input.server_changed_files.
capturedPaths []string
capturedSeen map[string]bool
// completed counts the run's finished steps, so a delta can report the
// running total without re-reading the store.
completed int
artifactDirs []string
}
func newRunState(key string, input stage.Values) *runState {
return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input}
}
func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState {
state := newRunState(decided.Target, stage.Values(decided.Output))
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
state.memory["input"] = memoryValue{output: original}
}
for _, step := range detail.Steps {
state.visits[step.Stage]++
if step.Sequence >= state.nextSequence {
state.nextSequence = step.Sequence + 1
}
if step.Status == "completed" {
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
state.completed++
}
// Seed the UI-surface accumulator from the persisted evidence so a
// resumed run reaches the same skip decision with no extra query.
state.recordChanges(step.FileChanges, step.Artifacts)
// Only completed, recorded visits are eligible for a later stage's
// review capability. The path is reconstructed by the runner below.
if step.Status == "completed" {
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
}
}
return state
}
// rebuildRetryState restores only the completed prefix. The failed visit is
// deliberately not counted and its step/evidence is not copied into the new
// attempt, so retries cannot duplicate execution or usage aggregates.
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState {
state := newRunState(failed.Stage, stage.Values(failed.Input))
state.nextSequence = failed.Sequence
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
state.memory["input"] = memoryValue{output: original}
}
for _, step := range detail.Steps {
if step.Sequence >= failed.Sequence {
break
}
state.visits[step.Stage]++
if step.Status == "completed" {
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
state.completed++
}
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
}
}
return state
}
// artifactAccess derives exact visit directories; it never consumes a path from
// a definition or model result. An unreadable or symlinked root fails closed.
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
root, err := filepath.EvalSymlinks(r.artifactsRoot)
if err != nil {
return stage.ArtifactAccess{}
}
runRoot := filepath.Join(root, strconv.FormatInt(runID, 10))
visitDirectory := func(visit int) (string, bool) {
if visit <= 0 {
return "", false
}
directory := filepath.Join(runRoot, strconv.Itoa(visit))
resolved, err := filepath.EvalSymlinks(directory)
if err != nil || resolved != directory {
return "", false
}
info, err := os.Stat(resolved)
if err != nil || !info.IsDir() {
return "", false
}
return resolved, true
}
access := stage.ArtifactAccess{}
if directory, ok := visitDirectory(sequence); ok {
resolved, err := filepath.EvalSymlinks(current)
if err != nil || resolved != directory {
return access
}
access.Write = directory
access.Read = append(access.Read, directory)
}
for _, priorSequence := range prior {
visit, err := strconv.Atoi(priorSequence)
if err != nil || visit <= 0 || visit >= sequence {
continue
}
if directory, ok := visitDirectory(visit); ok {
access.Read = append(access.Read, directory)
}
}
return access
}
func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient {
return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
}
func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient {
return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
}
// serverChangedFilesField is the reserved run-input memory field under which the
// runner publishes the server-captured changed paths. A stage selects it with
// `"server_changed_files":"input.server_changed_files"`, alongside another
// stage's declared list, so it sees the verified evidence, not just a model's
// claim of authorship.
const serverChangedFilesField = "server_changed_files"
// capturedList returns the full, first-seen-ordered set of server-captured
// changed paths, or a non-nil empty list when nothing was captured so the field
// still resolves as a list. It copies the accumulator so the retained evidence
// cannot be aliased by a published value. O(captured paths).
func (s *runState) capturedList() []string {
if len(s.capturedPaths) == 0 {
return []string{}
}
return append([]string(nil), s.capturedPaths...)
}
// publishServerChangedFiles writes the run's captured changed-path set into the
// run input memory value under serverChangedFilesField, creating the entry and
// its output map when absent. Called before memoryInput so a stage selecting
// input.server_changed_files always resolves; only that field is added and only
// a selecting stage copies it out, so state.input and other stages' input are
// unchanged. O(captured paths).
func (s *runState) publishServerChangedFiles() {
entry, ok := s.memory["input"]
if !ok || entry.output == nil {
entry = memoryValue{output: stage.Values{}}
}
entry.output[serverChangedFilesField] = s.capturedList()
s.memory["input"] = entry
}
// runIdentity is the pipeline identity and stored title a run's pushed frames
// carry (RunEventFrame.PipelineID/Name/Title/Version). It is remembered where a
// run is created, decided or canceled and stamped onto every frame that leaves
// emitRun, so a client that has never seen the run can build its list row from
// the frame without re-reading the whole list.
type runIdentity struct {
pipelineID int64
name string
title string
version int
}
// identityOfRunSummary is the one projection of a run row onto the identity its
// pushed frames carry, so the frame and the list row cannot disagree.
func identityOfRunSummary(s protocol.RunSummary) runIdentity {
return runIdentity{pipelineID: s.PipelineID, name: s.Name, title: s.Title, version: s.Version}
}
// rememberRunIdentity records the identity every later frame of run id carries.
// It is safe from any goroutine and takes its own lock (never r.mu), so it is
// callable from a path that already holds r.mu.
func (r *runner) rememberRunIdentity(id int64, ident runIdentity) {
if r.idents == nil {
return
}
r.identMu.Lock()
r.idents[id] = ident
r.identMu.Unlock()
}
// stampRunIdentity fills a frame's blank identity fields from the run's
// remembered identity, leaving any the emit site set untouched. A run with no
// remembered identity keeps them blank, so the client falls back to its resync.
func (r *runner) stampRunIdentity(frame *protocol.RunEventFrame) {
r.identMu.Lock()
ident, ok := r.idents[frame.RunID]
r.identMu.Unlock()
if !ok {
return
}
if frame.PipelineID == 0 {
frame.PipelineID = ident.pipelineID
}
if frame.Name == "" {
frame.Name = ident.name
}
if frame.Title == "" {
frame.Title = ident.title
}
if frame.Version == 0 {
frame.Version = ident.version
}
}
// emitRun pushes one run delta to the subscribed-client hub when one is wired,
// and does nothing otherwise (a runner built directly in a test has no hub).
// terminal marks a final transition, which the hub flushes without coalescing.
func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
if r.events == nil {
return
}
r.stampRunIdentity(&frame)
r.events.emit(frame, terminal)
}
func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
log.Printf("finish run %d: %v", id, err)
}
// Push the terminal delta before the attention notification, so a slow ntfy
// never delays the live update the app folds.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: status,
ResourceWaitSet: true,
Outcome: outcome,
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
}, true)
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
// The run is terminal: nothing more can notify for it.
r.forgetAttention(id)
}
// completedSteps counts the completed steps of a run detail; resume uses it to
// seed the terminal delta without another store read.
func completedSteps(steps []protocol.RunStep) int {
n := 0
for _, step := range steps {
if step.Status == "completed" {
n++
}
}
return n
}
// stageUsage carries the usage a model stage already measured for budget
// accounting onto the run record. It reports a cost only when the provider
// reported one and an in/out split only when the harness reported it, so an
// unknown cost or split is never presented as zero. It returns nil when no usage
// was observed, so the step stays absent rather than zero.
func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage {
if !snapshot.TokensKnown && !snapshot.USDKnown {
return nil
}
usage := &protocol.StageUsage{TokensKnown: snapshot.TokensKnown, CostKnown: snapshot.USDKnown}
if snapshot.TokensKnown {
usage.TokensTotal = snapshot.UsedTokens
if snapshot.SplitKnown {
usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true
}
}
if snapshot.USDKnown {
usage.CostUSD = snapshot.UsedUSD
}
return usage
}
// finishStage persists one step, refreshes the shared memory for a completed
// step, notifies and classifies the step's target. Both the executed path and
// the skipped path use it, so persistence, notification and terminal routing
// cannot drift between them. A persist or memory-copy failure is returned for
// the caller to fail the run; the caller still owns the model-specific
// verdict, quiescence and cancellation handling around it.
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
persistErr := r.store.FinishStep(save, id, step)
cancel()
if persistErr != nil {
return "", "", "", false, persistErr
}
if step.Status == "completed" {
values, copyErr := copyValues(stage.Values(step.Output))
if copyErr != nil {
return "", "", "", false, copyErr
}
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
r.notifyStep(id, name, workspace, selected.Notify, step)
state.input = stage.Values(step.Output)
status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
return status, outcome, diagnostic, terminal, nil
}
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
var held *pipeline.ResourceLease
resourceQuiet := true
release := func() error {
if held != nil {
err := r.releaseResource(*held, resourceQuiet)
held = nil
return err
}
return nil
}
defer func() {
if err := release(); err != nil {
log.Printf("resource cleanup run=%d: %v", id, err)
}
}()
finish := func(status, outcome, diagnostic string) {
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
}
defer func() {
if recovered := recover(); recovered != nil {
finish("failed", "error", "stage execution panicked")
}
}()
budgets := map[string]*stage.Budget{}
if state.key == "" {
finish("failed", "error", "stage returned an undeclared outcome")
return
}
// Resolve the effective provider/model for every model stage once, before the
// loop, so applying an override is an in-memory O(1) map lookup per visit and
// never re-parses the graph or re-resolves the whole definition. A run with no
// overrides skips the resolution entirely. Launch validation already ran this
// helper, so a failure here only guards a resumed run whose overrides drifted.
var effectiveModels map[string]pipeline.ModelOverride
if request.Model != "" || len(request.StageModels) > 0 {
resolved, err := pipeline.EffectiveModels(graph, request.Model, request.StageModels)
if err != nil {
finish("failed", "error", err.Error())
return
}
effectiveModels = resolved
}
for sequence := state.nextSequence; sequence <= 30; sequence++ {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
key := state.key
var def stage.Definition
var selected protocol.Stage
found := false
for _, s := range graph.Stages {
if s.Key == key {
selected = s
def = pipeline.StageDefinition(s)
found = true
break
}
}
if !found {
finish("failed", "error", "missing stage")
return
}
// StageDefinition clones Configs, so setting the effective pair on the
// clone is safe and the pinned graph is never mutated. Because
// ModelStage.Spec reads def.Configs and passes spec.Config into the repair
// path, the same effective model governs the stage and its repair.
if stage.EffectiveType(def.Type) == stage.ModelType {
if override, ok := effectiveModels[key]; ok {
def.Configs["provider"] = override.Provider
def.Configs["model"] = override.Model
}
}
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
maxVisits := 5
if selected.Limits != nil {
maxVisits = selected.Limits.MaxVisits
}
// Validate the next visit without consuming it. An already-exhausted
// stage must not queue for a resource it can never execute.
if !isHuman {
if state.visits[key]+1 > 5 {
finish("failed", "error", "stage visit budget exceeded")
return
}
if state.visits[key]+1 > maxVisits {
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
return
}
}
_, declaresSkip := def.Edges["skipped"]
skip := !isHuman && declaresSkip && !state.uiSurface
if selected.Resources != nil && !skip {
if isHuman || !pipeline.ValidResourceKey(selected.Resources.Exclusive) {
finish("failed", "error", "invalid persisted resource declaration")
return
}
lease, err := r.acquireResource(ctx, pipeline.ResourceVisit{Resource: selected.Resources.Exclusive, RunID: id, Stage: key, Sequence: sequence}, workspace, state.completed)
if err != nil {
if ctx.Err() != nil {
finish("canceled", "", err.Error())
} else {
finish("failed", "error", err.Error())
}
return
}
held, resourceQuiet = &lease, true
}
state.nextSequence = sequence + 1
state.visits[key]++
// Publish the run's server-captured changed paths into the input memory
// value before resolving bindings, so a stage selecting
// input.server_changed_files always resolves (an empty list when nothing
// was captured). Only a selecting stage copies it into its input.
state.publishServerChangedFiles()
var reads map[string]protocol.MemoryRead
input, reads, copyErr := memoryInput(state.input, selected.Memory, state.memory)
if copyErr != nil {
finish("failed", "error", copyErr.Error())
return
}
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
finish("failed", "error", err.Error())
return
}
// queued -> running: report the new current stage and the step start.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
ResourceWaitSet: true,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
}, false)
if isHuman {
// A human step is O(1): one BeginStep plus one MarkWaiting, no
// filediff capture, no artifacts directory and no workspace mutation.
evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges))
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
finish("failed", "error", err.Error())
return
}
waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence}
// A human step parks the run without a live goroutine; the delta is
// coalesced by the hub's timer, which keeps running after this returns.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusWaiting,
CurrentStage: key,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
}, false)
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
r.notifyRun(id, name, protocol.RunStatusWaiting, sequence, key, "")
return
}
// A stage that declares a `skipped` route is short-circuited when the run
// has recorded no UI surface: the stage would answer exactly that, so skip
// the model call and its capture work, record the step as completed and
// route along the declared edge. Only a declared route is honoured, and no
// other stage or outcome routing changes.
if skip {
reason := skipReasonLine(state)
step := protocol.RunStep{
Sequence: sequence,
Stage: key,
Status: "completed",
Outcome: "skipped",
Output: stage.Values{"summary": reason},
Target: def.Edges["skipped"],
Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads},
}
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
if finishErr != nil {
finish("failed", "error", finishErr.Error())
return
}
if terminal {
finish(status, outcome, diagnostic)
return
}
state.key = step.Target
continue
}
var tools []string
explicitTools := false
var toolErr error
if stage.EffectiveType(def.Type) == stage.ModelType {
tools, explicitTools, toolErr = stage.ToolNames(def)
}
if toolErr != nil {
finish("failed", "error", toolErr.Error())
return
}
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
// Every agent visit gets a per-run/step directory for the files it
// produces. Creation is best-effort: an unusable directory must never
// fail the run, and it is only attempted for model stages.
artifactsDir := ""
if stage.EffectiveType(def.Type) == stage.ModelType {
candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence))
if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil {
log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr)
} else {
artifactsDir = candidate
def.ArtifactsDir = candidate
}
}
operations := &stage.OperationLog{}
var landing stage.LandingAccess
delivery, deliveryErr := stage.DeliveryEnabled(def)
if deliveryErr != nil {
finish("failed", "error", deliveryErr.Error())
return
}
if delivery {
access, accessErr := landingAccess(workspace)
if accessErr != nil {
finish("failed", "error", "delivery access denied: "+accessErr.Error())
return
}
landing = access
}
client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools)
client.LandingAccess = landing
client.OnSession = func(session string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
}
var result stage.Result
var piClient *stage.PiClient
var err error
stageCtx := ctx
var modelConfig stage.ModelConfig
var budgetBefore stage.BudgetSnapshot
if stage.EffectiveType(def.Type) == stage.ModelType {
spec, specErr := (stage.ModelStage{}).Spec(def)
if specErr != nil {
finish("failed", "error", specErr.Error())
return
}
modelConfig = spec.Config
if budgets[key] == nil {
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
}
// The budget is shared across every visit to this stage key, so capture
// its cumulative usage before the visit and record only this visit's
// increment on the step, never the cumulative total again.
budgetBefore = budgets[key].Snapshot()
stageCtx = stage.WithBudget(ctx, budgets[key])
}
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
before := filediff.Capture(snapshotCtx, workspace)
snapshotCancel()
var stageSession string
panicked := false
if held != nil {
resourceQuiet = false
}
func() {
defer func() {
if recover() != nil {
panicked = true
err = fmt.Errorf("stage execution panicked")
}
}()
if ctx.Err() != nil {
err = ctx.Err()
return
}
if r.execute != nil {
timeout, _ := stage.StageTimeout(def)
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
defer cancel()
result, err = r.execute(executeCtx, def, input)
} else if modelConfig.Harness == "pi" {
configured, configErr := harness.Load()
if configErr != nil {
err = configErr
return
}
profile, ok := configured.Profiles["pi"]
if !ok {
err = fmt.Errorf("Pi is not configured on execution server")
return
}
piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools)
piClient.OnOwnership = func(session, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
}
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
stageSession = piClient.SessionID
client.Transcript = piClient.Transcript
} else if useAgent {
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
stageSession = client.SessionID
} else if stage.EffectiveType(def.Type) == stage.ModelType {
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if active {
stageSession = session
}
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
}
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
client.Transcript = plain.FormatAudit
} else {
result, err = stage.DefaultRegistry().Run(ctx, def, input)
}
}()
if held != nil {
resourceQuiet = r.resourceQuiescent(id, err, panicked)
if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true {
resourceQuiet = false
}
}
// Capture after cleanup, even when the stage's context was canceled.
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
after := filediff.Capture(snapshotCtx, workspace, before)
snapshotCancel()
changes := filediff.Compare(before, after)
if errors.Is(err, stage.ErrStageCleanup) {
changes.Status = "partial"
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
}
// Export the stage's session into its own artifacts directory before the
// bounded scan, so the existing listing and serving paths find the files
// with no special casing. This is bounded, best-effort evidence: it never
// waits for session idleness and a failure never changes the stage's
// status, outcome, target or result validation.
var sessionExport *protocol.SessionExport
if artifactsDir != "" {
if owner, ownerErr := r.store.RunSessionOwner(ctx, id, sequence); ownerErr == nil && owner.ID != "" {
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
}
}
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir)
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, SessionExport: sessionExport, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
// Stamp the effective provider/model this visit actually ran with for every
// model stage, including failures, so the app/TUI evidence reflects any
// override. Non-model stages keep empty values. The usage the stage already
// measured for budget accounting is carried onto the same step, never
// re-read from the harness or parsed from the transcript.
if stage.EffectiveType(def.Type) == stage.ModelType {
step.Provider = modelConfig.Provider
step.Model = modelConfig.Model
if budget := budgets[key]; budget != nil {
step.Usage = stageUsage(budget.Snapshot().Since(budgetBefore))
}
}
if budget := budgets[key]; budget != nil {
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
}
if err != nil {
step.Status = "failed"
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
if len(result.Output) > 0 {
step.Output["salvage"] = result.Output
}
}
if err == nil && result.Outcome == "success" {
target, targetErr := stage.VerificationTargetFor(def)
if targetErr != nil {
err = targetErr
} else if target != nil {
if r.verifyTarget == nil {
err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
} else {
evidence, verificationErr := r.verifyTarget(stageCtx, *target)
if verificationErr != nil {
err = verificationErr
} else {
err = stage.VerifyTargetEvidence(*target, evidence)
}
}
}
if err != nil {
step.Status = "failed"
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
}
}
if err == nil && delivery && result.Outcome == "success" {
checks, checksErr := stage.DeliveryChecks(def)
if checksErr != nil {
err = checksErr
} else {
gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout)
err = verifyDelivery(gateCtx, workspace, landing, checks)
gateCancel()
}
if err != nil {
step.Status = "failed"
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
}
}
// A failed or timed-out step carries the session pointer even when the
// bounded transcript would otherwise be only its stage_limits envelope.
if err != nil && stageSession != "" {
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
}
step.Target = def.Edges[step.Outcome]
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
for _, outcome := range selected.Limits.On {
if outcome == step.Outcome {
step.Control.LimitReached = true
step.Control.ModelTarget = step.Target
step.Target = protocol.FailureEnd
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
}
}
}
if panicked {
step.Target = protocol.FailureEnd
}
// Generate the one-line verdict reason once, at completion. It is
// best-effort: a summariser failure or timeout returns NoSummary and must
// never fail the stage or the run, so every error is swallowed. A failed
// step still gets a verdict from its error envelope.
if r.verdict != nil {
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
}
reasonCancel()
}
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
if finishErr != nil {
finish("failed", "error", finishErr.Error())
return
}
// Advance the in-memory UI-surface evidence only after the step is
// persisted, so the skip guard agrees with what the store holds.
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.completed++
if artifactsDir != "" {
state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence))
}
}
// Step finished: report its outcome, routing target, diagnostic and the
// artifacts recorded for the visit, plus the run's running step count.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
CompletedSteps: state.completed,
Artifacts: len(step.Artifacts),
Step: &protocol.RunStepDelta{
Sequence: step.Sequence,
Stage: step.Stage,
Status: step.Status,
Outcome: step.Outcome,
Target: step.Target,
Diagnostic: step.Diagnostic,
Artifacts: len(step.Artifacts),
},
}, false)
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
if checkErr != nil || len(sessions) > 0 {
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
return
}
}
if held != nil && !resourceQuiet {
finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection")
return
}
if err := release(); err != nil {
finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error())
return
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
if terminal {
finish(status, outcome, diagnostic)
return
}
state.key = step.Target
}
finish("failed", "error", "total stage budget exceeded")
}
// exportStageSession writes the finished stage's session into its own artifacts
// directory and returns the evidence for the step. A non-OpenCode harness (pi)
// has no native export path, so it records that limitation and writes no files.
// Any export error is recorded as a sanitized diagnostic and never returned to
// the stage: the export is read-only, bounded and best-effort.
func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport {
evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend}
if owner.Backend != "v1" && owner.Backend != "v2" {
evidence.Source = "unavailable"
evidence.Diagnostic = "session export unavailable: harness has no native export path"
return evidence
}
export := r.exportSession
if export == nil {
export = stage.ExportSession
}
// A canceled stage still gets one bounded, non-blocking attempt; the export
// never consumes the stage timeout and never waits for the session.
exportCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stage.SessionExportTimeout)
defer cancel()
result, err := export(exportCtx, stage.SessionExportRequest{
SessionID: owner.ID,
Backend: owner.Backend,
Directory: owner.Directory,
ArtifactsDir: artifactsDir,
RunID: runID,
Sequence: sequence,
Stage: key,
Provider: modelConfig.Provider,
Model: modelConfig.Model,
})
if err != nil {
evidence.Source = "native_export"
evidence.Diagnostic = stage.RedactEvidence(err.Error())
return evidence
}
evidence.Source = result.Source
evidence.HTML = result.HTML
evidence.HTMLBytes = result.HTMLBytes
evidence.Messages = result.Messages
evidence.Truncated = result.Truncated
evidence.Diagnostic = result.Diagnostic
return evidence
}
// setTitle renames a run: it replaces the stored operator-facing title without
// touching the run's input, steps or execution, then returns the updated detail.
func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) {
explicit, ok := summarise.ExplicitTitle(title)
if !ok {
return nil, fmt.Errorf("a title with usable text is required")
}
if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil {
return nil, err
}
return r.getRun(id)
}