package server import ( "context" "encoding/json" "errors" "fmt" "log" "maps" "os" "path/filepath" "strconv" "strings" "sync" "time" "git.a-wan.net/alan/pipelines/internal/filediff" "git.a-wan.net/alan/pipelines/internal/harness" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" "git.a-wan.net/alan/pipelines/internal/summarise" ) // activeRun records one in-flight run's cancellation seam and the resolved // workspace it is changing. A single map is the only active-run state, so the // workspace cannot drift from the cancel function. type activeRun struct { cancel context.CancelFunc workspace string } type runner struct { ctx context.Context store *pipeline.Store artifactsRoot string mu sync.Mutex active map[int64]activeRun execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error) // title generates the operator-facing run title from the run prompt. When nil // the store derives a fallback from the prompt with no model call, so tests // and unconfigured servers stay deterministic. title func(context.Context, string) summarise.Title // verdict generates the bounded one-line reason for a completed stage visit // from its output map. When nil the store derives the reason at read time with // no model call. It is best-effort: any failure must never fail the stage. verdict func(context.Context, map[string]any) string // events pushes run state deltas to subscribed clients. When nil (tests that // build a runner directly) emits are dropped, so nothing else changes. events *runEventHub // identMu guards idents: the pipeline identity and stored title (see // runIdentity) that every emitted frame of a run carries, so a client with no // cached row renders a brand-new run without re-reading the list. Populated // where a run is created, decided or canceled; a run with no remembered // identity keeps blank identity fields and the client resyncs once instead. identMu sync.Mutex idents map[int64]runIdentity // exportSession writes a finished stage's session into its own artifacts // directory. It defaults to stage.ExportSession; a failure is best-effort and // never changes the stage's status, outcome, target or result validation. exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) // verifyTarget is installed only by server wiring for a controlled fixture // workflow. Leaving it nil fails closed for target-aware definitions. verifyTarget stage.VerificationRunner wg sync.WaitGroup schedulerWG sync.WaitGroup // attentionMu guards attentionSent, the per-run set of run-attention events // already delivered — by a stage's own notify policy or by the run-level // default — so neither republishes the same genuine event. attentionMu sync.Mutex attentionSent map[int64]map[string]bool } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { sessions, err := s.ActiveSessionOwners(ctx) if err != nil { return nil, err } for _, owner := range sessions { stop, cancel := context.WithTimeout(ctx, 20*time.Second) var err error if owner.Backend == "pi" { err = stage.InterruptPi(stop, owner.Reference) } else { err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID) } cancel() if err != nil { if resource, checkErr := s.HasResourceOwner(ctx, owner.RunID); checkErr == nil && resource { log.Printf("resource owner run=%d remains quarantined: cleanup not confirmed", owner.RunID) continue } return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } if err := s.ConfirmRecoveredSession(ctx, owner); err != nil { return nil, err } } if err := s.RecoverRuns(ctx); err != nil { return nil, err } root := filepath.Join(s.Directory(), "artifacts") return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, exportSession: stage.ExportSession}, nil } func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) { if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 { return nil, fmt.Errorf("an explicit version and request key are required") } workspace, err := os.Getwd() if err != nil { return nil, err } workspace, err = filepath.EvalSymlinks(workspace) if err != nil { return nil, err } if request.Workspace != "" { if !filepath.IsAbs(request.Workspace) { return nil, fmt.Errorf("run workspace must be an absolute directory") } selected, err := filepath.EvalSymlinks(request.Workspace) if err != nil { return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err) } info, err := os.Stat(selected) if err != nil || !info.IsDir() { return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace) } workspace = selected } if len(request.WritePaths) > 20 { return nil, fmt.Errorf("too many write paths") } for _, path := range request.WritePaths { if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") { return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths") } parent := filepath.Join(workspace, path) for { _, err := os.Lstat(parent) if err == nil { break } if !os.IsNotExist(err) { return nil, err } next := filepath.Dir(parent) if next == parent { return nil, fmt.Errorf("write path is unavailable") } parent = next } resolved, err := filepath.EvalSymlinks(parent) if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) { return nil, fmt.Errorf("write path must stay within the run directory") } } request.Input = maps.Clone(request.Input) if request.Input == nil { request.Input = map[string]any{} } request.Input["pwd"] = workspace if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" { return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt") } d, err := r.store.Get(r.ctx, request.PipelineID, request.Version) if err != nil { return nil, err } if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil { return nil, err } // Validate the launch-time model overrides against the pinned graph before // admission: an unknown stage key or a malformed provider/model must never // insert a run row. The resolution is O(#overrides + #stages) with no model or // network call, and the same helper is the authoritative precedence rule. if _, err := pipeline.EffectiveModels(d.Graph, request.Model, request.StageModels); err != nil { return nil, err } for _, s := range d.Graph.Stages { if stage.EffectiveType(s.Type) != stage.ModelType { continue } if s.Configs["harness"] == "pi" && r.execute == nil { configured, err := harness.Load() if err != nil { return nil, err } if _, ok := configured.Profiles["pi"]; !ok { return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there") } } tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s)) if err != nil { return nil, err } if !explicit { request.AllowTools = true } for _, tool := range tools { if tool == "edit" || tool == "shell" { request.AllowTools = true } } } // Decide admission and idempotency under the lock, but generate the title // only for a request that will actually start a run: a rejected or replayed // request must not pay for a bounded model generation, and a live one must // not hold the lock (stalling listings and cancellation) while a model is // called. An unavailable or slow model falls back to the prompt's first line. r.mu.Lock() if r.ctx.Err() != nil { r.mu.Unlock() return nil, r.ctx.Err() } owners, err := r.store.ActiveSessionOwners(r.ctx) if err != nil { r.mu.Unlock() return nil, err } // Refuse only an orphaned session: one whose owning run is not active now. // A session owned by a currently active run (another run in another // workspace) must not block new work. for _, owner := range owners { if _, ok := r.active[owner.RunID]; !ok { if resource, checkErr := r.store.HasResourceOwner(r.ctx, owner.RunID); checkErr == nil && resource && owner.Directory != workspace { continue } r.mu.Unlock() return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") } } // Idempotent repeats are accepted even while the original run is active, and // a replay reuses the stored title instead of generating a new one. existing, err := r.store.RunByKey(r.ctx, request.RequestKey) if err != nil { r.mu.Unlock() return nil, err } if existing == 0 { if err := r.checkWorkspaceFree(workspace); err != nil { r.mu.Unlock() return nil, err } } r.mu.Unlock() var runTitle summarise.Title if existing == 0 { // An explicit operator title skips the model call; otherwise the // configured generator names the run, falling back to the prompt's // first line when it is unavailable or slow. if explicit, ok := summarise.ExplicitTitle(request.Title); ok { runTitle = explicit } else if r.title != nil { prompt, _ := request.Input["prompt"].(string) runTitle = r.title(r.ctx, prompt) } } r.mu.Lock() defer r.mu.Unlock() // Another run may have reserved this workspace while the title was // generated, so re-check before reserving it; a race on the request key is // handled by the store's idempotent insert. if existing == 0 { if err := r.checkWorkspaceFree(workspace); err != nil { return nil, err } } id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Model: request.Model, StageModels: request.StageModels}, runTitle, retryOf...) if err != nil { return nil, err } // The run row now exists: read it once for its resolved identity (pipeline // id/name, the store-derived title, the pinned version), so the queued frame // and every later frame carry what a client needs to render a brand-new run // without re-reading the list. A failed read only costs that seed: the run is // still launched below and its frames carry no identity, so the client falls // back to its one resync. detail, readErr := r.store.GetRun(r.ctx, id) if readErr != nil { detail = nil } else if detail != nil { r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) } if created { if request.RetrySource > 0 && request.RetryMode != "restart" { checkpoint, checkpointErr := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput) if checkpointErr != nil { return nil, checkpointErr } if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil { return nil, err } } // Emit the created delta before launching: the queued frame reaches // subscribers before the running/step-started transition, so a client // sees the run appear and then advance. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusQueued, CreatedAt: time.Now().UTC().Format(time.RFC3339), }, false) r.launch(id, workspace, func(ctx context.Context) { r.run(ctx, id, d, workspace, request) }) } if detail != nil { _, detail.Executing = r.active[id] r.overlayEffectiveModels(detail) } if readErr != nil { return nil, readErr } return detail, nil } // launch registers an active run's cancellation seam and workspace, adds it to // the wait group and starts fn on a derived cancellable context that cleans up // the registration when it returns. A fresh run and a resumed decision both go // through it, so registration and teardown live once. Callers must hold r.mu. func (r *runner) launch(id int64, workspace string, fn func(context.Context)) { ctx, cancel := context.WithCancel(r.ctx) r.active[id] = activeRun{cancel: cancel, workspace: workspace} r.wg.Add(1) go func() { defer r.wg.Done() defer cancel() defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }() fn(ctx) }() } // checkWorkspaceFree refuses a run when an active run is already changing the // same resolved workspace. The workspace is the only scope a run reserves, and // one active run per workspace keeps this scan finding at most one match, so the // message is deterministic. Callers must hold r.mu. func (r *runner) checkWorkspaceFree(workspace string) error { for id, entry := range r.active { if entry.workspace == workspace { return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace) } } return nil } func (r *runner) getRun(id int64) (*protocol.RunDetail, error) { r.mu.Lock() defer r.mu.Unlock() d, err := r.store.GetRun(r.ctx, id) if d != nil { _, d.Executing = r.active[id] for _, s := range d.Steps { if s.AgentActive { d.Executing = true } } r.overlayEffectiveModels(d) } return d, err } // overlayEffectiveModels replaces each model stage's provider/model in a returned // run's graph with the effective pair the run actually uses, so the app and TUI // stage-definition surfaces show the override with no client change. It is // presentation only: the persisted graph_json and the pinned source returned by // `show` are never rewritten. A run with no overrides is left untouched. func (r *runner) overlayEffectiveModels(d *protocol.RunDetail) { if d == nil || (d.Model == "" && len(d.StageModels) == 0) { return } effective, err := pipeline.EffectiveModels(d.Graph, d.Model, d.StageModels) if err != nil { return } for i := range d.Graph.Stages { s := &d.Graph.Stages[i] if stage.EffectiveType(s.Type) != stage.ModelType { continue } override, ok := effective[s.Key] if !ok { continue } if s.Configs == nil { s.Configs = map[string]string{} } s.Configs["provider"] = override.Provider s.Configs["model"] = override.Model } } func (r *runner) listRuns() ([]protocol.RunSummary, error) { r.mu.Lock() defer r.mu.Unlock() items, err := r.store.ListRuns(r.ctx) for i := range items { _, items[i].Executing = r.active[items[i].ID] if items[i].Status == "failed" && !items[i].Executing { d, err := r.store.GetRun(r.ctx, items[i].ID) if err != nil { return nil, err } for _, step := range d.Steps { if step.AgentActive { items[i].Executing = true } } } } return items, err } type retryCheckpoint struct { detail *protocol.RunDetail failed protocol.RunStep handoff map[string]any } const maxRetryInputBytes = 64 * 1024 func retryInputOverride(base map[string]any, override map[string]any) (map[string]any, error) { if override == nil { return base, nil } encoded, err := json.Marshal(override) if err != nil || len(encoded) > maxRetryInputBytes { return nil, fmt.Errorf("retry input must be a JSON object no larger than %d bytes", maxRetryInputBytes) } reserved := map[string]bool{"pwd": true, "workspace": true, "write_paths": true, "allow_tools": true, "pipeline_id": true, "pipeline_version": true, "request_key": true, "retry_mode": true, "retry_source": true, "retry_stage": true, "retry_visit": true} merged := map[string]any{} for key, value := range base { merged[key] = value } for key, value := range override { if reserved[key] { return nil, fmt.Errorf("retry input field %q is reserved", key) } merged[key] = value } return merged, nil } // retryStageContractEqual compares the complete persisted stage contract. A // retry may carry completed output across a published version only when the // stages that produced that output are unchanged; comparing only effective // types would allow prompt, config, edge, resource, or limit changes through. func retryStageContractEqual(left, right protocol.Stage) bool { a, err := json.Marshal(left) if err != nil { return false } b, err := json.Marshal(right) if err != nil { return false } return string(a) == string(b) } func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) { return r.retrySourceCheckpointSelected(sourceID, pinned, "", 0, nil) } func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.PipelineDetail, selectedStage string, selectedVisit int, override map[string]any) (retryCheckpoint, error) { source, err := r.store.GetRun(r.ctx, sourceID) if err != nil { return retryCheckpoint{}, err } if source.Status != "failed" || source.FinishedAt == "" { return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID) } if source.PipelineID != pinned.ID { return retryCheckpoint{}, fmt.Errorf("retry source %d belongs to a different pipeline", sourceID) } if len(source.Steps) == 0 { return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID) } failed := source.Steps[len(source.Steps)-1] if selectedStage != "" { failed = protocol.RunStep{} visits := 0 for _, candidate := range source.Steps { if candidate.Stage != selectedStage { continue } visits++ if selectedVisit == 0 && visits == 1 { failed = candidate } if selectedVisit > 0 && visits == selectedVisit { failed = candidate } } if failed.Stage == "" { return retryCheckpoint{}, fmt.Errorf("retry stage %q was not recorded in run %d", selectedStage, sourceID) } if selectedVisit == 0 { count := 0 for _, candidate := range source.Steps { if candidate.Stage == selectedStage { count++ } } if count > 1 { return retryCheckpoint{}, fmt.Errorf("retry stage %q was visited %d times; --visit is required", selectedStage, count) } } if selectedVisit > visits { return retryCheckpoint{}, fmt.Errorf("retry stage %q has no visit %d", selectedStage, selectedVisit) } } if (failed.Status != "completed" && failed.Status != "failed") || (selectedStage == "" && failed.Target != protocol.FailureEnd) { return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID) } var handoff map[string]any if failed.Input == nil { return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID) } if _, err := retryInputOverride(failed.Input, override); err != nil { return retryCheckpoint{}, err } // Keep the source-stage input immutable in the checkpoint. The selected // stage's memory bindings must be resolved from the original prefix before // this override is applied at invocation time. handoff = map[string]any{"input": failed.Input, "override": override, "sequence": failed.Sequence, "stage": failed.Stage, "visit": failed.Sequence} prefix := map[string]any{} for _, step := range source.Steps { if step.Sequence >= failed.Sequence || step.Status != "completed" { continue } prefix[step.Stage] = step.Output } handoff["prefix"] = prefix // A newer explicitly selected version may continue only when every recorded // prefix stage still exists with the same kind. Its outputs are evidence from // the source attempt and cannot be safely handed to a changed stage contract. sourceStages := map[string]protocol.Stage{} for _, declared := range source.Graph.Stages { sourceStages[declared.Key] = declared } for _, step := range source.Steps { if step.Sequence >= failed.Sequence { break } var pinnedStage *protocol.Stage for i := range pinned.Graph.Stages { if pinned.Graph.Stages[i].Key == step.Stage { pinnedStage = &pinned.Graph.Stages[i] break } } previous, ok := sourceStages[step.Stage] if pinnedStage == nil || !ok || !retryStageContractEqual(previous, *pinnedStage) { return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, step.Stage) } } var sourceSelected protocol.Stage var sourceSelectedOK bool if sourceSelected, sourceSelectedOK = sourceStages[failed.Stage]; !sourceSelectedOK { return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in its recorded graph", sourceID) } pinnedStages := map[string]protocol.Stage{} for _, declared := range pinned.Graph.Stages { pinnedStages[declared.Key] = declared } // The retry will continue through every stage that the source execution // already recorded after the selected checkpoint. Validate those stages // before admitting the new run as well: otherwise an explicitly selected // version can pass prefix/selected-stage validation and fail later in the // loop with the less useful "missing stage" error. for _, step := range source.Steps { if step.Sequence <= failed.Sequence { continue } if _, ok := pinnedStages[step.Stage]; !ok { return retryCheckpoint{}, fmt.Errorf("retry source %d selected version is missing later stage %q", sourceID, step.Stage) } } for _, declared := range pinned.Graph.Stages { if declared.Key != failed.Stage { continue } if stage.EffectiveType(sourceSelected.Type) != stage.EffectiveType(declared.Type) { return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible selected stage %q", sourceID, failed.Stage) } typ := stage.EffectiveType(declared.Type) if typ == stage.HumanType || typ == stage.NtfyType { return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage) } if typ == stage.ModelType { if _, _, toolErr := stage.ToolNames(pipeline.StageDefinition(declared)); toolErr != nil { return retryCheckpoint{}, toolErr } } return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil } return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID) } func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("retry requires a positive run ID") } original, err := r.getRun(request.RunID) if err != nil { return nil, err } if original.Executing { return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID) } if original.Status != "failed" || original.FinishedAt == "" { return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status) } if request.RequestKey == "" { return nil, fmt.Errorf("retry requires an idempotency request key") } // Resolve an already admitted request before validating optional retry // selectors. This makes replay after a restart genuinely idempotent even // when the original request used an input override that the replay omits. if existing, err := r.store.RunByKey(r.ctx, request.RequestKey); err != nil { return nil, err } else if existing > 0 { return r.store.GetRun(r.ctx, existing) } if request.RetryVisit < 0 || (request.RetryVisit > 0 && request.RetryStage == "") { return nil, fmt.Errorf("retry visit requires a positive --stage visit selection") } mode := request.RetryMode if mode == "" { mode = "resume" } if mode != "resume" && mode != "restart" { return nil, fmt.Errorf("retry mode must be resume or restart") } version := original.Version if request.RetryVersion > 0 { version = request.RetryVersion } d, err := r.store.Get(r.ctx, original.PipelineID, version) if err != nil { return nil, err } if mode == "resume" { if _, err := r.retrySourceCheckpointSelected(original.ID, d, request.RetryStage, request.RetryVisit, request.RetryInput); err != nil { return nil, err } } return r.start(protocol.Message{PipelineID: original.PipelineID, Version: version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, RetryStage: request.RetryStage, RetryVisit: request.RetryVisit, RetryInput: request.RetryInput, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID) } func (r *runner) cancel(id int64) error { r.mu.Lock() entry, ok := r.active[id] r.mu.Unlock() if ok { entry.cancel() return nil } // A run that is waiting on a person has no live goroutine, so cancel it at // the persistence layer instead. This keeps cancel working across a restart. // The single follow-up read seeds the terminal delta; cancel is a cold // operator action, not a per-transition hot path. if err := r.store.CancelWaiting(r.ctx, id); err != nil { return err } // A canceled run is terminal: nothing more can notify for it. r.forgetAttention(id) if r.events != nil { if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil { r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: detail.Workspace, Status: detail.Status, Outcome: detail.Outcome, CurrentStage: detail.CurrentStage, CompletedSteps: detail.CompletedSteps, FinishedAt: detail.FinishedAt, }, true) } } return nil } // decide records an operator's choice for a waiting human stage and resumes the // run along the chosen edge. ResolveDecision owns validation and idempotency, so // decide does no option parsing of its own. A resumed run is registered and // launched exactly like a fresh one but starts from the persisted state. func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("decide requires a positive run ID") } r.mu.Lock() detail, err := r.store.GetRun(r.ctx, request.RunID) if err != nil { r.mu.Unlock() return nil, err } if _, busy := r.active[detail.ID]; busy { r.mu.Unlock() return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID) } if detail.Status == protocol.RunStatusWaiting { if err := r.checkWorkspaceFree(detail.Workspace); err != nil { r.mu.Unlock() return nil, err } } step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback) if err != nil { r.mu.Unlock() return nil, err } if applied { // Re-seed the identity so the resumed frames (this one and the resumed // loop's) carry it even after a restart, when no creation was observed. r.rememberRunIdentity(detail.ID, identityOfRunSummary(detail.RunSummary)) for i := range detail.Steps { if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage { detail.Steps[i] = step break } } // The decision closes the waiting step and returns the run to running; // report that before the resumed loop emits its own deltas. r.emitRun(protocol.RunEventFrame{ RunID: detail.ID, Workspace: detail.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{ Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, }, }, false) r.launch(detail.ID, detail.Workspace, func(ctx context.Context) { r.resume(ctx, detail.ID, detail, detail.Workspace, step) }) } r.mu.Unlock() return r.getRun(request.RunID) } func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) { if request.RetrySource > 0 && request.RetryMode != "restart" { source, err := r.store.GetRun(ctx, request.RetrySource) if err != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) return } checkpoint, err := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput) if err != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) return } storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id) if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil { if checkpointErr == nil { checkpointErr = fmt.Errorf("retry checkpoint is incomplete") } r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil) return } if input, ok := storedHandoff["input"].(map[string]any); ok { checkpoint.failed.Input = input } else { r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil) return } state := rebuildRetryState(source, checkpoint.failed) if override, ok := storedHandoff["override"].(map[string]any); ok { state.retryOverride = stage.Values(override) } r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state) return } original, copyErr := copyValues(stage.Values(request.Input)) if copyErr != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input)) return } input := stage.Values(request.Input) state := newRunState(d.Graph.Stages[0].Key, input) state.memory["input"] = memoryValue{output: original} r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state) } // terminalFinish classifies a stage's resolved target. A terminal key yields the // run's final status, outcome and diagnostic; an empty target means the stage // named an undeclared outcome; any other value is a stage key and is not // terminal. resume and the loop tail share it, so terminal routing lives once. func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) { switch target { case protocol.SuccessEnd: return "succeeded", "success", "", true case protocol.FailureEnd: if diagnostic == "" { diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome) } return "failed", "error", diagnostic, true case "": return "failed", "error", "stage returned an undeclared outcome", true } return "", "", "", false } // resume continues a decided run from the state already persisted for its run // detail: memory from the run input and the completed steps, visits counted per // stage, and the next sequence above the highest recorded one. It then runs the // shared loop from the decision's target; no full-table scan and no extra // per-step query are introduced. func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) { // A resumed run's frames carry the same identity as its creation, so re-seed // it here too: resume is also reachable from a store read after a restart. r.rememberRunIdentity(id, identityOfRunSummary(d.RunSummary)) // The decision may route straight to a terminal, which has no stage to run. if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal { r.finishRunState(id, d.Name, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Sequence, step.Stage, stage.Values(step.Output)) return } state := rebuildRunState(d, step) r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools, Model: d.Model, StageModels: d.StageModels}, state) } // runState is the loop's whole mutable state, extracted so a fresh run and a // resumed run share one implementation. nextSequence is set before each visit so // a wait that returns to the caller records where the run will continue. type runState struct { key string nextSequence int visits map[string]int memory map[string]memoryValue input stage.Values // changedPaths retains a bounded sample of the run's recorded changed paths // and changedCount the full total, so the skip guard reads only in-memory // evidence. uiSurface is latched once any recorded change touched a UI // surface. Both are folded in by recordChanges (ui_surface.go). changedPaths []string changedCount int uiSurface bool // capturedPaths is the full, deduplicated, first-seen-ordered set of the // run's server-captured changed paths, distinct from the bounded // changedPaths sample used only by the skip reason. capturedSeen is the // single dedupe set. Both are folded in by recordChanges (ui_surface.go) and // published to a selecting stage as input.server_changed_files. capturedPaths []string capturedSeen map[string]bool // completed counts the run's finished steps, so a delta can report the // running total without re-reading the store. completed int artifactDirs []string // retryOverride is applied once, after the selected stage's memory bindings // have been resolved. It is intentionally not part of run input or memory. retryOverride stage.Values } func newRunState(key string, input stage.Values) *runState { return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input} } func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState { state := newRunState(decided.Target, stage.Values(decided.Output)) if original, err := copyValues(stage.Values(detail.Input)); err == nil { state.memory["input"] = memoryValue{output: original} } for _, step := range detail.Steps { state.visits[step.Stage]++ if step.Sequence >= state.nextSequence { state.nextSequence = step.Sequence + 1 } if step.Status == "completed" { if values, err := copyValues(stage.Values(step.Output)); err == nil { state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } state.completed++ } // Seed the UI-surface accumulator from the persisted evidence so a // resumed run reaches the same skip decision with no extra query. state.recordChanges(step.FileChanges, step.Artifacts) // Only completed, recorded visits are eligible for a later stage's // review capability. The path is reconstructed by the runner below. if step.Status == "completed" { state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) } } return state } // rebuildRetryState restores only the completed prefix. The failed visit is // deliberately not counted and its step/evidence is not copied into the new // attempt, so retries cannot duplicate execution or usage aggregates. func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep) *runState { state := newRunState(failed.Stage, stage.Values(failed.Input)) state.nextSequence = failed.Sequence if original, err := copyValues(stage.Values(detail.Input)); err == nil { state.memory["input"] = memoryValue{output: original} } for _, step := range detail.Steps { if step.Sequence >= failed.Sequence { break } state.visits[step.Stage]++ if step.Status == "completed" { if values, err := copyValues(stage.Values(step.Output)); err == nil { state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } state.completed++ } state.recordChanges(step.FileChanges, step.Artifacts) if step.Status == "completed" { state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) } } return state } // artifactAccess derives exact visit directories; it never consumes a path from // a definition or model result. An unreadable or symlinked root fails closed. func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess { root, err := filepath.EvalSymlinks(r.artifactsRoot) if err != nil { return stage.ArtifactAccess{} } runRoot := filepath.Join(root, strconv.FormatInt(runID, 10)) visitDirectory := func(visit int) (string, bool) { if visit <= 0 { return "", false } directory := filepath.Join(runRoot, strconv.Itoa(visit)) resolved, err := filepath.EvalSymlinks(directory) if err != nil || resolved != directory { return "", false } info, err := os.Stat(resolved) if err != nil || !info.IsDir() { return "", false } return resolved, true } access := stage.ArtifactAccess{} if directory, ok := visitDirectory(sequence); ok { resolved, err := filepath.EvalSymlinks(current) if err != nil || resolved != directory { return access } access.Write = directory access.Read = append(access.Read, directory) } for _, priorSequence := range prior { visit, err := strconv.Atoi(priorSequence) if err != nil || visit <= 0 || visit >= sequence { continue } if directory, ok := visitDirectory(visit); ok { access.Read = append(access.Read, directory) } } return access } func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient { return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} } func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient { return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} } // serverChangedFilesField is the reserved run-input memory field under which the // runner publishes the server-captured changed paths. A stage selects it with // `"server_changed_files":"input.server_changed_files"`, alongside another // stage's declared list, so it sees the verified evidence, not just a model's // claim of authorship. const serverChangedFilesField = "server_changed_files" // capturedList returns the full, first-seen-ordered set of server-captured // changed paths, or a non-nil empty list when nothing was captured so the field // still resolves as a list. It copies the accumulator so the retained evidence // cannot be aliased by a published value. O(captured paths). func (s *runState) capturedList() []string { if len(s.capturedPaths) == 0 { return []string{} } return append([]string(nil), s.capturedPaths...) } // publishServerChangedFiles writes the run's captured changed-path set into the // run input memory value under serverChangedFilesField, creating the entry and // its output map when absent. Called before memoryInput so a stage selecting // input.server_changed_files always resolves; only that field is added and only // a selecting stage copies it out, so state.input and other stages' input are // unchanged. O(captured paths). func (s *runState) publishServerChangedFiles() { entry, ok := s.memory["input"] if !ok || entry.output == nil { entry = memoryValue{output: stage.Values{}} } entry.output[serverChangedFilesField] = s.capturedList() s.memory["input"] = entry } // runIdentity is the pipeline identity and stored title a run's pushed frames // carry (RunEventFrame.PipelineID/Name/Title/Version). It is remembered where a // run is created, decided or canceled and stamped onto every frame that leaves // emitRun, so a client that has never seen the run can build its list row from // the frame without re-reading the whole list. type runIdentity struct { pipelineID int64 name string title string version int } // identityOfRunSummary is the one projection of a run row onto the identity its // pushed frames carry, so the frame and the list row cannot disagree. func identityOfRunSummary(s protocol.RunSummary) runIdentity { return runIdentity{pipelineID: s.PipelineID, name: s.Name, title: s.Title, version: s.Version} } // rememberRunIdentity records the identity every later frame of run id carries. // It is safe from any goroutine and takes its own lock (never r.mu), so it is // callable from a path that already holds r.mu. func (r *runner) rememberRunIdentity(id int64, ident runIdentity) { if r.idents == nil { return } r.identMu.Lock() r.idents[id] = ident r.identMu.Unlock() } // stampRunIdentity fills a frame's blank identity fields from the run's // remembered identity, leaving any the emit site set untouched. A run with no // remembered identity keeps them blank, so the client falls back to its resync. func (r *runner) stampRunIdentity(frame *protocol.RunEventFrame) { r.identMu.Lock() ident, ok := r.idents[frame.RunID] r.identMu.Unlock() if !ok { return } if frame.PipelineID == 0 { frame.PipelineID = ident.pipelineID } if frame.Name == "" { frame.Name = ident.name } if frame.Title == "" { frame.Title = ident.title } if frame.Version == 0 { frame.Version = ident.version } } // emitRun pushes one run delta to the subscribed-client hub when one is wired, // and does nothing otherwise (a runner built directly in a test has no hub). // terminal marks a final transition, which the hub flushes without coalescing. func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) { if r.events == nil { return } r.stampRunIdentity(&frame) r.events.emit(frame, terminal) } func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil { log.Printf("finish run %d: %v", id, err) } // Push the terminal delta before the attention notification, so a slow ntfy // never delays the live update the app folds. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: status, ResourceWaitSet: true, Outcome: outcome, CurrentStage: currentStage, CompletedSteps: completed, FinishedAt: time.Now().UTC().Format(time.RFC3339), }, true) r.notifyRun(id, name, status, sequence, currentStage, diagnostic) // The run is terminal: nothing more can notify for it. r.forgetAttention(id) } // completedSteps counts the completed steps of a run detail; resume uses it to // seed the terminal delta without another store read. func completedSteps(steps []protocol.RunStep) int { n := 0 for _, step := range steps { if step.Status == "completed" { n++ } } return n } // stageUsage carries the usage a model stage already measured for budget // accounting onto the run record. It reports a cost only when the provider // reported one and an in/out split only when the harness reported it, so an // unknown cost or split is never presented as zero. It returns nil when no usage // was observed, so the step stays absent rather than zero. func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage { if !snapshot.TokensKnown && !snapshot.USDKnown { return nil } usage := &protocol.StageUsage{TokensKnown: snapshot.TokensKnown, CostKnown: snapshot.USDKnown} if snapshot.TokensKnown { usage.TokensTotal = snapshot.UsedTokens if snapshot.SplitKnown { usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true } } if snapshot.USDKnown { usage.CostUSD = snapshot.UsedUSD } return usage } // finishStage persists one step, refreshes the shared memory for a completed // step, notifies and classifies the step's target. Both the executed path and // the skipped path use it, so persistence, notification and terminal routing // cannot drift between them. A persist or memory-copy failure is returned for // the caller to fail the run; the caller still owns the model-specific // verdict, quiescence and cancellation handling around it. func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) { save, cancel := context.WithTimeout(context.Background(), 10*time.Second) persistErr := r.store.FinishStep(save, id, step) cancel() if persistErr != nil { return "", "", "", false, persistErr } if step.Status == "completed" { values, copyErr := copyValues(stage.Values(step.Output)) if copyErr != nil { return "", "", "", false, copyErr } state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } r.notifyStep(id, name, workspace, selected.Notify, step) state.input = stage.Values(step.Output) status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic) return status, outcome, diagnostic, terminal, nil } func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) { var held *pipeline.ResourceLease resourceQuiet := true release := func() error { if held != nil { err := r.releaseResource(*held, resourceQuiet) held = nil return err } return nil } defer func() { if err := release(); err != nil { log.Printf("resource cleanup run=%d: %v", id, err) } }() finish := func(status, outcome, diagnostic string) { r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input) } defer func() { if recovered := recover(); recovered != nil { finish("failed", "error", "stage execution panicked") } }() budgets := map[string]*stage.Budget{} if state.key == "" { finish("failed", "error", "stage returned an undeclared outcome") return } // Resolve the effective provider/model for every model stage once, before the // loop, so applying an override is an in-memory O(1) map lookup per visit and // never re-parses the graph or re-resolves the whole definition. A run with no // overrides skips the resolution entirely. Launch validation already ran this // helper, so a failure here only guards a resumed run whose overrides drifted. var effectiveModels map[string]pipeline.ModelOverride if request.Model != "" || len(request.StageModels) > 0 { resolved, err := pipeline.EffectiveModels(graph, request.Model, request.StageModels) if err != nil { finish("failed", "error", err.Error()) return } effectiveModels = resolved } for sequence := state.nextSequence; sequence <= 30; sequence++ { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } key := state.key var def stage.Definition var selected protocol.Stage found := false for _, s := range graph.Stages { if s.Key == key { selected = s def = pipeline.StageDefinition(s) found = true break } } if !found { finish("failed", "error", "missing stage") return } // StageDefinition clones Configs, so setting the effective pair on the // clone is safe and the pinned graph is never mutated. Because // ModelStage.Spec reads def.Configs and passes spec.Config into the repair // path, the same effective model governs the stage and its repair. if stage.EffectiveType(def.Type) == stage.ModelType { if override, ok := effectiveModels[key]; ok { def.Configs["provider"] = override.Provider def.Configs["model"] = override.Model } } isHuman := stage.EffectiveType(def.Type) == stage.HumanType maxVisits := 5 if selected.Limits != nil { maxVisits = selected.Limits.MaxVisits } // Validate the next visit without consuming it. An already-exhausted // stage must not queue for a resource it can never execute. if !isHuman { if state.visits[key]+1 > 5 { finish("failed", "error", "stage visit budget exceeded") return } if state.visits[key]+1 > maxVisits { finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits)) return } } _, declaresSkip := def.Edges["skipped"] skip := !isHuman && declaresSkip && !state.uiSurface if selected.Resources != nil && !skip { if isHuman || (selected.Resources.Exclusive == "" && selected.Resources.Pool == "") || (selected.Resources.Exclusive != "" && selected.Resources.Pool != "") || (selected.Resources.Exclusive != "" && !pipeline.ValidResourceKey(selected.Resources.Exclusive)) || (selected.Resources.Pool != "" && !pipeline.ValidResourceKey(selected.Resources.Pool)) { finish("failed", "error", "invalid persisted resource declaration") return } resource := selected.Resources.Exclusive pool := selected.Resources.Pool if pool != "" { resource = pool } lease, err := r.acquireResourceWithPool(ctx, pipeline.ResourceVisit{Resource: resource, RunID: id, Stage: key, Sequence: sequence}, pool, workspace, state.completed) if err != nil { if ctx.Err() != nil { finish("canceled", "", err.Error()) } else { finish("failed", "error", err.Error()) } return } held, resourceQuiet = &lease, true } state.nextSequence = sequence + 1 state.visits[key]++ // Publish the run's server-captured changed paths into the input memory // value before resolving bindings, so a stage selecting // input.server_changed_files always resolves (an empty list when nothing // was captured). Only a selecting stage copies it into its input. state.publishServerChangedFiles() var reads map[string]protocol.MemoryRead input, reads, copyErr := memoryInput(state.input, selected.Memory, state.memory) if copyErr != nil { finish("failed", "error", copyErr.Error()) return } if state.retryOverride != nil { input, copyErr = retryInputOverride(input, state.retryOverride) if copyErr != nil { finish("failed", "error", copyErr.Error()) return } // A selected-stage override belongs only to this invocation, not to // a later visit reached through the stage graph. state.retryOverride = nil } if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } finish("failed", "error", err.Error()) return } // queued -> running: report the new current stage and the step start. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CurrentStage: key, ResourceWaitSet: true, CompletedSteps: state.completed, Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, }, false) if isHuman { // A human step is O(1): one BeginStep plus one MarkWaiting, no // filediff capture, no artifacts directory and no workspace mutation. evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges)) if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } finish("failed", "error", err.Error()) return } waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence} // A human step parks the run without a live goroutine; the delta is // coalesced by the hub's timer, which keeps running after this returns. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, CurrentStage: key, CompletedSteps: state.completed, Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting}, }, false) r.notifyStep(id, name, workspace, selected.Notify, waitStep) r.notifyRun(id, name, protocol.RunStatusWaiting, sequence, key, "") return } // A stage that declares a `skipped` route is short-circuited when the run // has recorded no UI surface: the stage would answer exactly that, so skip // the model call and its capture work, record the step as completed and // route along the declared edge. Only a declared route is honoured, and no // other stage or outcome routing changes. if skip { reason := skipReasonLine(state) step := protocol.RunStep{ Sequence: sequence, Stage: key, Status: "completed", Outcome: "skipped", Output: stage.Values{"summary": reason}, Target: def.Edges["skipped"], Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}, } status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step) if finishErr != nil { finish("failed", "error", finishErr.Error()) return } if terminal { finish(status, outcome, diagnostic) return } state.key = step.Target continue } var tools []string explicitTools := false var toolErr error if stage.EffectiveType(def.Type) == stage.ModelType { tools, explicitTools, toolErr = stage.ToolNames(def) } if toolErr != nil { finish("failed", "error", toolErr.Error()) return } useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools) // Every agent visit gets a per-run/step directory for the files it // produces. Creation is best-effort: an unusable directory must never // fail the run, and it is only attempted for model stages. artifactsDir := "" if stage.EffectiveType(def.Type) == stage.ModelType { candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence)) if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil { log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr) } else { artifactsDir = candidate def.ArtifactsDir = candidate } } operations := &stage.OperationLog{} var landing stage.LandingAccess delivery, deliveryErr := stage.DeliveryEnabled(def) if deliveryErr != nil { finish("failed", "error", deliveryErr.Error()) return } if delivery { access, accessErr := landingAccess(workspace) if accessErr != nil { finish("failed", "error", "delivery access denied: "+accessErr.Error()) return } landing = access } client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools) client.LandingAccess = landing client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference) } var result stage.Result var piClient *stage.PiClient var err error stageCtx := ctx if held != nil && held.Target != nil { stageCtx = stage.WithResourceTarget(stageCtx, held.Target) } var modelConfig stage.ModelConfig var budgetBefore stage.BudgetSnapshot if stage.EffectiveType(def.Type) == stage.ModelType { spec, specErr := (stage.ModelStage{}).Spec(def) if specErr != nil { finish("failed", "error", specErr.Error()) return } modelConfig = spec.Config if budgets[key] == nil { budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens) } // The budget is shared across every visit to this stage key, so capture // its cumulative usage before the visit and record only this visit's // increment on the step, never the cumulative total again. budgetBefore = budgets[key].Snapshot() stageCtx = stage.WithBudget(stageCtx, budgets[key]) } snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second) before := filediff.Capture(snapshotCtx, workspace) snapshotCancel() var stageSession string panicked := false if held != nil { resourceQuiet = false } func() { defer func() { if recover() != nil { panicked = true err = fmt.Errorf("stage execution panicked") } }() if ctx.Err() != nil { err = ctx.Err() return } if r.execute != nil { timeout, _ := stage.StageTimeout(def) executeCtx, cancel := context.WithTimeout(stageCtx, timeout) defer cancel() result, err = r.execute(executeCtx, def, input) } else if modelConfig.Harness == "pi" { configured, configErr := harness.Load() if configErr != nil { err = configErr return } profile, ok := configured.Profiles["pi"] if !ok { err = fmt.Errorf("Pi is not configured on execution server") return } piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools) piClient.OnOwnership = func(session, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, "pi", reference) } result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input) stageSession = piClient.SessionID client.Transcript = piClient.Transcript } else if useAgent { result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input) stageSession = client.SessionID } else if stage.EffectiveType(def.Type) == stage.ModelType { plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir} plain.OnGenerationSession = func(session, backend, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if active { stageSession = session } return r.store.RunSession(c, id, sequence, session, active, backend, reference) } result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input) client.Transcript = plain.FormatAudit } else { result, err = stage.DefaultRegistry().Run(stageCtx, def, input) } }() if held != nil { resourceQuiet = r.resourceQuiescent(id, err, panicked) if confirmed, reported := result.Output[protocol.ResourceCleanupConfirmedKey]; reported && confirmed != true { resourceQuiet = false } } // Capture after cleanup, even when the stage's context was canceled. snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) after := filediff.Capture(snapshotCtx, workspace, before) snapshotCancel() changes := filediff.Compare(before, after) if errors.Is(err, stage.ErrStageCleanup) { changes.Status = "partial" changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing." } // Export the stage's session into its own artifacts directory before the // bounded scan, so the existing listing and serving paths find the files // with no special casing. This is bounded, best-effort evidence: it never // waits for session idleness and a failure never changes the stage's // status, outcome, target or result validation. var sessionExport *protocol.SessionExport if artifactsDir != "" { if owner, ownerErr := r.store.RunSessionOwner(ctx, id, sequence); ownerErr == nil && owner.ID != "" { sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig) } } artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir) step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, SessionExport: sessionExport, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated} // Stamp the effective provider/model this visit actually ran with for every // model stage, including failures, so the app/TUI evidence reflects any // override. Non-model stages keep empty values. The usage the stage already // measured for budget accounting is carried onto the same step, never // re-read from the harness or parsed from the transcript. if stage.EffectiveType(def.Type) == stage.ModelType { step.Provider = modelConfig.Provider step.Model = modelConfig.Model if budget := budgets[key]; budget != nil { step.Usage = stageUsage(budget.Snapshot().Since(budgetBefore)) } } if budget := budgets[key]; budget != nil { step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries) } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} if len(result.Output) > 0 { step.Output["salvage"] = result.Output } } if err == nil && result.Outcome == "success" { target, targetErr := stage.VerificationTargetFor(def) if targetErr != nil { err = targetErr } else if target != nil { if r.verifyTarget == nil { err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) } else { evidence, verificationErr := r.verifyTarget(stageCtx, *target) if verificationErr != nil { err = verificationErr } else { err = stage.VerifyTargetEvidence(*target, evidence) } } } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} } } if err == nil && delivery && result.Outcome == "success" { checks, checksErr := stage.DeliveryChecks(def) if checksErr != nil { err = checksErr } else { gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout) err = verifyDelivery(gateCtx, workspace, landing, checks) gateCancel() } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} } } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. if err != nil && stageSession != "" { step.Transcript = stage.RetainSession(step.Transcript, stageSession) } step.Target = def.Edges[step.Outcome] step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads} if held != nil && held.Target != nil { step.Control.ResourceTarget = held.Target } if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits { for _, outcome := range selected.Limits.On { if outcome == step.Outcome { step.Control.LimitReached = true step.Control.ModelTarget = step.Target step.Target = protocol.FailureEnd step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome) } } } if panicked { step.Target = protocol.FailureEnd } // Generate the one-line verdict reason once, at completion. It is // best-effort: a summariser failure or timeout returns NoSummary and must // never fail the stage or the run, so every error is swallowed. A failed // step still gets a verdict from its error envelope. if r.verdict != nil { reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout) if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary { step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel} } reasonCancel() } status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step) if finishErr != nil { finish("failed", "error", finishErr.Error()) return } // Advance the in-memory UI-surface evidence only after the step is // persisted, so the skip guard agrees with what the store holds. state.recordChanges(step.FileChanges, step.Artifacts) if step.Status == "completed" { state.completed++ if artifactsDir != "" { state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence)) } } // Step finished: report its outcome, routing target, diagnostic and the // artifacts recorded for the visit, plus the run's running step count. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CurrentStage: key, CompletedSteps: state.completed, Artifacts: len(step.Artifacts), Step: &protocol.RunStepDelta{ Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts), }, }, false) if useAgent || errors.Is(err, stage.ErrStageCleanup) { sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id) if checkErr != nil || len(sessions) > 0 { finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work") return } } if held != nil && !resourceQuiet { finish("failed", "error", "resource cleanup/restoration not confirmed; ownership quarantined pending inspection") return } if err := release(); err != nil { finish("failed", "error", "resource release failed; retained ownership needs reconciliation: "+err.Error()) return } if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } if terminal { finish(status, outcome, diagnostic) return } state.key = step.Target } finish("failed", "error", "total stage budget exceeded") } // exportStageSession writes the finished stage's session into its own artifacts // directory and returns the evidence for the step. A non-OpenCode harness (pi) // has no native export path, so it records that limitation and writes no files. // Any export error is recorded as a sanitized diagnostic and never returned to // the stage: the export is read-only, bounded and best-effort. func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport { evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend} if owner.Backend != "v1" && owner.Backend != "v2" { evidence.Source = "unavailable" evidence.Diagnostic = "session export unavailable: harness has no native export path" return evidence } export := r.exportSession if export == nil { export = stage.ExportSession } // A canceled stage still gets one bounded, non-blocking attempt; the export // never consumes the stage timeout and never waits for the session. exportCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stage.SessionExportTimeout) defer cancel() result, err := export(exportCtx, stage.SessionExportRequest{ SessionID: owner.ID, Backend: owner.Backend, Directory: owner.Directory, ArtifactsDir: artifactsDir, RunID: runID, Sequence: sequence, Stage: key, Provider: modelConfig.Provider, Model: modelConfig.Model, }) if err != nil { evidence.Source = "native_export" evidence.Diagnostic = stage.RedactEvidence(err.Error()) return evidence } evidence.Source = result.Source evidence.HTML = result.HTML evidence.HTMLBytes = result.HTMLBytes evidence.Messages = result.Messages evidence.Truncated = result.Truncated evidence.Diagnostic = result.Diagnostic return evidence } // setTitle renames a run: it replaces the stored operator-facing title without // touching the run's input, steps or execution, then returns the updated detail. func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) { explicit, ok := summarise.ExplicitTitle(title) if !ok { return nil, fmt.Errorf("a title with usable text is required") } if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil { return nil, err } return r.getRun(id) }