package server import ( "context" "errors" "fmt" "log" "maps" "os" "path/filepath" "strconv" "strings" "sync" "time" "git.a-wan.net/alan/pipelines/internal/filediff" "git.a-wan.net/alan/pipelines/internal/harness" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" "git.a-wan.net/alan/pipelines/internal/summarise" ) // activeRun records one in-flight run's cancellation seam and the resolved // workspace it is changing. A single map is the only active-run state, so the // workspace cannot drift from the cancel function. type activeRun struct { cancel context.CancelFunc workspace string } type runner struct { ctx context.Context store *pipeline.Store artifactsRoot string mu sync.Mutex active map[int64]activeRun execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error) // title generates the operator-facing run title from the run prompt. When nil // the store derives a fallback from the prompt with no model call, so tests // and unconfigured servers stay deterministic. title func(context.Context, string) summarise.Title // verdict generates the bounded one-line reason for a completed stage visit // from its output map. When nil the store derives the reason at read time with // no model call. It is best-effort: any failure must never fail the stage. verdict func(context.Context, map[string]any) string // events pushes run state deltas to subscribed clients. When nil (tests that // build a runner directly) emits are dropped, so nothing else changes. events *runEventHub wg sync.WaitGroup schedulerWG sync.WaitGroup } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { sessions, err := s.ActiveSessionOwners(ctx) if err != nil { return nil, err } for _, owner := range sessions { stop, cancel := context.WithTimeout(ctx, 20*time.Second) var err error if owner.Backend == "pi" { err = stage.InterruptPi(stop, owner.Reference) } else { err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID) } cancel() if err != nil { return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } } if err := s.RecoverRuns(ctx); err != nil { return nil, err } root := filepath.Join(s.Directory(), "artifacts") return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}}, nil } func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) { if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 { return nil, fmt.Errorf("an explicit version and request key are required") } workspace, err := os.Getwd() if err != nil { return nil, err } workspace, err = filepath.EvalSymlinks(workspace) if err != nil { return nil, err } if request.Workspace != "" { if !filepath.IsAbs(request.Workspace) { return nil, fmt.Errorf("run workspace must be an absolute directory") } selected, err := filepath.EvalSymlinks(request.Workspace) if err != nil { return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err) } info, err := os.Stat(selected) if err != nil || !info.IsDir() { return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace) } workspace = selected } if len(request.WritePaths) > 20 { return nil, fmt.Errorf("too many write paths") } for _, path := range request.WritePaths { if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") { return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths") } parent := filepath.Join(workspace, path) for { _, err := os.Lstat(parent) if err == nil { break } if !os.IsNotExist(err) { return nil, err } next := filepath.Dir(parent) if next == parent { return nil, fmt.Errorf("write path is unavailable") } parent = next } resolved, err := filepath.EvalSymlinks(parent) if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) { return nil, fmt.Errorf("write path must stay within the run directory") } } request.Input = maps.Clone(request.Input) if request.Input == nil { request.Input = map[string]any{} } request.Input["pwd"] = workspace if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" { return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt") } d, err := r.store.Get(r.ctx, request.PipelineID, request.Version) if err != nil { return nil, err } if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil { return nil, err } // Validate the launch-time model overrides against the pinned graph before // admission: an unknown stage key or a malformed provider/model must never // insert a run row. The resolution is O(#overrides + #stages) with no model or // network call, and the same helper is the authoritative precedence rule. if _, err := pipeline.EffectiveModels(d.Graph, request.Model, request.StageModels); err != nil { return nil, err } for _, s := range d.Graph.Stages { if stage.EffectiveType(s.Type) != stage.ModelType { continue } if s.Configs["harness"] == "pi" && r.execute == nil { configured, err := harness.Load() if err != nil { return nil, err } if _, ok := configured.Profiles["pi"]; !ok { return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there") } } tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s)) if err != nil { return nil, err } if !explicit { request.AllowTools = true } for _, tool := range tools { if tool == "edit" || tool == "shell" { request.AllowTools = true } } } // Decide admission and idempotency under the lock, but generate the title // only for a request that will actually start a run: a rejected or replayed // request must not pay for a bounded model generation, and a live one must // not hold the lock (stalling listings and cancellation) while a model is // called. An unavailable or slow model falls back to the prompt's first line. r.mu.Lock() if r.ctx.Err() != nil { r.mu.Unlock() return nil, r.ctx.Err() } owners, err := r.store.ActiveSessionOwners(r.ctx) if err != nil { r.mu.Unlock() return nil, err } // Refuse only an orphaned session: one whose owning run is not active now. // A session owned by a currently active run (another run in another // workspace) must not block new work. for _, owner := range owners { if _, ok := r.active[owner.RunID]; !ok { r.mu.Unlock() return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") } } // Idempotent repeats are accepted even while the original run is active, and // a replay reuses the stored title instead of generating a new one. existing, err := r.store.RunByKey(r.ctx, request.RequestKey) if err != nil { r.mu.Unlock() return nil, err } if existing == 0 { if err := r.checkWorkspaceFree(workspace); err != nil { r.mu.Unlock() return nil, err } } r.mu.Unlock() var runTitle summarise.Title if existing == 0 { // An explicit operator title skips the model call; otherwise the // configured generator names the run, falling back to the prompt's // first line when it is unavailable or slow. if explicit, ok := summarise.ExplicitTitle(request.Title); ok { runTitle = explicit } else if r.title != nil { prompt, _ := request.Input["prompt"].(string) runTitle = r.title(r.ctx, prompt) } } r.mu.Lock() defer r.mu.Unlock() // Another run may have reserved this workspace while the title was // generated, so re-check before reserving it; a race on the request key is // handled by the store's idempotent insert. if existing == 0 { if err := r.checkWorkspaceFree(workspace); err != nil { return nil, err } } id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Model: request.Model, StageModels: request.StageModels}, runTitle, retryOf...) if err != nil { return nil, err } if created { // Emit the created delta before launching: the queued frame reaches // subscribers before the running/step-started transition, so a client // sees the run appear and then advance. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusQueued, CreatedAt: time.Now().UTC().Format(time.RFC3339), }, false) r.launch(id, workspace, func(ctx context.Context) { r.run(ctx, id, d, workspace, request) }) } detail, err := r.store.GetRun(r.ctx, id) if detail != nil { _, detail.Executing = r.active[id] r.overlayEffectiveModels(detail) } return detail, err } // launch registers an active run's cancellation seam and workspace, adds it to // the wait group and starts fn on a derived cancellable context that cleans up // the registration when it returns. A fresh run and a resumed decision both go // through it, so registration and teardown live once. Callers must hold r.mu. func (r *runner) launch(id int64, workspace string, fn func(context.Context)) { ctx, cancel := context.WithCancel(r.ctx) r.active[id] = activeRun{cancel: cancel, workspace: workspace} r.wg.Add(1) go func() { defer r.wg.Done() defer cancel() defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }() fn(ctx) }() } // checkWorkspaceFree refuses a run when an active run is already changing the // same resolved workspace. The workspace is the only scope a run reserves, and // one active run per workspace keeps this scan finding at most one match, so the // message is deterministic. Callers must hold r.mu. func (r *runner) checkWorkspaceFree(workspace string) error { for id, entry := range r.active { if entry.workspace == workspace { return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace) } } return nil } func (r *runner) getRun(id int64) (*protocol.RunDetail, error) { r.mu.Lock() defer r.mu.Unlock() d, err := r.store.GetRun(r.ctx, id) if d != nil { _, d.Executing = r.active[id] for _, s := range d.Steps { if s.AgentActive { d.Executing = true } } r.overlayEffectiveModels(d) } return d, err } // overlayEffectiveModels replaces each model stage's provider/model in a returned // run's graph with the effective pair the run actually uses, so the app and TUI // stage-definition surfaces show the override with no client change. It is // presentation only: the persisted graph_json and the pinned source returned by // `show` are never rewritten. A run with no overrides is left untouched. func (r *runner) overlayEffectiveModels(d *protocol.RunDetail) { if d == nil || (d.Model == "" && len(d.StageModels) == 0) { return } effective, err := pipeline.EffectiveModels(d.Graph, d.Model, d.StageModels) if err != nil { return } for i := range d.Graph.Stages { s := &d.Graph.Stages[i] if stage.EffectiveType(s.Type) != stage.ModelType { continue } override, ok := effective[s.Key] if !ok { continue } if s.Configs == nil { s.Configs = map[string]string{} } s.Configs["provider"] = override.Provider s.Configs["model"] = override.Model } } func (r *runner) listRuns() ([]protocol.RunSummary, error) { r.mu.Lock() defer r.mu.Unlock() items, err := r.store.ListRuns(r.ctx) for i := range items { _, items[i].Executing = r.active[items[i].ID] if items[i].Status == "failed" && !items[i].Executing { d, err := r.store.GetRun(r.ctx, items[i].ID) if err != nil { return nil, err } for _, step := range d.Steps { if step.AgentActive { items[i].Executing = true } } } } return items, err } func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("retry requires a positive run ID") } original, err := r.getRun(request.RunID) if err != nil { return nil, err } if original.Executing { return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID) } if original.Status != "failed" || original.FinishedAt == "" { return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status) } return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID) } func (r *runner) cancel(id int64) error { r.mu.Lock() entry, ok := r.active[id] r.mu.Unlock() if ok { entry.cancel() return nil } // A run that is waiting on a person has no live goroutine, so cancel it at // the persistence layer instead. This keeps cancel working across a restart. // The single follow-up read seeds the terminal delta; cancel is a cold // operator action, not a per-transition hot path. if err := r.store.CancelWaiting(r.ctx, id); err != nil { return err } if r.events != nil { if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil { r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: detail.Workspace, Status: detail.Status, Outcome: detail.Outcome, CurrentStage: detail.CurrentStage, CompletedSteps: detail.CompletedSteps, FinishedAt: detail.FinishedAt, }, true) } } return nil } // decide records an operator's choice for a waiting human stage and resumes the // run along the chosen edge. ResolveDecision owns validation and idempotency, so // decide does no option parsing of its own. A resumed run is registered and // launched exactly like a fresh one but starts from the persisted state. func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("decide requires a positive run ID") } r.mu.Lock() detail, err := r.store.GetRun(r.ctx, request.RunID) if err != nil { r.mu.Unlock() return nil, err } if _, busy := r.active[detail.ID]; busy { r.mu.Unlock() return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID) } if detail.Status == protocol.RunStatusWaiting { if err := r.checkWorkspaceFree(detail.Workspace); err != nil { r.mu.Unlock() return nil, err } } step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback) if err != nil { r.mu.Unlock() return nil, err } if applied { for i := range detail.Steps { if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage { detail.Steps[i] = step break } } // The decision closes the waiting step and returns the run to running; // report that before the resumed loop emits its own deltas. r.emitRun(protocol.RunEventFrame{ RunID: detail.ID, Workspace: detail.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{ Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, }, }, false) r.launch(detail.ID, detail.Workspace, func(ctx context.Context) { r.resume(ctx, detail.ID, detail, detail.Workspace, step) }) } r.mu.Unlock() return r.getRun(request.RunID) } func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) { original, copyErr := copyValues(stage.Values(request.Input)) if copyErr != nil { r.finishRunState(id, workspace, "failed", "error", copyErr.Error(), 0, "", stage.Values(request.Input)) return } input := stage.Values(request.Input) state := newRunState(d.Graph.Stages[0].Key, input) state.memory["input"] = memoryValue{output: original} r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state) } // terminalFinish classifies a stage's resolved target. A terminal key yields the // run's final status, outcome and diagnostic; an empty target means the stage // named an undeclared outcome; any other value is a stage key and is not // terminal. resume and the loop tail share it, so terminal routing lives once. func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) { switch target { case protocol.SuccessEnd: return "succeeded", "success", "", true case protocol.FailureEnd: if diagnostic == "" { diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome) } return "failed", "error", diagnostic, true case "": return "failed", "error", "stage returned an undeclared outcome", true } return "", "", "", false } // resume continues a decided run from the state already persisted for its run // detail: memory from the run input and the completed steps, visits counted per // stage, and the next sequence above the highest recorded one. It then runs the // shared loop from the decision's target; no full-table scan and no extra // per-step query are introduced. func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) { // The decision may route straight to a terminal, which has no stage to run. if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal { r.finishRunState(id, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Stage, stage.Values(step.Output)) return } state := rebuildRunState(d, step) r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools, Model: d.Model, StageModels: d.StageModels}, state) } // runState is the loop's whole mutable state, extracted so a fresh run and a // resumed run share one implementation. nextSequence is set before each visit so // a wait that returns to the caller records where the run will continue. type runState struct { key string nextSequence int visits map[string]int memory map[string]memoryValue input stage.Values // changedPaths retains a bounded sample of the run's recorded changed paths // and changedCount the full total, so the skip guard reads only in-memory // evidence. uiSurface is latched once any recorded change touched a UI // surface. Both are folded in by recordChanges (ui_surface.go). changedPaths []string changedCount int uiSurface bool // capturedPaths is the full, deduplicated, first-seen-ordered set of the // run's server-captured changed paths, distinct from the bounded // changedPaths sample used only by the skip reason. capturedSeen is the // single dedupe set. Both are folded in by recordChanges (ui_surface.go) and // published to a selecting stage as input.server_changed_files. capturedPaths []string capturedSeen map[string]bool // completed counts the run's finished steps, so a delta can report the // running total without re-reading the store. completed int } func newRunState(key string, input stage.Values) *runState { return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input} } func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState { state := newRunState(decided.Target, stage.Values(decided.Output)) if original, err := copyValues(stage.Values(detail.Input)); err == nil { state.memory["input"] = memoryValue{output: original} } for _, step := range detail.Steps { state.visits[step.Stage]++ if step.Sequence >= state.nextSequence { state.nextSequence = step.Sequence + 1 } if step.Status == "completed" { if values, err := copyValues(stage.Values(step.Output)); err == nil { state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } state.completed++ } // Seed the UI-surface accumulator from the persisted evidence so a // resumed run reaches the same skip decision with no extra query. state.recordChanges(step.FileChanges, step.Artifacts) } return state } // serverChangedFilesField is the reserved run-input memory field under which the // runner publishes the server-captured changed paths. A stage selects it with // `"server_changed_files":"input.server_changed_files"`, alongside another // stage's declared list, so it sees the verified evidence, not just a model's // claim of authorship. const serverChangedFilesField = "server_changed_files" // capturedList returns the full, first-seen-ordered set of server-captured // changed paths, or a non-nil empty list when nothing was captured so the field // still resolves as a list. It copies the accumulator so the retained evidence // cannot be aliased by a published value. O(captured paths). func (s *runState) capturedList() []string { if len(s.capturedPaths) == 0 { return []string{} } return append([]string(nil), s.capturedPaths...) } // publishServerChangedFiles writes the run's captured changed-path set into the // run input memory value under serverChangedFilesField, creating the entry and // its output map when absent. Called before memoryInput so a stage selecting // input.server_changed_files always resolves; only that field is added and only // a selecting stage copies it out, so state.input and other stages' input are // unchanged. O(captured paths). func (s *runState) publishServerChangedFiles() { entry, ok := s.memory["input"] if !ok || entry.output == nil { entry = memoryValue{output: stage.Values{}} } entry.output[serverChangedFilesField] = s.capturedList() s.memory["input"] = entry } // emitRun pushes one run delta to the subscribed-client hub when one is wired, // and does nothing otherwise (a runner built directly in a test has no hub). // terminal marks a final transition, which the hub flushes without coalescing. func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) { if r.events == nil { return } r.events.emit(frame, terminal) } func (r *runner) finishRunState(id int64, workspace, status, outcome, diagnostic string, completed int, currentStage string, input stage.Values) { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil { log.Printf("finish run %d: %v", id, err) } r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: status, Outcome: outcome, CurrentStage: currentStage, CompletedSteps: completed, FinishedAt: time.Now().UTC().Format(time.RFC3339), }, true) } // completedSteps counts the completed steps of a run detail; resume uses it to // seed the terminal delta without another store read. func completedSteps(steps []protocol.RunStep) int { n := 0 for _, step := range steps { if step.Status == "completed" { n++ } } return n } // finishStage persists one step, refreshes the shared memory for a completed // step, notifies and classifies the step's target. Both the executed path and // the skipped path use it, so persistence, notification and terminal routing // cannot drift between them. A persist or memory-copy failure is returned for // the caller to fail the run; the caller still owns the model-specific // verdict, quiescence and cancellation handling around it. func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) { save, cancel := context.WithTimeout(context.Background(), 10*time.Second) persistErr := r.store.FinishStep(save, id, step) cancel() if persistErr != nil { return "", "", "", false, persistErr } if step.Status == "completed" { values, copyErr := copyValues(stage.Values(step.Output)) if copyErr != nil { return "", "", "", false, copyErr } state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } r.notifyStep(id, name, workspace, selected.Notify, step) state.input = stage.Values(step.Output) status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic) return status, outcome, diagnostic, terminal, nil } func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) { finish := func(status, outcome, diagnostic string) { r.finishRunState(id, workspace, status, outcome, diagnostic, state.completed, state.key, state.input) } defer func() { if recovered := recover(); recovered != nil { finish("failed", "error", "stage execution panicked") } }() budgets := map[string]*stage.Budget{} if state.key == "" { finish("failed", "error", "stage returned an undeclared outcome") return } // Resolve the effective provider/model for every model stage once, before the // loop, so applying an override is an in-memory O(1) map lookup per visit and // never re-parses the graph or re-resolves the whole definition. A run with no // overrides skips the resolution entirely. Launch validation already ran this // helper, so a failure here only guards a resumed run whose overrides drifted. var effectiveModels map[string]pipeline.ModelOverride if request.Model != "" || len(request.StageModels) > 0 { resolved, err := pipeline.EffectiveModels(graph, request.Model, request.StageModels) if err != nil { finish("failed", "error", err.Error()) return } effectiveModels = resolved } for sequence := state.nextSequence; sequence <= 30; sequence++ { state.nextSequence = sequence + 1 if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } key := state.key var def stage.Definition var selected protocol.Stage found := false for _, s := range graph.Stages { if s.Key == key { selected = s def = pipeline.StageDefinition(s) found = true break } } if !found { finish("failed", "error", "missing stage") return } // StageDefinition clones Configs, so setting the effective pair on the // clone is safe and the pinned graph is never mutated. Because // ModelStage.Spec reads def.Configs and passes spec.Config into the repair // path, the same effective model governs the stage and its repair. if stage.EffectiveType(def.Type) == stage.ModelType { if override, ok := effectiveModels[key]; ok { def.Configs["provider"] = override.Provider def.Configs["model"] = override.Model } } isHuman := stage.EffectiveType(def.Type) == stage.HumanType state.visits[key]++ maxVisits := 5 if selected.Limits != nil { maxVisits = selected.Limits.MaxVisits } // A human stage pauses for a person; neither the global visit budget nor // the per-stage max_visits may kill it. The shared 30-step run budget // still bounds every run. if !isHuman { if state.visits[key] > 5 { finish("failed", "error", "stage visit budget exceeded") return } if state.visits[key] > maxVisits { finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits)) return } } // Publish the run's server-captured changed paths into the input memory // value before resolving bindings, so a stage selecting // input.server_changed_files always resolves (an empty list when nothing // was captured). Only a selecting stage copies it into its input. state.publishServerChangedFiles() var reads map[string]protocol.MemoryRead input, reads, copyErr := memoryInput(state.input, selected.Memory, state.memory) if copyErr != nil { finish("failed", "error", copyErr.Error()) return } if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } finish("failed", "error", err.Error()) return } // queued -> running: report the new current stage and the step start. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CurrentStage: key, CompletedSteps: state.completed, Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, }, false) if isHuman { // A human step is O(1): one BeginStep plus one MarkWaiting, no // filediff capture, no artifacts directory and no workspace mutation. evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges)) if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil { if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } finish("failed", "error", err.Error()) return } waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence} // A human step parks the run without a live goroutine; the delta is // coalesced by the hub's timer, which keeps running after this returns. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, CurrentStage: key, CompletedSteps: state.completed, Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting}, }, false) r.notifyStep(id, name, workspace, selected.Notify, waitStep) return } // A stage that declares a `skipped` route is short-circuited when the run // has recorded no UI surface: the stage would answer exactly that, so skip // the model call and its capture work, record the step as completed and // route along the declared edge. Only a declared route is honoured, and no // other stage or outcome routing changes. if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface { reason := skipReasonLine(state) step := protocol.RunStep{ Sequence: sequence, Stage: key, Status: "completed", Outcome: "skipped", Output: stage.Values{"summary": reason}, Target: def.Edges["skipped"], Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}, } status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step) if finishErr != nil { finish("failed", "error", finishErr.Error()) return } if terminal { finish(status, outcome, diagnostic) return } state.key = step.Target continue } var tools []string explicitTools := false var toolErr error if stage.EffectiveType(def.Type) == stage.ModelType { tools, explicitTools, toolErr = stage.ToolNames(def) } if toolErr != nil { finish("failed", "error", toolErr.Error()) return } useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools) // Every agent visit gets a per-run/step directory for the files it // produces. Creation is best-effort: an unusable directory must never // fail the run, and it is only attempted for model stages. artifactsDir := "" if stage.EffectiveType(def.Type) == stage.ModelType { candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence)) if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil { log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr) } else { artifactsDir = candidate def.ArtifactsDir = candidate } } operations := &stage.OperationLog{} client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools} client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference) } var result stage.Result var piClient *stage.PiClient var err error stageCtx := ctx var modelConfig stage.ModelConfig if stage.EffectiveType(def.Type) == stage.ModelType { spec, specErr := (stage.ModelStage{}).Spec(def) if specErr != nil { finish("failed", "error", specErr.Error()) return } modelConfig = spec.Config if budgets[key] == nil { budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens) } stageCtx = stage.WithBudget(ctx, budgets[key]) } snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second) before := filediff.Capture(snapshotCtx, workspace) snapshotCancel() var stageSession string panicked := false func() { defer func() { if recover() != nil { panicked = true err = fmt.Errorf("stage execution panicked") } }() if r.execute != nil { timeout, _ := stage.StageTimeout(def) executeCtx, cancel := context.WithTimeout(stageCtx, timeout) defer cancel() result, err = r.execute(executeCtx, def, input) } else if modelConfig.Harness == "pi" { configured, configErr := harness.Load() if configErr != nil { err = configErr return } profile, ok := configured.Profiles["pi"] if !ok { err = fmt.Errorf("Pi is not configured on execution server") return } piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir} piClient.OnOwnership = func(session, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, "pi", reference) } result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input) stageSession = piClient.SessionID client.Transcript = piClient.Transcript } else if useAgent { result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input) stageSession = client.SessionID } else if stage.EffectiveType(def.Type) == stage.ModelType { plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir} plain.OnGenerationSession = func(session, backend, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if active { stageSession = session } return r.store.RunSession(c, id, sequence, session, active, backend, reference) } result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input) client.Transcript = plain.FormatAudit } else { result, err = stage.DefaultRegistry().Run(ctx, def, input) } }() // Capture after cleanup, even when the stage's context was canceled. snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) after := filediff.Capture(snapshotCtx, workspace, before) snapshotCancel() changes := filediff.Compare(before, after) if errors.Is(err, stage.ErrStageCleanup) { changes.Status = "partial" changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing." } artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir) step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated} // Stamp the effective provider/model this visit actually ran with for every // model stage, including failures, so the app/TUI evidence reflects any // override. Non-model stages keep empty values. if stage.EffectiveType(def.Type) == stage.ModelType { step.Provider = modelConfig.Provider step.Model = modelConfig.Model } if budget := budgets[key]; budget != nil { step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries) } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} if len(result.Output) > 0 { step.Output["salvage"] = result.Output } } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. if err != nil && stageSession != "" { step.Transcript = stage.RetainSession(step.Transcript, stageSession) } step.Target = def.Edges[step.Outcome] step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads} if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits { for _, outcome := range selected.Limits.On { if outcome == step.Outcome { step.Control.LimitReached = true step.Control.ModelTarget = step.Target step.Target = protocol.FailureEnd step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome) } } } if panicked { step.Target = protocol.FailureEnd } // Generate the one-line verdict reason once, at completion. It is // best-effort: a summariser failure or timeout returns NoSummary and must // never fail the stage or the run, so every error is swallowed. A failed // step still gets a verdict from its error envelope. if r.verdict != nil { reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout) if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary { step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel} } reasonCancel() } status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step) if finishErr != nil { finish("failed", "error", finishErr.Error()) return } // Advance the in-memory UI-surface evidence only after the step is // persisted, so the skip guard agrees with what the store holds. state.recordChanges(step.FileChanges, step.Artifacts) if step.Status == "completed" { state.completed++ } // Step finished: report its outcome, routing target, diagnostic and the // artifacts recorded for the visit, plus the run's running step count. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CurrentStage: key, CompletedSteps: state.completed, Artifacts: len(step.Artifacts), Step: &protocol.RunStepDelta{ Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts), }, }, false) if useAgent || errors.Is(err, stage.ErrStageCleanup) { sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id) if checkErr != nil || len(sessions) > 0 { finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work") return } } if ctx.Err() != nil { finish("canceled", "", ctx.Err().Error()) return } if terminal { finish(status, outcome, diagnostic) return } state.key = step.Target } finish("failed", "error", "total stage budget exceeded") } // setTitle renames a run: it replaces the stored operator-facing title without // touching the run's input, steps or execution, then returns the updated detail. func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) { explicit, ok := summarise.ExplicitTitle(title) if !ok { return nil, fmt.Errorf("a title with usable text is required") } if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil { return nil, err } return r.getRun(id) }