package server import ( "context" "encoding/json" "errors" "fmt" "log" "maps" "os" "path/filepath" "strconv" "strings" "sync" "time" "git.a-wan.net/alan/pipelines/internal/filediff" "git.a-wan.net/alan/pipelines/internal/harness" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" "git.a-wan.net/alan/pipelines/internal/summarise" "git.a-wan.net/alan/pipelines/internal/turn" "git.a-wan.net/alan/pipelines/internal/workenv" ) // activeRun records an in-flight or cleanup-unconfirmed run's cancellation seam // and resolved workspace reservation. A single map is the active-run state, so the // workspace cannot drift from the cancel function. type activeRun struct { cancel context.CancelFunc workspace string generation *byte } type runner struct { ctx context.Context store *pipeline.Store artifactsRoot string mu sync.Mutex active map[int64]activeRun execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error) // dirEnv resolves the run workspace's own environment (direnv/mise) at // run start; nil uses workenv.Default. dirEnv *workenv.Resolver exportRepairMu sync.Mutex // title generates the operator-facing run title from the run prompt. When nil // the store derives a fallback from the prompt with no model call, so tests // and unconfigured servers stay deterministic. title func(context.Context, string) summarise.Title // verdict generates the bounded one-line reason for a completed stage visit // from its output map. When nil the store derives the reason at read time with // no model call. It is best-effort: any failure must never fail the stage. verdict func(context.Context, map[string]any) string // events pushes run state deltas to subscribed clients. When nil (tests that // build a runner directly) emits are dropped, so nothing else changes. events *runEventHub // notes records and streams app notifications beside every ntfy // publication. Nil (a runner built directly in a test) records nothing. notes *appNotifier // identMu guards idents: the pipeline identity and stored title (see // runIdentity) that every emitted frame of a run carries, so a client with no // cached row renders a brand-new run without re-reading the list. Populated // where a run is created, decided or canceled; a run with no remembered // identity keeps blank identity fields and the client resyncs once instead. identMu sync.Mutex idents map[int64]runIdentity // exportSession writes a finished stage's session into its own artifacts // directory. It defaults to stage.ExportSession; a failure is best-effort and // never changes the stage's status, outcome, target or result validation. exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) // verifyTarget is installed only by server wiring for a controlled fixture // workflow. Leaving it nil fails closed for target-aware definitions. verifyTarget stage.VerificationRunner wg sync.WaitGroup schedulerWG sync.WaitGroup // attentionMu guards attentionSent, the per-run set of run-attention events // already delivered — by a stage's own notify policy or by the run-level // default — so neither republishes the same genuine event. attentionMu sync.Mutex attentionSent map[int64]map[string]bool // notifyMu guards notifyModes: the per-run auto/on/off override recorded at // admission. It is separate from the stage's own notify policy. notifyMu sync.Mutex notifyModes map[int64]string // sizeKick wakes the background run-size backfill (sizeLoop); nil when the // loop is not running (a runner built directly in a test). sizeKick chan struct{} // detach leaves harness sessions running when the server stops; reattach // holds the runs start-up recovery kept for reattachRuns to continue. detach bool reattach []reattachPlan } func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { return newRunnerAt(ctx, s, "") } // newRunnerAt builds a runner whose stage artifacts live under artifactsRoot // ("" means /artifacts). artifactsRoot is fixed before any // background goroutine (session export jobs, notifications, the size backfill) // starts and is never written afterwards, so those goroutines read it without // locking. func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) { return newRunnerReattaching(ctx, s, artifactsRoot, false) } // newRunnerReattaching builds the runner. With detach set, a server shutdown // leaves stage sessions running (see stage.ErrServerShutdown) and start-up // recovery re-adopts the runs a previous process left in flight instead of // interrupting them; reattachRuns then continues them. func newRunnerReattaching(ctx context.Context, s *pipeline.Store, artifactsRoot string, detach bool) (*runner, error) { if err := s.AcquireRunnerLock(); err != nil { return nil, err } // Store.Open is also used by offline tools, whose cwd is not execution // authority. Bind imported legacy defaults only here, before serving requests // or starting scheduler workers, and never rebind an explicit saved folder. workspace, err := pipeline.ResolveRunWorkspace("", false) if err != nil { return nil, err } if err = s.BindLegacyScheduleWorkspace(ctx, workspace); err != nil { return nil, err } var plans []reattachPlan keep, reasons, gone := map[int64]bool{}, map[int64]string{}, map[int64]bool{} if detach { if plans, reasons, gone, err = planReattach(ctx, s); err != nil { return nil, err } for _, plan := range plans { keep[plan.runID] = true } } sessions, err := s.ActiveSessionOwners(ctx) if err != nil { return nil, err } for _, owner := range sessions { if keep[owner.RunID] { continue // Re-adopted below: never interrupt work that survived. } var err error if !gone[owner.RunID] { stop, cancel := context.WithTimeout(ctx, 20*time.Second) err = interruptRecoveredSession(stop, owner) cancel() } if err != nil { return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err) } if err := s.ConfirmRecoveredSession(ctx, owner); err != nil { return nil, err } } if err := s.RecoverRunsKeeping(ctx, keep, reasons); err != nil { return nil, err } root := artifactsRoot if root == "" { root = filepath.Join(s.Directory(), "artifacts") } active := map[int64]activeRun{} rows, err := s.ListRuns(ctx) if err != nil { return nil, err } for _, row := range rows { if row.CleanupUnconfirmed { active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace} } } r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession, detach: detach, reattach: plans} if err := r.startSessionExportJobs(); err != nil { return nil, err } r.schedulerWG.Add(1) go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }() return r, nil } func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) { return r.startWithSchedule(request, nil, retryOf...) } func (r *runner) startWithSchedule(request protocol.Message, origin *pipeline.ScheduleAdmission, retryOf ...int64) (*protocol.RunDetail, error) { if err := protocol.ValidateOutputFormat(request.OutputFormat); err != nil { return nil, err } if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 { return nil, fmt.Errorf("an explicit version and request key are required") } workspace, err := pipeline.ResolveRunWorkspace(request.Workspace, origin != nil) if err != nil { return nil, err } if err = pipeline.ValidateRunWritePaths(workspace, request.WritePaths); err != nil { return nil, err } request.Input = maps.Clone(request.Input) if request.Input == nil { request.Input = map[string]any{} } request.Input["pwd"] = workspace if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" { return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt") } d, err := r.store.Get(r.ctx, request.PipelineID, request.Version) if err != nil { return nil, err } if d.ArchivedAt != "" { return nil, &pipeline.ArchivedError{Name: d.Name} } if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil { return nil, err } // Validate the launch-time model overrides against the pinned graph before // admission: an unknown stage key or a malformed provider/model must never // insert a run row. The resolution is O(#overrides + #stages) with no model or // network call, and the same helper is the authoritative precedence rule. if _, err := pipeline.EffectiveModels(d.Graph, request.Model, request.StageModels); err != nil { return nil, err } for _, s := range d.Graph.Stages { if stage.EffectiveType(s.Type) != stage.ModelType { continue } if s.Configs["harness"] == "pi" && r.execute == nil { configured, err := harness.Load() if err != nil { return nil, err } if _, ok := configured.Profiles["pi"]; !ok { return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there") } } tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s)) if err != nil { return nil, err } if !explicit { request.AllowTools = true } for _, tool := range tools { if tool == "edit" || tool == "shell" { request.AllowTools = true } } } // Decide admission and idempotency under the lock, but generate the title // only for a request that will actually start a run: a rejected or replayed // request must not pay for a bounded model generation, and a live one must // not hold the lock (stalling listings and cancellation) while a model is // called. An unavailable or slow model falls back to the prompt's first line. r.mu.Lock() if r.ctx.Err() != nil { r.mu.Unlock() return nil, r.ctx.Err() } owners, err := r.store.ActiveSessionOwners(r.ctx) if err != nil { r.mu.Unlock() return nil, err } // Refuse only an orphaned session: one whose owning run is not active now. // A session owned by a currently active run (another run in another // workspace) must not block new work. for _, owner := range owners { if _, ok := r.active[owner.RunID]; !ok { r.mu.Unlock() return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work") } } // Idempotent repeats are accepted even while the original run is active, and // a replay reuses the stored title instead of generating a new one. existing, err := r.store.RunByKey(r.ctx, request.RequestKey) if err != nil { r.mu.Unlock() return nil, err } if existing == 0 { if err := r.checkWorkspaceFree(workspace); err != nil { r.mu.Unlock() return nil, err } } r.mu.Unlock() var runTitle summarise.Title if existing == 0 { // An explicit operator title skips the model call; otherwise the // configured generator names the run, falling back to the prompt's // first line when it is unavailable or slow. if explicit, ok := summarise.ExplicitTitle(request.Title); ok { runTitle = explicit } else if r.title != nil { prompt, _ := request.Input["prompt"].(string) runTitle = r.title(r.ctx, prompt) } } r.mu.Lock() defer r.mu.Unlock() // Another run may have reserved this workspace while the title was // generated, so re-check before reserving it; a race on the request key is // handled by the store's idempotent insert. if existing == 0 { if err := r.checkWorkspaceFree(workspace); err != nil { return nil, err } } // Storage maintenance may have removed a clean inactive tree while an // admission was generating its title. Refuse that vanished workspace under // the same lock used by removal instead of launching into a missing tree. if info, err := os.Stat(workspace); err != nil || !info.IsDir() { return nil, fmt.Errorf("run workspace is no longer available: %s", workspace) } // A fresh run started from a repository's main checkout gets its own // worktree and branch, so runs never share a tree with each other or with // the operator's uncommitted work. A retry keeps its run's tree. isolated := "" if existing == 0 && request.RetrySource == 0 { isolateCtx, isolateCancel := context.WithTimeout(r.ctx, time.Minute) tree, isolateErr := isolateRunWorkspace(isolateCtx, workspace, runTitle.Text) isolateCancel() if isolateErr != nil { return nil, isolateErr } if tree != workspace { isolated = tree } workspace = tree request.Input["pwd"] = workspace } id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Schedule: origin, Model: request.Model, StageModels: request.StageModels, NotifyMode: request.NotifyMode, OutputFormat: request.OutputFormat}, runTitle, retryOf...) if isolated != "" && (err != nil || !created) { // The run was refused or an identical request won the race: the fresh // tree holds nothing yet. discardRunWorktree(isolated) } if err != nil { return nil, err } r.setNotifyMode(id, request.NotifyMode) // The run row now exists: read it once for its resolved identity (pipeline // id/name, the store-derived title, the pinned version), so the queued frame // and every later frame carry what a client needs to render a brand-new run // without re-reading the list. A failed read only costs that seed: the run is // still launched below and its frames carry no identity, so the client falls // back to its one resync. detail, readErr := r.store.GetRun(r.ctx, id) if readErr != nil { detail = nil } else if detail != nil { r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) } if created { if request.RetrySource > 0 && request.RetryMode != "restart" { checkpoint, checkpointErr := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput) if checkpointErr != nil { return nil, checkpointErr } if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil { return nil, err } } // Emit the created delta before launching: the queued frame reaches // subscribers before the running/step-started transition, so a client // sees the run appear and then advance. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusQueued, CreatedAt: time.Now().UTC().Format(time.RFC3339), }, false) r.launch(id, workspace, func(ctx context.Context) { r.run(ctx, id, d, workspace, request) }) } if detail != nil { _, detail.Executing = r.active[id] if detail.ExportPendingSequence > 0 { detail.Executing = false } r.overlayEffectiveModels(detail) } if readErr != nil { return nil, readErr } return detail, nil } // launch registers an active run's cancellation seam and workspace, adds it to // the wait group and starts fn on a derived cancellable context that cleans up // the registration when it returns. A fresh run and a resumed decision both go // through it, so registration and teardown live once. Callers must hold r.mu. func (r *runner) launch(id int64, workspace string, fn func(context.Context)) { ctx, cancel := r.runContext() generation := new(byte) r.active[id] = activeRun{cancel: cancel, workspace: workspace, generation: generation} r.wg.Add(1) go func() { defer r.wg.Done() defer cancel() defer func() { r.mu.Lock() defer r.mu.Unlock() if entry, ok := r.active[id]; !ok || entry.generation != generation { return // A parked launch must not remove the resumed reservation. } check, done := context.WithTimeout(context.Background(), 10*time.Second) defer done() detail, err := r.store.GetRun(check, id) if err != nil || detail.CleanupUnconfirmed { return // Fail closed: execution/cleanup has not settled durably. } if detail.Status == protocol.RunStatusWaiting && detail.CancelRequested { if err := r.store.CancelWaiting(check, id); err != nil { return } r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: detail.Workspace, Status: protocol.RunStatusCanceled, CancelRequested: true, CurrentStage: detail.CurrentStage, FinishedAt: time.Now().UTC().Format(time.RFC3339)}, true) } if detail.FinishedAt != "" || detail.Status == protocol.RunStatusWaiting { delete(r.active, id) } // A landed (or change-free) successful run's own worktree is no // longer needed; anything unlanded or uncommitted is kept. if detail.Status == "succeeded" && detail.FinishedAt != "" { workspace := detail.Workspace go func() { if removeLandedRunWorktree(workspace) { log.Printf("run %d: removed its landed worktree %s", id, workspace) } }() } }() fn(ctx) }() } // checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run // reserves an overlapping resolved workspace. Callers must hold r.mu. func (r *runner) checkWorkspaceFree(workspace string) error { return r.checkWorkspaceFreeExcept(workspace, 0) } func (r *runner) checkWorkspaceFreeExcept(workspace string, ownGate int64) error { for id, entry := range r.active { if workspacesOverlap(entry.workspace, workspace) { return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace) } } check, cancel := context.WithTimeout(context.WithoutCancel(r.ctx), 5*time.Second) defer cancel() reservations, err := r.store.SessionExportReservations(check) if err != nil { return fmt.Errorf("read pending export workspace reservations: %w", err) } for id, reserved := range reservations { if id != ownGate && workspacesOverlap(reserved, workspace) { return fmt.Errorf("run %d is awaiting its session export; workspace %s remains reserved", id, reserved) } } return nil } func workspacesOverlap(a, b string) bool { within := func(parent, child string) bool { rel, err := filepath.Rel(parent, child) return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) } return within(a, b) || within(b, a) } func (r *runner) getRun(id int64) (*protocol.RunDetail, error) { r.mu.Lock() defer r.mu.Unlock() d, err := r.store.GetRun(r.ctx, id) if d != nil { _, d.Executing = r.active[id] if d.ExportPendingSequence > 0 { d.Executing = false } for _, s := range d.Steps { if s.AgentActive { d.Executing = true } } r.overlayEffectiveModels(d) } return d, err } // overlayEffectiveModels replaces each model stage's provider/model in a returned // run's graph with the effective pair the run actually uses, so the app and TUI // stage-definition surfaces show the override with no client change. It is // presentation only: the persisted graph_json and the pinned source returned by // `show` are never rewritten. A run with no overrides is left untouched. func (r *runner) overlayEffectiveModels(d *protocol.RunDetail) { if d == nil || (d.Model == "" && len(d.StageModels) == 0) { return } effective, err := pipeline.EffectiveModels(d.Graph, d.Model, d.StageModels) if err != nil { return } for i := range d.Graph.Stages { s := &d.Graph.Stages[i] if stage.EffectiveType(s.Type) != stage.ModelType { continue } override, ok := effective[s.Key] if !ok { continue } if s.Configs == nil { s.Configs = map[string]string{} } s.Configs["provider"] = override.Provider s.Configs["model"] = override.Model } } func (r *runner) listRuns() ([]protocol.RunSummary, error) { r.mu.Lock() defer r.mu.Unlock() items, err := r.store.ListRuns(r.ctx) for i := range items { _, items[i].Executing = r.active[items[i].ID] if items[i].ExportPendingSequence > 0 { items[i].Executing = false } if items[i].Status == "failed" && !items[i].Executing { d, err := r.store.GetRun(r.ctx, items[i].ID) if err != nil { return nil, err } for _, step := range d.Steps { if step.AgentActive { items[i].Executing = true } } } } return items, err } type retryCheckpoint struct { detail *protocol.RunDetail failed protocol.RunStep handoff map[string]any } type retryPrefixRecord struct { stage string sequence int public []string publicKnown bool } const maxRetryInputBytes = 64 * 1024 func retryInputOverride(base map[string]any, override map[string]any) (map[string]any, error) { if override == nil { return base, nil } encoded, err := json.Marshal(override) if err != nil || len(encoded) > maxRetryInputBytes { return nil, fmt.Errorf("retry input must be a JSON object no larger than %d bytes", maxRetryInputBytes) } reserved := map[string]bool{"pwd": true, "workspace": true, "write_paths": true, "allow_tools": true, "pipeline_id": true, "pipeline_version": true, "request_key": true, "retry_mode": true, "retry_source": true, "retry_stage": true, "retry_visit": true, serverChangedFilesField: true} merged := map[string]any{} for key, value := range base { merged[key] = value } for key, value := range override { if reserved[key] { return nil, fmt.Errorf("retry input field %q is reserved", key) } merged[key] = value } return merged, nil } // retryStageContractEqual compares the complete persisted stage contract. A // retry may carry completed output across a published version only when the // stages that produced that output are unchanged; comparing only effective // types would allow prompt, config, edge, or limit changes through. func retryStageContractEqual(left, right protocol.Stage) bool { a, err := json.Marshal(left) if err != nil { return false } b, err := json.Marshal(right) if err != nil { return false } return string(a) == string(b) } func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) { return r.retrySourceCheckpointSelected(sourceID, pinned, "", 0, nil) } func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.PipelineDetail, selectedStage string, selectedVisit int, override map[string]any) (retryCheckpoint, error) { source, err := r.store.GetRun(r.ctx, sourceID) if err != nil { return retryCheckpoint{}, err } if source.Status != "failed" || source.FinishedAt == "" { return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID) } if source.PipelineID != pinned.ID { return retryCheckpoint{}, fmt.Errorf("retry source %d belongs to a different pipeline", sourceID) } if len(source.Steps) == 0 { return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID) } failed := source.Steps[len(source.Steps)-1] if selectedStage != "" { failed = protocol.RunStep{} visits := 0 for _, candidate := range source.Steps { if candidate.Stage != selectedStage { continue } visits++ if selectedVisit == 0 && visits == 1 { failed = candidate } if selectedVisit > 0 && visits == selectedVisit { failed = candidate } } if failed.Stage == "" { return retryCheckpoint{}, fmt.Errorf("retry stage %q was not recorded in run %d", selectedStage, sourceID) } if selectedVisit == 0 { count := 0 for _, candidate := range source.Steps { if candidate.Stage == selectedStage { count++ } } if count > 1 { return retryCheckpoint{}, fmt.Errorf("retry stage %q was visited %d times; --visit is required", selectedStage, count) } } if selectedVisit > visits { return retryCheckpoint{}, fmt.Errorf("retry stage %q has no visit %d", selectedStage, selectedVisit) } } if (failed.Status != "completed" && failed.Status != "failed") || (selectedStage == "" && failed.Target != protocol.FailureEnd) { return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID) } var handoff map[string]any if failed.Input == nil { return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID) } if _, err := retryInputOverride(failed.Input, override); err != nil { return retryCheckpoint{}, err } // Keep the source-stage input immutable in the checkpoint. The selected // stage's memory bindings must be resolved from the original prefix before // this override is applied at invocation time. handoff = map[string]any{"input": failed.Input, "override": override, "sequence": failed.Sequence, "stage": failed.Stage, "visit": failed.Sequence} prefix, prefixRecords, prefixErr := r.retryPrefix(sourceID, source, failed.Sequence) if prefixErr != nil { return retryCheckpoint{}, prefixErr } handoff["prefix"] = prefix serializedPrefix := make([]map[string]any, 0, len(prefixRecords)) for _, record := range prefixRecords { entry := map[string]any{"stage": record.stage, "sequence": record.sequence} if record.publicKnown { // Persist the exact additions so a later retry can rebuild memory // and carry public/ without reading this attempt's rows. entry["public"] = record.public } serializedPrefix = append(serializedPrefix, entry) } handoff["prefix_provenance"] = serializedPrefix // A newer explicitly selected version may continue only when every recorded // prefix stage still exists with the same kind. Its outputs are evidence from // the source attempt and cannot be safely handed to a changed stage contract. sourceStages := map[string]protocol.Stage{} for _, declared := range source.Graph.Stages { sourceStages[declared.Key] = declared } for _, record := range prefixRecords { _, found := retryPrefixStep(source, record) if !found { continue } var pinnedStage *protocol.Stage for i := range pinned.Graph.Stages { if pinned.Graph.Stages[i].Key == record.stage { pinnedStage = &pinned.Graph.Stages[i] break } } previous, ok := sourceStages[record.stage] if pinnedStage == nil || !ok || !retryStageContractEqual(previous, *pinnedStage) { return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, record.stage) } } var sourceSelected protocol.Stage var sourceSelectedOK bool if sourceSelected, sourceSelectedOK = sourceStages[failed.Stage]; !sourceSelectedOK { return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in its recorded graph", sourceID) } pinnedStages := map[string]protocol.Stage{} for _, declared := range pinned.Graph.Stages { pinnedStages[declared.Key] = declared } // The retry will continue through every stage that the source execution // already recorded after the selected checkpoint. Validate those stages // before admitting the new run as well: otherwise an explicitly selected // version can pass prefix/selected-stage validation and fail later in the // loop with the less useful "missing stage" error. for _, step := range source.Steps { if step.Sequence <= failed.Sequence { continue } if _, ok := pinnedStages[step.Stage]; !ok { return retryCheckpoint{}, fmt.Errorf("retry source %d selected version is missing later stage %q", sourceID, step.Stage) } } for _, declared := range pinned.Graph.Stages { if declared.Key != failed.Stage { continue } if stage.EffectiveType(sourceSelected.Type) != stage.EffectiveType(declared.Type) { return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible selected stage %q", sourceID, failed.Stage) } typ := stage.EffectiveType(declared.Type) if typ == stage.HumanType || typ == stage.NtfyType { return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage) } if typ == stage.ModelType { if _, _, toolErr := stage.ToolNames(pipeline.StageDefinition(declared)); toolErr != nil { return retryCheckpoint{}, toolErr } } return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil } return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID) } func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("retry requires a positive run ID") } original, err := r.getRun(request.RunID) if err != nil { return nil, err } if original.Executing { return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID) } if original.Status != "failed" || original.FinishedAt == "" { return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status) } if request.RequestKey == "" { return nil, fmt.Errorf("retry requires an idempotency request key") } // Resolve an already admitted request before validating optional retry // selectors. This makes replay after a restart genuinely idempotent even // when the original request used an input override that the replay omits. if existing, err := r.store.RunByKey(r.ctx, request.RequestKey); err != nil { return nil, err } else if existing > 0 { return r.store.GetRun(r.ctx, existing) } if blocked, err := r.store.SessionExportBlocked(r.ctx, original.ID); err != nil { return nil, err } else if blocked { return nil, fmt.Errorf("run %d has a retained computation awaiting export-only recovery; model retry refused", original.ID) } if request.RetryVisit < 0 || (request.RetryVisit > 0 && request.RetryStage == "") { return nil, fmt.Errorf("retry visit requires a positive --stage visit selection") } mode := request.RetryMode if mode == "" { mode = "resume" } if mode != "resume" && mode != "restart" { return nil, fmt.Errorf("retry mode must be resume or restart") } version := original.Version if request.RetryVersion > 0 { version = request.RetryVersion } d, err := r.store.Get(r.ctx, original.PipelineID, version) if err != nil { return nil, err } if mode == "resume" { if _, err := r.retrySourceCheckpointSelected(original.ID, d, request.RetryStage, request.RetryVisit, request.RetryInput); err != nil { return nil, err } } notifyMode := request.NotifyMode if notifyMode == "" { notifyMode, _ = r.store.RunNotifyMode(r.ctx, original.ID) } return r.start(protocol.Message{PipelineID: original.PipelineID, Version: version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, RetryStage: request.RetryStage, RetryVisit: request.RetryVisit, RetryInput: request.RetryInput, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels, NotifyMode: notifyMode, OutputFormat: original.OutputFormat}, original.ID) } func (r *runner) cancel(id int64) error { r.mu.Lock() defer r.mu.Unlock() entry, ok := r.active[id] if err := r.store.Cancel(r.ctx, id); err != nil { return err } if ok { entry.cancel() } else { // Only a parked human run is known to have no live executor. if err := r.store.CancelWaiting(r.ctx, id); err != nil { return err } } r.forgetAttention(id) if r.events != nil { if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil { r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary)) r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: detail.Workspace, Status: detail.Status, Outcome: detail.Outcome, CurrentStage: detail.CurrentStage, CompletedSteps: detail.CompletedSteps, FinishedAt: detail.FinishedAt, CancelRequested: detail.CancelRequested, CleanupUnconfirmed: detail.CleanupUnconfirmed, }, !ok) } } return nil } // decide records an operator's choice for a waiting human stage and resumes the // run along the chosen edge. ResolveDecision owns validation and idempotency, so // decide does no option parsing of its own. A resumed run is registered and // launched exactly like a fresh one but starts from the persisted state. func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) { if request.RunID <= 0 { return nil, fmt.Errorf("decide requires a positive run ID") } r.mu.Lock() detail, err := r.store.GetRun(r.ctx, request.RunID) if err != nil { r.mu.Unlock() return nil, err } // finishRunState persists the terminal status under r.mu, but the run stays // in r.active until its goroutine finishes notifying and the launch teardown // runs. A run whose terminal state is already durable (FinishedAt is only set // once cleanup is confirmed) has no executor left to race with a decision, so // ResolveDecision answers it ("already decided" or "not waiting") and never // resumes a terminal run. A run that is still executing is refused. if _, busy := r.active[detail.ID]; busy && detail.FinishedAt == "" { r.mu.Unlock() return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID) } if detail.ExportPendingSequence > 0 { r.mu.Unlock() return nil, fmt.Errorf("run %d is awaiting session export, not a human decision", detail.ID) } if detail.Status == protocol.RunStatusWaiting { if err := r.checkWorkspaceFree(detail.Workspace); err != nil { r.mu.Unlock() return nil, err } } step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback) if err != nil { r.mu.Unlock() return nil, err } if applied { // Re-seed the identity so the resumed frames (this one and the resumed // loop's) carry it even after a restart, when no creation was observed. r.rememberRunIdentity(detail.ID, identityOfRunSummary(detail.RunSummary)) for i := range detail.Steps { if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage { detail.Steps[i] = step break } } // The decision closes the waiting step and returns the run to running; // report that before the resumed loop emits its own deltas. r.emitRun(protocol.RunEventFrame{ RunID: detail.ID, Workspace: detail.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{ Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, }, }, false) r.launch(detail.ID, detail.Workspace, func(ctx context.Context) { r.resume(ctx, detail.ID, detail, detail.Workspace, step) }) } r.mu.Unlock() return r.getRun(request.RunID) } func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) { if request.RetrySource > 0 && request.RetryMode != "restart" { source, err := r.store.GetRun(ctx, request.RetrySource) if err != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) return } checkpoint, err := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput) if err != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) return } storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id) if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil { if checkpointErr == nil { checkpointErr = fmt.Errorf("retry checkpoint is incomplete") } r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil) return } if input, ok := storedHandoff["input"].(map[string]any); ok { checkpoint.failed.Input = input } else { r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil) return } state := rebuildRetryState(source, checkpoint.failed, storedHandoff) if override, ok := storedHandoff["override"].(map[string]any); ok { state.retryOverride = stage.Values(override) } // The retry is a new run with a new artifact root. Carry the completed // prefix's recorded public files before the first visit, so memory does // not name files this attempt cannot read. if err := r.carryRetryPublic(id, request.RetrySource, checkpoint.failed.Sequence, storedHandoff, source); err != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil) return } r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state) return } original, copyErr := copyValues(stage.Values(request.Input)) if copyErr != nil { r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input)) return } input := stage.Values(request.Input) state := newRunState(d.Graph.Stages[0].Key, input) state.memory["input"] = memoryValue{output: original} r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state) } // terminalFinish classifies a stage's resolved target. A terminal key yields the // run's final status, outcome and diagnostic; an empty target means the stage // named an undeclared outcome; any other value is a stage key and is not // terminal. resume and the loop tail share it, so terminal routing lives once. func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) { switch target { case protocol.SuccessEnd: return "succeeded", "success", "", true case protocol.FailureEnd: if diagnostic == "" { diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome) } return "failed", "error", diagnostic, true case "": return "failed", "error", "stage returned an undeclared outcome", true } return "", "", "", false } // resume continues a decided run from the state already persisted for its run // detail: memory from the run input and the completed steps, visits counted per // stage, and the next sequence above the highest recorded one. It then runs the // shared loop from the decision's target; no full-table scan and no extra // per-step query are introduced. func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) { // A resumed run's frames carry the same identity as its creation, so re-seed // it here too: resume is also reachable from a store read after a restart. r.rememberRunIdentity(id, identityOfRunSummary(d.RunSummary)) // The decision may route straight to a terminal, which has no stage to run. if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal { r.finishRunState(id, d.Name, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Sequence, step.Stage, stage.Values(step.Output)) return } state := rebuildRunState(d, step) r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools, Model: d.Model, StageModels: d.StageModels}, state) } // runState is the loop's whole mutable state, extracted so a fresh run and a // resumed run share one implementation. nextSequence is set before each visit so // a wait that returns to the caller records where the run will continue. type runState struct { key string nextSequence int visits map[string]int memory map[string]memoryValue input stage.Values // changedPaths retains a bounded sample of the run's recorded changed paths // and changedCount the full total, so the skip guard reads only in-memory // evidence. uiSurface is latched once any visit captured a rendered (image) // artifact. captureIncomplete counts visits whose file-change capture was // missing, partial or unavailable; any such visit forbids the server skip // fallback. All are folded in by recordChanges (ui_surface.go). changedPaths []string changedCount int uiSurface bool captureIncomplete int // capturedPaths is the full, deduplicated, first-seen-ordered set of the // run's server-captured changed paths, distinct from the bounded // changedPaths sample used only by the skip reason. capturedSeen is the // single dedupe set. Both are folded in by recordChanges (ui_surface.go) and // published to a selecting stage as input.server_changed_files. capturedPaths []string capturedSeen map[string]bool // completed counts the run's finished steps, so a delta can report the // running total without re-reading the store. completed int artifactDirs []string // prompt is the original run prompt; lastOutput is the previous visit's // markdown and turns is every completed turn in order. Together they render // the next model visit's input.md, memory.md and history.md. prompt string lastOutput string turns []turn.MemoryEntry // retryOverride is applied once, after the selected stage's memory bindings // have been resolved. It is intentionally not part of run input or memory. retryOverride stage.Values priorBudgets map[string]stage.BudgetSnapshot // adopt names the visit to re-adopt from a previous server process instead // of starting it; it is consumed by the loop's first iteration. adopt *adoptedVisit } func newRunState(key string, input stage.Values) *runState { prompt, _ := input["prompt"].(string) return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input, prompt: prompt} } func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState { state := newRunState(decided.Target, stage.Values(decided.Output)) if prompt, ok := detail.Input["prompt"].(string); ok { state.prompt = prompt } if original, err := copyValues(stage.Values(detail.Input)); err == nil { state.memory["input"] = memoryValue{output: original} } publicSeen := map[string]bool{} state.priorBudgets = map[string]stage.BudgetSnapshot{} for _, step := range detail.Steps { // The runtime stamps this cumulative snapshot on every executed model // visit. Reusing it preserves per-stage limits across export recovery. var audit struct { Limits struct { Budget stage.BudgetSnapshot `json:"budget"` } `json:"stage_limits"` } if json.Unmarshal([]byte(step.Transcript), &audit) == nil { state.priorBudgets[step.Stage] = audit.Limits.Budget } state.visits[step.Stage]++ if step.Sequence >= state.nextSequence { state.nextSequence = step.Sequence + 1 } added := publicArtifacts(step.Artifacts, publicSeen) if step.Status == "completed" { if values, err := copyValues(stage.Values(step.Output)); err == nil { state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } state.completed++ state.turns = append(state.turns, turn.MemoryEntry{Stage: step.Stage, Visit: state.visits[step.Stage], Outcome: step.Outcome, Output: turnOutput(step.Output), PublicFiles: added}) state.lastOutput = turnOutput(step.Output) } // Seed the UI-surface accumulator from the persisted evidence so a // resumed run reaches the same skip decision with no extra query. state.recordPersistedStep(step) // Only completed, recorded visits are eligible for a later stage's // review capability. The path is reconstructed by the runner below. if step.Status == "completed" { state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) } } return state } // rebuildRetryState restores only the completed prefix. The failed visit is // deliberately not counted and its step/evidence is not copied into the new // attempt, so retries cannot duplicate execution or usage aggregates. func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep, handoff map[string]any) *runState { state := newRunState(failed.Stage, stage.Values(failed.Input)) // Resume at the failed sequence. The retry is a new run with a new run ID // and artifact root, so this sequence's turn directory does not overlap the // source attempt's files. state.nextSequence = failed.Sequence if prompt, ok := detail.Input["prompt"].(string); ok { state.prompt = prompt } if original, err := copyValues(stage.Values(detail.Input)); err == nil { state.memory["input"] = memoryValue{output: original} } publicSeen := map[string]bool{} _, hasCheckpointPrefix := handoff["prefix"] if !hasCheckpointPrefix { for _, step := range detail.Steps { if step.Sequence >= failed.Sequence { break } state.visits[step.Stage]++ added := publicArtifacts(step.Artifacts, publicSeen) if step.Status == "completed" { if values, err := copyValues(stage.Values(step.Output)); err == nil { state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } state.completed++ state.turns = append(state.turns, turn.MemoryEntry{Stage: step.Stage, Visit: state.visits[step.Stage], Outcome: step.Outcome, Output: turnOutput(step.Output), PublicFiles: added}) state.lastOutput = turnOutput(step.Output) } state.recordPersistedStep(step) if step.Status == "completed" { state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence))) } } } // A selected-stage retry has no persisted rows for its inherited prefix. // Restore it from the immutable checkpoint instead of guessing from the // retry source's immediate steps (which is what made retry-of-retry fail // closed with "memory source ... has not completed"). if prefix, ok := handoff["prefix"].(map[string]any); ok { // The checkpoint carries outputs, not file-change captures, so an inherited // prefix is unobserved evidence: the server skip fallback must not apply. state.captureIncomplete++ for _, record := range retryPrefixRecords(handoff["prefix_provenance"]) { output, exists := prefix[record.stage].(map[string]any) if !exists { continue } values, err := copyValues(stage.Values(output)) if err != nil { continue } state.memory[record.stage] = memoryValue{output: values, sequence: record.sequence} state.visits[record.stage]++ state.completed++ state.turns = append(state.turns, turn.MemoryEntry{Stage: record.stage, Visit: state.visits[record.stage], Output: turnOutput(values), PublicFiles: recordPublicFiles(record, detail, publicSeen)}) state.lastOutput = turnOutput(values) state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(record.sequence))) } } return state } // retryPrefix returns the completed prefix from the source attempt plus the // immutable prefix carried by that attempt's checkpoint. A retry attempt only // persists the selected stage onward, so immediate source steps are not a // complete representation of its memory. func (r *runner) retryPrefix(sourceID int64, source *protocol.RunDetail, failedSequence int) (map[string]any, []retryPrefixRecord, error) { prefix := map[string]any{} records := []retryPrefixRecord{} inheritedPublic := map[string]bool{} if source.RetryMode == "resume" && source.RetrySource > 0 { checkpointSource, mode, _, inherited, err := r.store.RetryCheckpoint(r.ctx, sourceID) if err != nil { return nil, nil, fmt.Errorf("retry source %d has no immutable prefix checkpoint: %w", sourceID, err) } if checkpointSource != source.RetrySource || mode != "resume" { return nil, nil, fmt.Errorf("retry source %d has an invalid prefix checkpoint", sourceID) } if values, ok := inherited["prefix"].(map[string]any); ok { for key, value := range values { prefix[key] = value } } for _, record := range retryPrefixRecords(inherited["prefix_provenance"]) { records = append(records, record) for _, name := range record.public { inheritedPublic[name] = true } } } publicBySequence := retryPublicBySequence(source, failedSequence) for _, step := range source.Steps { if step.Sequence >= failedSequence || step.Status != "completed" { continue } prefix[step.Stage] = step.Output added := make([]string, 0, len(publicBySequence[step.Sequence])) for _, name := range publicBySequence[step.Sequence] { // Carried prefix files reappear in this attempt's inventory but were // added by an earlier turn, not this one. if !inheritedPublic[name] { added = append(added, name) } } records = append(records, retryPrefixRecord{stage: step.Stage, sequence: step.Sequence, public: added, publicKnown: true}) } return prefix, records, nil } func retryPrefixRecords(raw any) []retryPrefixRecord { items, ok := raw.([]any) if !ok { return nil } result := make([]retryPrefixRecord, 0, len(items)) for _, item := range items { entry, ok := item.(map[string]any) if !ok { continue } stageName, ok := entry["stage"].(string) if !ok || stageName == "" { continue } sequence, ok := entry["sequence"].(float64) if !ok || sequence < 1 { continue } record := retryPrefixRecord{stage: stageName, sequence: int(sequence)} if raw, ok := entry["public"]; ok { record.public = stringValues(raw) record.publicKnown = true } result = append(result, record) } return result } // stringValues accepts the JSON-decoded forms of a string list in a retry // checkpoint and returns a copy; unknown element types are ignored. func stringValues(raw any) []string { switch values := raw.(type) { case []string: return append([]string(nil), values...) case []any: result := make([]string, 0, len(values)) for _, value := range values { if text, ok := value.(string); ok { result = append(result, text) } } return result default: return nil } } func retryPrefixStep(detail *protocol.RunDetail, record retryPrefixRecord) (protocol.RunStep, bool) { for _, step := range detail.Steps { if step.Stage == record.stage && step.Sequence == record.sequence { return step, true } } // Inherited rows intentionally are not present in a selected retry. The // contract check still uses the stage key; this marker keeps that check // independent from model-claimed output. return protocol.RunStep{Stage: record.stage, Sequence: record.sequence}, true } // artifactAccess derives exact turn directories; it never consumes a path from // a definition or model result. The current visit may read and write its own // turn and the shared public directory. Earlier visits are not readable: their // public files are already shared and their private/ files are never handed to // a later stage. An unreadable or symlinked root fails closed. func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess { _ = prior // prior private directories remain deliberately inaccessible root, err := filepath.EvalSymlinks(r.artifactsRoot) if err != nil { return stage.ArtifactAccess{} } runRoot := filepath.Join(root, strconv.FormatInt(runID, 10)) directory := func(path string) (string, bool) { if path == "" { return "", false } resolved, err := filepath.EvalSymlinks(path) if err != nil || !strings.HasPrefix(resolved, runRoot+string(os.PathSeparator)) { return "", false } info, err := os.Stat(resolved) if err != nil || !info.IsDir() { return "", false } return resolved, true } access := stage.ArtifactAccess{} currentDir, ok := directory(current) if !ok { return access } // The current directory must be this visit's turn, not another stage's turn // in the same run. expectedPrefix := fmt.Sprintf("%02d-", sequence) if filepath.Dir(currentDir) != filepath.Join(runRoot, "stages") || !strings.HasPrefix(filepath.Base(currentDir), expectedPrefix) { return access } if private, ok := directory(filepath.Join(currentDir, "private")); ok { // The model may write its own private/ files, never the server-owned // input.md, memory.md or output.md. A missing private/ grants no write // scope; falling back to the turn directory would widen the permission. access.Write = private } access.Read = append(access.Read, currentDir) if public, ok := directory(filepath.Join(runRoot, "public")); ok { access.WriteShared = public access.Read = append(access.Read, public) } return access } func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient { return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} } func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient { return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)} } // serverChangedFilesField is the reserved run-input memory field under which the // runner publishes the server-captured changed paths. A stage selects it with // `"server_changed_files":"input.server_changed_files"`, alongside another // stage's declared list, so it sees the verified evidence, not just a model's // claim of authorship. const serverChangedFilesField = "server_changed_files" // capturedList returns the full, first-seen-ordered set of server-captured // changed paths, or a non-nil empty list when nothing was captured so the field // still resolves as a list. It copies the accumulator so the retained evidence // cannot be aliased by a published value. O(captured paths). func (s *runState) capturedList() []string { if len(s.capturedPaths) == 0 { return []string{} } return append([]string(nil), s.capturedPaths...) } // publishServerChangedFiles writes the run's captured changed-path set into the // run input memory value under serverChangedFilesField, creating the entry and // its output map when absent. Called before memoryInput so a stage selecting // input.server_changed_files always resolves; only that field is added and only // a selecting stage copies it out, so state.input and other stages' input are // unchanged. O(captured paths). func (s *runState) publishServerChangedFiles() { entry, ok := s.memory["input"] if !ok || entry.output == nil { entry = memoryValue{output: stage.Values{}} } entry.output[serverChangedFilesField] = s.capturedList() s.memory["input"] = entry } // runIdentity is the pipeline identity and stored title a run's pushed frames // carry (RunEventFrame.PipelineID/Name/Title/Version). It is remembered where a // run is created, decided or canceled and stamped onto every frame that leaves // emitRun, so a client that has never seen the run can build its list row from // the frame without re-reading the whole list. type runIdentity struct { scheduled bool scheduleID int64 scheduleName string pipelineID int64 name string title string version int } // identityOfRunSummary is the one projection of a run row onto the identity its // pushed frames carry, so the frame and the list row cannot disagree. func identityOfRunSummary(s protocol.RunSummary) runIdentity { return runIdentity{scheduled: s.Scheduled, scheduleID: s.ScheduleID, scheduleName: s.ScheduleName, pipelineID: s.PipelineID, name: s.Name, title: s.Title, version: s.Version} } // rememberRunIdentity records the identity every later frame of run id carries. // It is safe from any goroutine and takes its own lock (never r.mu), so it is // callable from a path that already holds r.mu. func (r *runner) rememberRunIdentity(id int64, ident runIdentity) { r.identMu.Lock() if r.idents == nil { r.idents = map[int64]runIdentity{} } r.idents[id] = ident r.identMu.Unlock() } // stampRunIdentity fills a frame's blank identity fields from the run's // remembered identity, leaving any the emit site set untouched. A run with no // remembered identity keeps them blank, so the client falls back to its resync. func (r *runner) stampRunIdentity(frame *protocol.RunEventFrame) { r.identMu.Lock() ident, ok := r.idents[frame.RunID] r.identMu.Unlock() if !ok { return } frame.Scheduled = ident.scheduled frame.ScheduleID = ident.scheduleID frame.ScheduleName = ident.scheduleName if frame.PipelineID == 0 { frame.PipelineID = ident.pipelineID } if frame.Name == "" { frame.Name = ident.name } if frame.Title == "" { frame.Title = ident.title } if frame.Version == 0 { frame.Version = ident.version } } // emitRun pushes one run delta to the subscribed-client hub when one is wired, // and does nothing otherwise (a runner built directly in a test has no hub). // terminal marks a final transition, which the hub flushes without coalescing. func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) { // A run that is no longer waiting has no decision pending: its waiting // notifications are cleared on every device. if r.notes != nil && frame.Status != "" && frame.Status != protocol.RunStatusWaiting { c, cancel := context.WithTimeout(context.Background(), 5*time.Second) r.notes.clearFor(c, pipeline.AppNotificationFilter{RunID: frame.RunID, Kinds: []string{protocol.NotificationRunWaiting}}) cancel() } if r.events == nil { return } r.stampRunIdentity(&frame) frame.NotifyMode = r.notifyMode(frame.RunID) r.events.emit(frame, terminal) } func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) { r.mu.Lock() c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if status == "succeeded" { if err := r.finalizeDeliverable(c, id, sequence, input); err != nil { status, outcome, diagnostic = "failed", "error", "deliverable contract: "+err.Error() } } // Measure the run's footprint while it is still recorded as active: its // files are final, and measuring after the terminal status is written would // widen the window in which a client sees the run finished but still owned. // The status write re-marks the cache stale, so the background sizer then // records the exact value including the final output row. size := r.finishedSize(id) if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil { log.Printf("finish run %d: %v", id, err) } // The store atomically makes a stop request win over a late success. detail, getErr := r.store.GetRun(c, id) if getErr != nil || detail == nil { r.mu.Unlock() return } status, outcome, diagnostic = detail.Status, detail.Outcome, detail.Diagnostic r.mu.Unlock() if detail.FinishedAt == "" { return // Cleanup is not confirmed: never emit a false terminal event. } // Push the terminal delta before the attention notification, so a slow ntfy // never delays the live update the app folds. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: status, Outcome: outcome, CurrentStage: currentStage, CompletedSteps: completed, FinishedAt: detail.FinishedAt, SizeBytes: size, }, true) r.kickSizes() if status != protocol.RunStatusCanceled { r.notifyTerminalStep(id, status) } r.deliverRunNotificationBatch(time.Now(), 1) // The run is terminal: nothing more can notify for it. r.forgetAttention(id) r.forgetNotifyMode(id) } // completedSteps counts the completed steps of a run detail; resume uses it to // seed the terminal delta without another store read. func completedSteps(steps []protocol.RunStep) int { n := 0 for _, step := range steps { if step.Status == "completed" { n++ } } return n } // stageUsage carries the usage a model stage already measured for budget // accounting onto the run record. It reports a cost only when the provider // reported one and an in/out split only when the harness reported it, so an // unknown cost or split is never presented as zero. It returns nil when no usage // was observed, so the step stays absent rather than zero. func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage { if !snapshot.TokensKnown && !snapshot.USDKnown { return nil } usage := &protocol.StageUsage{TokensKnown: snapshot.TokensKnown, CostKnown: snapshot.USDKnown} if snapshot.TokensKnown { usage.TokensTotal = snapshot.UsedTokens if snapshot.SplitKnown { usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true // The rest of the component split is kept too: for a caching harness // most of the total is cache reads, which a client shows as cached // rather than fresh tokens. usage.TokensReasoning, usage.CacheRead, usage.CacheWrite, usage.CacheKnown = snapshot.TokensReasoning, snapshot.CacheRead, snapshot.CacheWrite, true } } if snapshot.USDKnown { usage.CostUSD = snapshot.UsedUSD } return usage } // finishStage persists one step, refreshes the shared memory for a completed // step, notifies and classifies the step's target. Both the executed path and // the skipped path use it, so persistence, notification and terminal routing // cannot drift between them. A persist or memory-copy failure is returned for // the caller to fail the run; the caller still owns the model-specific // verdict, quiescence and cancellation handling around it. // turnOutputOnly reports the turn protocol's single markdown body. The verdict // summariser is written for the old field map (summary/option/reason); given a // document it treats it as untrusted reference material and invents a reason. func turnOutputOnly(output stage.Values) bool { if len(output) != 1 { return false } body, ok := output["markdown"] if !ok { return false } _, isText := body.(string) return isText } func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) { r.mu.Lock() save, cancel := context.WithTimeout(context.Background(), 10*time.Second) detail, getErr := r.store.GetRun(save, id) if getErr != nil { cancel() r.mu.Unlock() return "", "", "", false, getErr } if detail.CancelRequested { step.Status, step.Outcome, step.Target = "canceled", "", "" step.Diagnostic = "Canceled by operator; execution returned" } persistErr := r.store.FinishStep(save, id, step) cancel() r.mu.Unlock() if persistErr != nil { return "", "", "", false, persistErr } if detail.CancelRequested { return "canceled", "", step.Diagnostic, true, nil } if step.Status == "completed" { values, copyErr := copyValues(stage.Values(step.Output)) if copyErr != nil { return "", "", "", false, copyErr } state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence} } state.input = stage.Values(step.Output) status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic) if !terminal { r.notifyStep(id, name, workspace, selected.Notify, step) } return status, outcome, diagnostic, terminal, nil } func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) { finish := func(status, outcome, diagnostic string) { r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input) } // canceled settles an operator cancel. A server shutdown is not a cancel: // the run is left exactly as persisted for the next process to continue. canceled := func() { if stage.Detached(ctx) { log.Printf("run %d: server stopping; run left in flight for the next start to continue", id) return } finish("canceled", "", ctx.Err().Error()) } defer func() { if recovered := recover(); recovered != nil { finish("failed", "error", "stage execution panicked") } }() budgets := map[string]*stage.Budget{} if state.key == "" { finish("failed", "error", "stage returned an undeclared outcome") return } // Every process started for this run (stages, the stage service, checks, // captures) gets the workspace's own environment, as a terminal there would. ctx = r.withRunEnvironment(ctx, id, workspace) // Resolve the effective provider/model for every model stage once, before the // loop, so applying an override is an in-memory O(1) map lookup per visit and // never re-parses the graph or re-resolves the whole definition. A run with no // overrides skips the resolution entirely. Launch validation already ran this // helper, so a failure here only guards a resumed run whose overrides drifted. var effectiveModels map[string]pipeline.ModelOverride if request.Model != "" || len(request.StageModels) > 0 { resolved, err := pipeline.EffectiveModels(graph, request.Model, request.StageModels) if err != nil { finish("failed", "error", err.Error()) return } effectiveModels = resolved } for sequence := state.nextSequence; sequence <= 30; sequence++ { if ctx.Err() != nil { canceled() return } key := state.key var def stage.Definition var selected protocol.Stage found := false for _, s := range graph.Stages { if s.Key == key { selected = s def = pipeline.StageDefinition(s) found = true break } } if !found { finish("failed", "error", "missing stage") return } // StageDefinition clones Configs, so setting the effective pair on the // clone is safe and the pinned graph is never mutated. Because // ModelStage.Spec reads def.Configs and passes spec.Config into the repair // path, the same effective model governs the stage and its repair. if stage.EffectiveType(def.Type) == stage.ModelType { if override, ok := effectiveModels[key]; ok { def.Configs["provider"] = override.Provider def.Configs["model"] = override.Model } } isHuman := stage.EffectiveType(def.Type) == stage.HumanType maxVisits := 5 if selected.Limits != nil { maxVisits = selected.Limits.MaxVisits } // Validate the next visit without consuming it. An already-exhausted // stage must not begin work it can never execute. if !isHuman { if state.visits[key]+1 > 5 { finish("failed", "error", "stage visit budget exceeded") return } if state.visits[key]+1 > maxVisits { finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits)) return } } // adopted is the visit a previous server process began and left running // across its restart: its step row, turn files and session already exist. adopted := state.adopt if adopted != nil && adopted.sequence != sequence { adopted = nil } state.adopt = nil _, declaresSkip := def.Edges["skipped"] skip := adopted == nil && !isHuman && declaresSkip && autoSkipEligible(def.Configs, state) state.nextSequence = sequence + 1 state.visits[key]++ // Publish the run's server-captured changed paths into the input memory // value before resolving bindings, so a stage selecting // input.server_changed_files always resolves (an empty list when nothing // was captured). Only a selecting stage copies it into its input. state.publishServerChangedFiles() var reads map[string]protocol.MemoryRead bindings := selected.Memory if stage.EffectiveType(def.Type) == stage.ModelType { // A model stage reads the turn's memory.md and input.md files; the // retired per-field memory block no longer selects its input. bindings = nil } input, reads, copyErr := memoryInput(state.input, bindings, state.memory) if copyErr != nil { finish("failed", "error", copyErr.Error()) return } if changed := state.capturedList(); stage.EffectiveType(def.Type) == stage.ModelType && len(changed) > 0 { // The turn files, not a memory binding, carry the stage context. The // prompt reads this reserved value so a committer still sees the // server-captured paths that distinguish this run from pre-existing // dirty work. An empty list adds nothing and leaves the entry input // unchanged. input[serverChangedFilesField] = changed } var retryInput stage.Values if state.retryOverride != nil { input, copyErr = retryInputOverride(input, state.retryOverride) if copyErr != nil { finish("failed", "error", copyErr.Error()) return } // Keep the merged values for this visit's input.md: the model reads // files, not the persisted step row, so the override must be visible // there. A selected-stage override belongs only to this invocation, // not to a later visit reached through the stage graph. retryInput = input state.retryOverride = nil } if adopted == nil { if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil { if ctx.Err() != nil { canceled() return } finish("failed", "error", err.Error()) return } } // queued -> running: report the new current stage and the step start. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CurrentStage: key, CompletedSteps: state.completed, Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning}, }, false) if isHuman { // A human step is O(1): one BeginStep plus one MarkWaiting, no // filediff capture, no artifacts directory and no workspace mutation. evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges)) r.mu.Lock() if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil { r.mu.Unlock() if ctx.Err() != nil { canceled() return } finish("failed", "error", err.Error()) return } waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence} // A human step parks the run without a live goroutine; the delta is // coalesced by the hub's timer, which keeps running after this returns. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, CurrentStage: key, CompletedSteps: state.completed, Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting}, }, false) // Publishing can be slow. Once waiting is visible, release execution // atomically so a decision is not spuriously rejected as "active". delete(r.active, id) r.mu.Unlock() r.notifyStep(id, name, workspace, selected.Notify, waitStep) r.deliverRunNotificationBatch(time.Now(), 1) return } // Server skip fallback: a stage that declares a `skipped` route is // short-circuited only when the run's evidence is complete and empty // (autoSkipEligible, ui_surface.go): every visit recorded a complete // capture with no file changes and no rendered artifact. Then skip the // model call and its capture work, record the step as completed with a // verdict that names the server fallback, and route along the declared // edge. Partial, unavailable or missing evidence always runs the model. if skip { reason := skipReasonLine() step := protocol.RunStep{ Sequence: sequence, Stage: key, Status: "completed", Outcome: "skipped", Output: stage.Values{"summary": reason}, Target: def.Edges["skipped"], Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}, Verdict: &protocol.StageVerdict{Reason: reason, ReasonSource: protocol.VerdictReasonFallback}, } status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step) if finishErr != nil { finish("failed", "error", finishErr.Error()) return } if terminal { finish(status, outcome, diagnostic) return } state.key = step.Target continue } var tools []string explicitTools := false var toolErr error if stage.EffectiveType(def.Type) == stage.ModelType { tools, explicitTools, toolErr = stage.ToolNames(def) } if toolErr != nil { finish("failed", "error", toolErr.Error()) return } useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools) // Every model visit gets a file-based turn: input.md, memory.md and // history.md are written before execution, output.md after it. Creation is part of the // protocol, so an unusable turn directory fails the run instead of // silently degrading to the retired JSON envelope. artifactsDir := "" var visitTurn turn.Paths var publicBefore map[string]bool if stage.EffectiveType(def.Type) == stage.ModelType { visitTurn = r.turnPaths(id, sequence, key) if adopted == nil { if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil { finish("failed", "error", turnErr.Error()) return } } artifactsDir = visitTurn.TurnDir def.ArtifactsDir = artifactsDir def.Turn = stageTurnPaths(visitTurn) publicBefore = publicFileSet(visitTurn.RunDir) } operations := &stage.OperationLog{} var landing stage.LandingAccess delivery, deliveryErr := stage.DeliveryEnabled(def) if deliveryErr != nil { finish("failed", "error", deliveryErr.Error()) return } if delivery { access, accessErr := landingAccess(workspace) if accessErr != nil { finish("failed", "error", "delivery access denied: "+accessErr.Error()) return } landing = access } client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools) client.LandingAccess = landing client.OnSession = func(session string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference) } var result stage.Result var piClient *stage.PiClient var err error stageCtx := ctx if adopted != nil { // Attach to the surviving session instead of prompting again. client.Backend, client.Reference = adopted.owner.Backend, adopted.owner.Reference stageCtx = stage.WithResumeSession(stageCtx, adopted.owner.ID) } var modelConfig stage.ModelConfig var budgetBefore stage.BudgetSnapshot if stage.EffectiveType(def.Type) == stage.ModelType { spec, specErr := (stage.ModelStage{}).Spec(def) if specErr != nil { finish("failed", "error", specErr.Error()) return } modelConfig = spec.Config if budgets[key] == nil { budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens) if used, ok := state.priorBudgets[key]; ok { budgets[key].RestoreUsage(used) } } // The budget is shared across every visit to this stage key, so capture // its cumulative usage before the visit and record only this visit's // increment on the step, never the cumulative total again. budgetBefore = budgets[key].Snapshot() stageCtx = stage.WithBudget(stageCtx, budgets[key]) } snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second) before := filediff.Capture(snapshotCtx, workspace) snapshotCancel() var stageSession string panicked := false func() { defer func() { if recover() != nil { panicked = true err = fmt.Errorf("stage execution panicked") } }() if ctx.Err() != nil { err = ctx.Err() return } if r.execute != nil { timeout, _ := stage.StageTimeout(def) executeCtx, cancel := context.WithTimeout(stageCtx, timeout) defer cancel() result, err = r.execute(executeCtx, def, input) } else if modelConfig.Harness == "pi" { configured, configErr := harness.Load() if configErr != nil { err = configErr return } profile, ok := configured.Profiles["pi"] if !ok { err = fmt.Errorf("Pi is not configured on execution server") return } piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools) piClient.OnOwnership = func(session, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() return r.store.RunSession(c, id, sequence, session, active, "pi", reference) } result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input) stageSession = piClient.SessionID client.Transcript = piClient.Transcript } else if useAgent { result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input) stageSession = client.SessionID } else if stage.EffectiveType(def.Type) == stage.ModelType { plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir} if adopted != nil { plain.Backend, plain.Reference = adopted.owner.Backend, adopted.owner.Reference } plain.OnGenerationSession = func(session, backend, reference string, active bool) error { c, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if active { stageSession = session } return r.store.RunSession(c, id, sequence, session, active, backend, reference) } result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input) client.Transcript = plain.FormatAudit } else { result, err = stage.DefaultRegistry().Run(stageCtx, def, input) } }() if r.detach && stage.Detached(ctx) { // The server is stopping. The stage left its session running and // owned (or never admitted one); nothing about this visit is settled // here. The next process re-adopts the session and continues. log.Printf("run %d stage %s #%d: server stopping; stage left running for the next start to reattach", id, key, sequence) return } ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second) activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id) ownershipCancel() if ownershipErr != nil || len(activeSessions) > 0 { err = errors.Join(stage.ErrStageCleanup, err, ownershipErr) } if ctx.Err() != nil && !errors.Is(err, stage.ErrStageCleanup) { err = ctx.Err() // Ignore a context-unaware executor's late success. } if stage.EffectiveType(def.Type) == stage.ModelType && err == nil { ensureTurnOutput(visitTurn, result) } // Capture after cleanup, even when the stage's context was canceled. snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) after := filediff.Capture(snapshotCtx, workspace, before) snapshotCancel() changes := filediff.Compare(before, after) if adopted != nil && changes.Status == "complete" { // The pre-visit snapshot belonged to the stopped process; only the // changes made after reattaching were compared. changes.Status = "partial" changes.Diagnostic = "Server restarted during this stage; changes made before the restart are not listed." } if errors.Is(err, stage.ErrStageCleanup) { changes.Status = "partial" changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing." save, cancel := context.WithTimeout(context.Background(), 10*time.Second) if markErr := r.store.MarkCleanupUnconfirmed(save, id); markErr != nil { log.Printf("mark cleanup unconfirmed for run %d: %v", id, markErr) } cancel() } // Only settled, exact recorded ownership can satisfy the handoff gate. // A canceled execution must not cancel this ownership read silently. var sessionExport *protocol.SessionExport exportRequired := artifactsDir != "" && r.execute == nil if artifactsDir != "" { ownerCtx, ownerCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) owner, ownerErr := r.store.RepairSessionOwner(ownerCtx, id, sequence) if ownerErr != nil && r.execute != nil { // Only genuinely unowned fake executors are exempt. A recorded // session with absent ownership metadata must still fail closed. if recorded, readErr := r.store.RunSessionOwner(ownerCtx, id, sequence); readErr != nil || recorded.ID != "" { exportRequired = true if readErr == nil { stageSession = recorded.ID } } } ownerCancel() if ownerErr == nil && owner.ID != "" { exportRequired = true // Owned test executors exercise the real gate too. stageSession = owner.ID if !errors.Is(err, stage.ErrStageCleanup) { sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig) } } else if exportRequired { sessionExport = &protocol.SessionExport{Status: "failed", Source: "native_export", Diagnostic: "exact settled session ownership unavailable; handoff refused"} } } artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir, visitTurn.PublicDir) step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, SessionExport: sessionExport, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated} // Stamp the effective provider/model this visit actually ran with for every // model stage, including failures, so the app/TUI evidence reflects any // override. Non-model stages keep empty values. The usage the stage already // measured for budget accounting is carried onto the same step, never // re-read from the harness or parsed from the transcript. if stage.EffectiveType(def.Type) == stage.ModelType { step.Provider = modelConfig.Provider step.Model = modelConfig.Model if budget := budgets[key]; budget != nil { step.Usage = stageUsage(budget.Snapshot().Since(budgetBefore)) } } if budget := budgets[key]; budget != nil { step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries) } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} if len(result.Output) > 0 { step.Output["salvage"] = result.Output } } if err == nil && result.Outcome == "success" { target, targetErr := stage.VerificationTargetFor(def) if targetErr != nil { err = targetErr } else if target != nil { if r.verifyTarget == nil { err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture) } else { evidence, verificationErr := r.verifyTarget(stageCtx, *target) if verificationErr != nil { err = verificationErr } else { err = stage.VerifyTargetEvidence(*target, evidence) } } } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} } } if err == nil && delivery && result.Outcome == "success" { checks, checksErr := stage.DeliveryChecks(def) if checksErr != nil { err = checksErr } else { gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout) var catchUp string catchUp, err = verifyDelivery(gateCtx, workspace, landing, checks, r.deliveryCommitMessage(id)) gateCancel() // A failed gate already carries the note in its diagnostic. A step // diagnostic on success renders as a stop, so success is logged. if catchUp != "" { log.Printf("run %d stage %s: %s", id, key, catchUp) } } if err != nil { step.Status = "failed" step.Diagnostic = err.Error() step.Outcome = "error" step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input} } } // A failed or timed-out step carries the session pointer even when the // bounded transcript would otherwise be only its stage_limits envelope. if err != nil && stageSession != "" { step.Transcript = stage.RetainSession(step.Transcript, stageSession) } step.Target = def.Edges[step.Outcome] if errors.Is(err, stage.ErrStageCleanup) { // Retain the partial evidence without pretending that the agent stopped. step.Status, step.Outcome, step.Target = "running", "", "" save, cancel := context.WithTimeout(context.Background(), 10*time.Second) if saveErr := r.store.FinishStep(save, id, step); saveErr != nil { log.Printf("save cleanup evidence for run %d: %v", id, saveErr) } cancel() r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CleanupUnconfirmed: true}, false) return } step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads} if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits { for _, outcome := range selected.Limits.On { if outcome == step.Outcome { step.Control.LimitReached = true step.Control.ModelTarget = step.Target step.Target = protocol.FailureEnd step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome) } } } if panicked { step.Target = protocol.FailureEnd } _, _, _, failedTerminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic) if exportRequired && !completeSessionExport(sessionExport) && !(err != nil && failedTerminal && step.Target != protocol.SuccessEnd) { // Preserve the real computation, files, outcome and target. In // particular an export failure is not an agent failure/retry request. save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) saveErr := r.store.FinishStepExportBlocked(save, id, step) cancel() if saveErr != nil { finish("failed", "error", saveErr.Error()) return } // No finishStage, memory publication, recordTurn or next executor. check, done := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) detail, readErr := r.store.GetRun(check, id) done() if readErr != nil { log.Printf("read parked export gate for run %d: %v", id, readErr) return // The checkpoint is durable; never fabricate a terminal. } if detail.CancelRequested && detail.ExportPendingSequence == 0 { finish("canceled", "", "Canceled by operator; computed visit retained") return } r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, ExportPendingSequence: sequence, CurrentStage: key, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts)}}, true) return } // Generate the one-line verdict reason once, at completion. It is // best-effort: a summariser failure or timeout returns NoSummary and must // never fail the stage or the run, so every error is swallowed. A failed // step still gets a verdict from its error envelope. A turn output is one // markdown body, which the field-map summariser reads as untrusted data and // answers with an invented rationale; those steps get the honest read-time // fallback instead. if r.verdict != nil && !turnOutputOnly(step.Output) { reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout) if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary { step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel} } reasonCancel() } status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step) if finishErr != nil { finish("failed", "error", finishErr.Error()) return } if ctx.Err() != nil { canceled() return } // Advance the in-memory UI-surface evidence only after the step is // persisted, so the skip guard agrees with what the store holds. state.recordChanges(step.FileChanges, step.Artifacts) if step.Status == "completed" { added := []string(nil) if stage.EffectiveType(def.Type) == stage.ModelType { added = addedPublicFiles(publicBefore, publicFileSet(visitTurn.RunDir)) } state.recordTurn(step, added) state.completed++ if artifactsDir != "" { state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence)) } } // Step finished: report its outcome, routing target, diagnostic and the // artifacts recorded for the visit, plus the run's running step count. r.emitRun(protocol.RunEventFrame{ RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CurrentStage: key, CompletedSteps: state.completed, Artifacts: len(step.Artifacts), Step: &protocol.RunStepDelta{ Sequence: step.Sequence, Stage: step.Stage, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts), }, }, false) if ctx.Err() != nil { canceled() return } if terminal { finish(status, outcome, diagnostic) return } state.key = step.Target } finish("failed", "error", "total stage budget exceeded") } // exportStageSession writes the finished stage's session into its own artifacts // directory and returns verified evidence. The caller enforces the handoff gate; // an export failure never replaces the computation's result. func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport { evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend, Directory: owner.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339)} defer classifySessionExport(evidence) if owner.Backend != "v1" && owner.Backend != "v2" && !(owner.Backend == "pi" && owner.Reference != "") { evidence.Source = "unavailable" evidence.Diagnostic = "session export unavailable: harness has no native export path" return evidence } export := r.exportSession if export == nil { export = stage.ExportSession } // A canceled stage still gets one bounded, non-blocking attempt; the export // never consumes the stage timeout and never waits for the session. exportCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stage.SessionExportTimeout) defer cancel() result, err := export(exportCtx, stage.SessionExportRequest{ SessionID: owner.ID, Backend: owner.Backend, Reference: owner.Reference, Directory: owner.Directory, ArtifactsDir: artifactsDir, RunID: runID, Sequence: sequence, Stage: key, Provider: modelConfig.Provider, Model: modelConfig.Model, }) evidence.Attempts = result.Attempts if err != nil { evidence.Source = "native_export" evidence.Diagnostic = stage.RedactEvidence(err.Error()) return evidence } evidence.Source = result.Source evidence.Status = result.Status evidence.HTML = result.HTML evidence.HTMLBytes = result.HTMLBytes evidence.Messages = result.Messages evidence.Truncated = result.Truncated evidence.Diagnostic = result.Diagnostic if result.Status != "" && result.Status != "complete" && evidence.Diagnostic == "" { evidence.Diagnostic = "native session export reported " + result.Status } verifySessionExportFile(evidence, artifactsDir) return evidence } // setTitle renames a run: it replaces the stored operator-facing title without // touching the run's input, steps or execution, then returns the updated detail. func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) { explicit, ok := summarise.ExplicitTitle(title) if !ok { return nil, fmt.Errorf("a title with usable text is required") } if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil { return nil, err } return r.getRun(id) }