Files
pipelines/internal/server/runner.go
T
alan 890d267c71 run attention: publish to ntfy and surface what needs the operator
Nothing told the operator that a run was waiting for a person or had failed; they
found out by going looking. A run that needs attention now reaches the ntfy they
already run, and the app answers "what needs me now" in one live, ordered surface.

Server (smallest change, reusing the existing publisher):
- internal/server/notify.go adds notifyRun, called from finishRunState (failed)
  and when a run parks on a human (runner.go runLoop). It publishes to the same
  already-configured `system` destination through the unchanged stage.PublishNtfy,
  so there is no second publisher, topic or service and the token handling is the
  stage's own. runAttentionEvent maps only waiting and failed; queued, running,
  succeeded, canceled and interrupted publish nothing.
- No storms: runner.attentionSent records sequence:event per run before publishing,
  so a run that stays waiting is never republished on a re-request or restart, and
  a stage notify policy that already covers the same event marks it first so the
  run-level default is suppressed. The set is dropped when the run is terminal. A
  delivery failure is logged, never fails the run, and is not retried.
- The payload names the run and pipeline and carries a click action
  pipelines://run/<id> (validated: absolute URL, no userinfo, no fragment, no
  credentials). PublishNtfy gained the optional click parameter; existing callers
  pass "".

App:
- AndroidManifest/MainActivity declare the pipelines://run/<id> deep link
  (singleTop); runIdFromDeepLink parses it and PipelinesApp opens that run once,
  clearing the link. It opens a run; it starts nothing.
- ui/Attention.kt is the pure model (status -> Decision/Failed/Finished or
  nothing, ordered by kind then newest-first) and ui/components/AttentionSurface.kt
  is the floating indicator plus bottom sheet. It is derived from the same run
  events the server already pushes, so an entry appears with no refresh and clears
  when resolved; a finished run clears once looked at, and any entry can be
  dismissed, persisted in SettingsRepository. The existing destinations, pill and
  bottom strip are untouched and no competing navigation surface is added.

Checked before committing:
- Packaged authoring skill references: internal/author/skill/references/
  notifications.md documents the automatic run-attention notifications and the
  app surface. The only packaged "template" carrying notify is that reference; no
  pipeline definition template exists to update and install.go's managed file list
  is unchanged (notifications.md already existed).
- Repository guidelines: docs/ntfy.md gains an "Automatic run-attention
  notifications" section and no longer claims "No block means no notifications";
  the top-level README phone-notifications note now says a waiting/failed run
  notifies automatically; android/README.md gains the dated feature section and
  supersedes the previous section's note that the attention surface was absent.
- docker.md, install docs and the CLI/TUI docs describe mechanisms this change
  does not alter; left as they are.

Tests: go test ./internal/server/... ./internal/stage/... ./internal/author/...
all ok; notify_test.go covers the event mapping and the real HTTP delivery
(destination, topic, click, no duplicate). Android :app:testDebugUnitTest ->
BUILD SUCCESSFUL, 58 suites / 531 tests / 0 failures (new AttentionTest, 13 cases,
covering ordering, appearing from a push, clearing on resolution, dismissal, the
empty case, persistence and the deep link). Device screenshots and the phone's own
notification are visual review's per the brief's device policy.

Scope: android/ plus the smallest server change; no new dependency, no run started
from the app, no installer or Docker change.
2026-10-17 16:11:57 +01:00

1073 lines
41 KiB
Go

package server
import (
"context"
"errors"
"fmt"
"log"
"maps"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"git.a-wan.net/alan/pipelines/internal/filediff"
"git.a-wan.net/alan/pipelines/internal/harness"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"git.a-wan.net/alan/pipelines/internal/summarise"
)
// activeRun records one in-flight run's cancellation seam and the resolved
// workspace it is changing. A single map is the only active-run state, so the
// workspace cannot drift from the cancel function.
type activeRun struct {
cancel context.CancelFunc
workspace string
}
type runner struct {
ctx context.Context
store *pipeline.Store
artifactsRoot string
mu sync.Mutex
active map[int64]activeRun
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
// title generates the operator-facing run title from the run prompt. When nil
// the store derives a fallback from the prompt with no model call, so tests
// and unconfigured servers stay deterministic.
title func(context.Context, string) summarise.Title
// verdict generates the bounded one-line reason for a completed stage visit
// from its output map. When nil the store derives the reason at read time with
// no model call. It is best-effort: any failure must never fail the stage.
verdict func(context.Context, map[string]any) string
// events pushes run state deltas to subscribed clients. When nil (tests that
// build a runner directly) emits are dropped, so nothing else changes.
events *runEventHub
wg sync.WaitGroup
schedulerWG sync.WaitGroup
// attentionMu guards attentionSent, the per-run set of run-attention events
// already delivered — by a stage's own notify policy or by the run-level
// default — so neither republishes the same genuine event.
attentionMu sync.Mutex
attentionSent map[int64]map[string]bool
}
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
sessions, err := s.ActiveSessionOwners(ctx)
if err != nil {
return nil, err
}
for _, owner := range sessions {
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
var err error
if owner.Backend == "pi" {
err = stage.InterruptPi(stop, owner.Reference)
} else {
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
}
cancel()
if err != nil {
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
}
}
if err := s.RecoverRuns(ctx); err != nil {
return nil, err
}
root := filepath.Join(s.Directory(), "artifacts")
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, attentionSent: map[int64]map[string]bool{}}, nil
}
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
return nil, fmt.Errorf("an explicit version and request key are required")
}
workspace, err := os.Getwd()
if err != nil {
return nil, err
}
workspace, err = filepath.EvalSymlinks(workspace)
if err != nil {
return nil, err
}
if request.Workspace != "" {
if !filepath.IsAbs(request.Workspace) {
return nil, fmt.Errorf("run workspace must be an absolute directory")
}
selected, err := filepath.EvalSymlinks(request.Workspace)
if err != nil {
return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err)
}
info, err := os.Stat(selected)
if err != nil || !info.IsDir() {
return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace)
}
workspace = selected
}
if len(request.WritePaths) > 20 {
return nil, fmt.Errorf("too many write paths")
}
for _, path := range request.WritePaths {
if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") {
return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths")
}
parent := filepath.Join(workspace, path)
for {
_, err := os.Lstat(parent)
if err == nil {
break
}
if !os.IsNotExist(err) {
return nil, err
}
next := filepath.Dir(parent)
if next == parent {
return nil, fmt.Errorf("write path is unavailable")
}
parent = next
}
resolved, err := filepath.EvalSymlinks(parent)
if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) {
return nil, fmt.Errorf("write path must stay within the run directory")
}
}
request.Input = maps.Clone(request.Input)
if request.Input == nil {
request.Input = map[string]any{}
}
request.Input["pwd"] = workspace
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
}
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
if err != nil {
return nil, err
}
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
return nil, err
}
// Validate the launch-time model overrides against the pinned graph before
// admission: an unknown stage key or a malformed provider/model must never
// insert a run row. The resolution is O(#overrides + #stages) with no model or
// network call, and the same helper is the authoritative precedence rule.
if _, err := pipeline.EffectiveModels(d.Graph, request.Model, request.StageModels); err != nil {
return nil, err
}
for _, s := range d.Graph.Stages {
if stage.EffectiveType(s.Type) != stage.ModelType {
continue
}
if s.Configs["harness"] == "pi" && r.execute == nil {
configured, err := harness.Load()
if err != nil {
return nil, err
}
if _, ok := configured.Profiles["pi"]; !ok {
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
}
}
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
if err != nil {
return nil, err
}
if !explicit {
request.AllowTools = true
}
for _, tool := range tools {
if tool == "edit" || tool == "shell" {
request.AllowTools = true
}
}
}
// Decide admission and idempotency under the lock, but generate the title
// only for a request that will actually start a run: a rejected or replayed
// request must not pay for a bounded model generation, and a live one must
// not hold the lock (stalling listings and cancellation) while a model is
// called. An unavailable or slow model falls back to the prompt's first line.
r.mu.Lock()
if r.ctx.Err() != nil {
r.mu.Unlock()
return nil, r.ctx.Err()
}
owners, err := r.store.ActiveSessionOwners(r.ctx)
if err != nil {
r.mu.Unlock()
return nil, err
}
// Refuse only an orphaned session: one whose owning run is not active now.
// A session owned by a currently active run (another run in another
// workspace) must not block new work.
for _, owner := range owners {
if _, ok := r.active[owner.RunID]; !ok {
r.mu.Unlock()
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
}
}
// Idempotent repeats are accepted even while the original run is active, and
// a replay reuses the stored title instead of generating a new one.
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
if err != nil {
r.mu.Unlock()
return nil, err
}
if existing == 0 {
if err := r.checkWorkspaceFree(workspace); err != nil {
r.mu.Unlock()
return nil, err
}
}
r.mu.Unlock()
var runTitle summarise.Title
if existing == 0 {
// An explicit operator title skips the model call; otherwise the
// configured generator names the run, falling back to the prompt's
// first line when it is unavailable or slow.
if explicit, ok := summarise.ExplicitTitle(request.Title); ok {
runTitle = explicit
} else if r.title != nil {
prompt, _ := request.Input["prompt"].(string)
runTitle = r.title(r.ctx, prompt)
}
}
r.mu.Lock()
defer r.mu.Unlock()
// Another run may have reserved this workspace while the title was
// generated, so re-check before reserving it; a race on the request key is
// handled by the store's idempotent insert.
if existing == 0 {
if err := r.checkWorkspaceFree(workspace); err != nil {
return nil, err
}
}
id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Model: request.Model, StageModels: request.StageModels}, runTitle, retryOf...)
if err != nil {
return nil, err
}
if created {
// Emit the created delta before launching: the queued frame reaches
// subscribers before the running/step-started transition, so a client
// sees the run appear and then advance.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusQueued,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}, false)
r.launch(id, workspace, func(ctx context.Context) {
r.run(ctx, id, d, workspace, request)
})
}
detail, err := r.store.GetRun(r.ctx, id)
if detail != nil {
_, detail.Executing = r.active[id]
r.overlayEffectiveModels(detail)
}
return detail, err
}
// launch registers an active run's cancellation seam and workspace, adds it to
// the wait group and starts fn on a derived cancellable context that cleans up
// the registration when it returns. A fresh run and a resumed decision both go
// through it, so registration and teardown live once. Callers must hold r.mu.
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
ctx, cancel := context.WithCancel(r.ctx)
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
r.wg.Add(1)
go func() {
defer r.wg.Done()
defer cancel()
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
fn(ctx)
}()
}
// checkWorkspaceFree refuses a run when an active run is already changing the
// same resolved workspace. The workspace is the only scope a run reserves, and
// one active run per workspace keeps this scan finding at most one match, so the
// message is deterministic. Callers must hold r.mu.
func (r *runner) checkWorkspaceFree(workspace string) error {
for id, entry := range r.active {
if entry.workspace == workspace {
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
}
}
return nil
}
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
r.mu.Lock()
defer r.mu.Unlock()
d, err := r.store.GetRun(r.ctx, id)
if d != nil {
_, d.Executing = r.active[id]
for _, s := range d.Steps {
if s.AgentActive {
d.Executing = true
}
}
r.overlayEffectiveModels(d)
}
return d, err
}
// overlayEffectiveModels replaces each model stage's provider/model in a returned
// run's graph with the effective pair the run actually uses, so the app and TUI
// stage-definition surfaces show the override with no client change. It is
// presentation only: the persisted graph_json and the pinned source returned by
// `show` are never rewritten. A run with no overrides is left untouched.
func (r *runner) overlayEffectiveModels(d *protocol.RunDetail) {
if d == nil || (d.Model == "" && len(d.StageModels) == 0) {
return
}
effective, err := pipeline.EffectiveModels(d.Graph, d.Model, d.StageModels)
if err != nil {
return
}
for i := range d.Graph.Stages {
s := &d.Graph.Stages[i]
if stage.EffectiveType(s.Type) != stage.ModelType {
continue
}
override, ok := effective[s.Key]
if !ok {
continue
}
if s.Configs == nil {
s.Configs = map[string]string{}
}
s.Configs["provider"] = override.Provider
s.Configs["model"] = override.Model
}
}
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
r.mu.Lock()
defer r.mu.Unlock()
items, err := r.store.ListRuns(r.ctx)
for i := range items {
_, items[i].Executing = r.active[items[i].ID]
if items[i].Status == "failed" && !items[i].Executing {
d, err := r.store.GetRun(r.ctx, items[i].ID)
if err != nil {
return nil, err
}
for _, step := range d.Steps {
if step.AgentActive {
items[i].Executing = true
}
}
}
}
return items, err
}
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("retry requires a positive run ID")
}
original, err := r.getRun(request.RunID)
if err != nil {
return nil, err
}
if original.Executing {
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
}
if original.Status != "failed" || original.FinishedAt == "" {
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
}
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
}
func (r *runner) cancel(id int64) error {
r.mu.Lock()
entry, ok := r.active[id]
r.mu.Unlock()
if ok {
entry.cancel()
return nil
}
// A run that is waiting on a person has no live goroutine, so cancel it at
// the persistence layer instead. This keeps cancel working across a restart.
// The single follow-up read seeds the terminal delta; cancel is a cold
// operator action, not a per-transition hot path.
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
return err
}
// A canceled run is terminal: nothing more can notify for it.
r.forgetAttention(id)
if r.events != nil {
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: detail.Workspace,
Status: detail.Status,
Outcome: detail.Outcome,
CurrentStage: detail.CurrentStage,
CompletedSteps: detail.CompletedSteps,
FinishedAt: detail.FinishedAt,
}, true)
}
}
return nil
}
// decide records an operator's choice for a waiting human stage and resumes the
// run along the chosen edge. ResolveDecision owns validation and idempotency, so
// decide does no option parsing of its own. A resumed run is registered and
// launched exactly like a fresh one but starts from the persisted state.
func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
if request.RunID <= 0 {
return nil, fmt.Errorf("decide requires a positive run ID")
}
r.mu.Lock()
detail, err := r.store.GetRun(r.ctx, request.RunID)
if err != nil {
r.mu.Unlock()
return nil, err
}
if _, busy := r.active[detail.ID]; busy {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
}
if detail.Status == protocol.RunStatusWaiting {
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
r.mu.Unlock()
return nil, err
}
}
step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback)
if err != nil {
r.mu.Unlock()
return nil, err
}
if applied {
for i := range detail.Steps {
if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage {
detail.Steps[i] = step
break
}
}
// The decision closes the waiting step and returns the run to running;
// report that before the resumed loop emits its own deltas.
r.emitRun(protocol.RunEventFrame{
RunID: detail.ID,
Workspace: detail.Workspace,
Status: protocol.RunStatusRunning,
CurrentStage: step.Stage,
CompletedSteps: completedSteps(detail.Steps),
Step: &protocol.RunStepDelta{
Sequence: step.Sequence,
Stage: step.Stage,
Status: step.Status,
Outcome: step.Outcome,
Target: step.Target,
Diagnostic: step.Diagnostic,
},
}, false)
r.launch(detail.ID, detail.Workspace, func(ctx context.Context) {
r.resume(ctx, detail.ID, detail, detail.Workspace, step)
})
}
r.mu.Unlock()
return r.getRun(request.RunID)
}
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
original, copyErr := copyValues(stage.Values(request.Input))
if copyErr != nil {
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
return
}
input := stage.Values(request.Input)
state := newRunState(d.Graph.Stages[0].Key, input)
state.memory["input"] = memoryValue{output: original}
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
}
// terminalFinish classifies a stage's resolved target. A terminal key yields the
// run's final status, outcome and diagnostic; an empty target means the stage
// named an undeclared outcome; any other value is a stage key and is not
// terminal. resume and the loop tail share it, so terminal routing lives once.
func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) {
switch target {
case protocol.SuccessEnd:
return "succeeded", "success", "", true
case protocol.FailureEnd:
if diagnostic == "" {
diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome)
}
return "failed", "error", diagnostic, true
case "":
return "failed", "error", "stage returned an undeclared outcome", true
}
return "", "", "", false
}
// resume continues a decided run from the state already persisted for its run
// detail: memory from the run input and the completed steps, visits counted per
// stage, and the next sequence above the highest recorded one. It then runs the
// shared loop from the decision's target; no full-table scan and no extra
// per-step query are introduced.
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
// The decision may route straight to a terminal, which has no stage to run.
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
r.finishRunState(id, d.Name, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Sequence, step.Stage, stage.Values(step.Output))
return
}
state := rebuildRunState(d, step)
r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools, Model: d.Model, StageModels: d.StageModels}, state)
}
// runState is the loop's whole mutable state, extracted so a fresh run and a
// resumed run share one implementation. nextSequence is set before each visit so
// a wait that returns to the caller records where the run will continue.
type runState struct {
key string
nextSequence int
visits map[string]int
memory map[string]memoryValue
input stage.Values
// changedPaths retains a bounded sample of the run's recorded changed paths
// and changedCount the full total, so the skip guard reads only in-memory
// evidence. uiSurface is latched once any recorded change touched a UI
// surface. Both are folded in by recordChanges (ui_surface.go).
changedPaths []string
changedCount int
uiSurface bool
// capturedPaths is the full, deduplicated, first-seen-ordered set of the
// run's server-captured changed paths, distinct from the bounded
// changedPaths sample used only by the skip reason. capturedSeen is the
// single dedupe set. Both are folded in by recordChanges (ui_surface.go) and
// published to a selecting stage as input.server_changed_files.
capturedPaths []string
capturedSeen map[string]bool
// completed counts the run's finished steps, so a delta can report the
// running total without re-reading the store.
completed int
}
func newRunState(key string, input stage.Values) *runState {
return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input}
}
func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState {
state := newRunState(decided.Target, stage.Values(decided.Output))
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
state.memory["input"] = memoryValue{output: original}
}
for _, step := range detail.Steps {
state.visits[step.Stage]++
if step.Sequence >= state.nextSequence {
state.nextSequence = step.Sequence + 1
}
if step.Status == "completed" {
if values, err := copyValues(stage.Values(step.Output)); err == nil {
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
state.completed++
}
// Seed the UI-surface accumulator from the persisted evidence so a
// resumed run reaches the same skip decision with no extra query.
state.recordChanges(step.FileChanges, step.Artifacts)
}
return state
}
// serverChangedFilesField is the reserved run-input memory field under which the
// runner publishes the server-captured changed paths. A stage selects it with
// `"server_changed_files":"input.server_changed_files"`, alongside another
// stage's declared list, so it sees the verified evidence, not just a model's
// claim of authorship.
const serverChangedFilesField = "server_changed_files"
// capturedList returns the full, first-seen-ordered set of server-captured
// changed paths, or a non-nil empty list when nothing was captured so the field
// still resolves as a list. It copies the accumulator so the retained evidence
// cannot be aliased by a published value. O(captured paths).
func (s *runState) capturedList() []string {
if len(s.capturedPaths) == 0 {
return []string{}
}
return append([]string(nil), s.capturedPaths...)
}
// publishServerChangedFiles writes the run's captured changed-path set into the
// run input memory value under serverChangedFilesField, creating the entry and
// its output map when absent. Called before memoryInput so a stage selecting
// input.server_changed_files always resolves; only that field is added and only
// a selecting stage copies it out, so state.input and other stages' input are
// unchanged. O(captured paths).
func (s *runState) publishServerChangedFiles() {
entry, ok := s.memory["input"]
if !ok || entry.output == nil {
entry = memoryValue{output: stage.Values{}}
}
entry.output[serverChangedFilesField] = s.capturedList()
s.memory["input"] = entry
}
// emitRun pushes one run delta to the subscribed-client hub when one is wired,
// and does nothing otherwise (a runner built directly in a test has no hub).
// terminal marks a final transition, which the hub flushes without coalescing.
func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
if r.events == nil {
return
}
r.events.emit(frame, terminal)
}
func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
log.Printf("finish run %d: %v", id, err)
}
// Push the terminal delta before the attention notification, so a slow ntfy
// never delays the live update the app folds.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: status,
Outcome: outcome,
CurrentStage: currentStage,
CompletedSteps: completed,
FinishedAt: time.Now().UTC().Format(time.RFC3339),
}, true)
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
// The run is terminal: nothing more can notify for it.
r.forgetAttention(id)
}
// completedSteps counts the completed steps of a run detail; resume uses it to
// seed the terminal delta without another store read.
func completedSteps(steps []protocol.RunStep) int {
n := 0
for _, step := range steps {
if step.Status == "completed" {
n++
}
}
return n
}
// finishStage persists one step, refreshes the shared memory for a completed
// step, notifies and classifies the step's target. Both the executed path and
// the skipped path use it, so persistence, notification and terminal routing
// cannot drift between them. A persist or memory-copy failure is returned for
// the caller to fail the run; the caller still owns the model-specific
// verdict, quiescence and cancellation handling around it.
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
persistErr := r.store.FinishStep(save, id, step)
cancel()
if persistErr != nil {
return "", "", "", false, persistErr
}
if step.Status == "completed" {
values, copyErr := copyValues(stage.Values(step.Output))
if copyErr != nil {
return "", "", "", false, copyErr
}
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
}
r.notifyStep(id, name, workspace, selected.Notify, step)
state.input = stage.Values(step.Output)
status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
return status, outcome, diagnostic, terminal, nil
}
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
finish := func(status, outcome, diagnostic string) {
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
}
defer func() {
if recovered := recover(); recovered != nil {
finish("failed", "error", "stage execution panicked")
}
}()
budgets := map[string]*stage.Budget{}
if state.key == "" {
finish("failed", "error", "stage returned an undeclared outcome")
return
}
// Resolve the effective provider/model for every model stage once, before the
// loop, so applying an override is an in-memory O(1) map lookup per visit and
// never re-parses the graph or re-resolves the whole definition. A run with no
// overrides skips the resolution entirely. Launch validation already ran this
// helper, so a failure here only guards a resumed run whose overrides drifted.
var effectiveModels map[string]pipeline.ModelOverride
if request.Model != "" || len(request.StageModels) > 0 {
resolved, err := pipeline.EffectiveModels(graph, request.Model, request.StageModels)
if err != nil {
finish("failed", "error", err.Error())
return
}
effectiveModels = resolved
}
for sequence := state.nextSequence; sequence <= 30; sequence++ {
state.nextSequence = sequence + 1
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
key := state.key
var def stage.Definition
var selected protocol.Stage
found := false
for _, s := range graph.Stages {
if s.Key == key {
selected = s
def = pipeline.StageDefinition(s)
found = true
break
}
}
if !found {
finish("failed", "error", "missing stage")
return
}
// StageDefinition clones Configs, so setting the effective pair on the
// clone is safe and the pinned graph is never mutated. Because
// ModelStage.Spec reads def.Configs and passes spec.Config into the repair
// path, the same effective model governs the stage and its repair.
if stage.EffectiveType(def.Type) == stage.ModelType {
if override, ok := effectiveModels[key]; ok {
def.Configs["provider"] = override.Provider
def.Configs["model"] = override.Model
}
}
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
state.visits[key]++
maxVisits := 5
if selected.Limits != nil {
maxVisits = selected.Limits.MaxVisits
}
// A human stage pauses for a person; neither the global visit budget nor
// the per-stage max_visits may kill it. The shared 30-step run budget
// still bounds every run.
if !isHuman {
if state.visits[key] > 5 {
finish("failed", "error", "stage visit budget exceeded")
return
}
if state.visits[key] > maxVisits {
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
return
}
}
// Publish the run's server-captured changed paths into the input memory
// value before resolving bindings, so a stage selecting
// input.server_changed_files always resolves (an empty list when nothing
// was captured). Only a selecting stage copies it into its input.
state.publishServerChangedFiles()
var reads map[string]protocol.MemoryRead
input, reads, copyErr := memoryInput(state.input, selected.Memory, state.memory)
if copyErr != nil {
finish("failed", "error", copyErr.Error())
return
}
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
finish("failed", "error", err.Error())
return
}
// queued -> running: report the new current stage and the step start.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
}, false)
if isHuman {
// A human step is O(1): one BeginStep plus one MarkWaiting, no
// filediff capture, no artifacts directory and no workspace mutation.
evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges))
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
finish("failed", "error", err.Error())
return
}
waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence}
// A human step parks the run without a live goroutine; the delta is
// coalesced by the hub's timer, which keeps running after this returns.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusWaiting,
CurrentStage: key,
CompletedSteps: state.completed,
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
}, false)
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
r.notifyRun(id, name, protocol.RunStatusWaiting, sequence, key, "")
return
}
// A stage that declares a `skipped` route is short-circuited when the run
// has recorded no UI surface: the stage would answer exactly that, so skip
// the model call and its capture work, record the step as completed and
// route along the declared edge. Only a declared route is honoured, and no
// other stage or outcome routing changes.
if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface {
reason := skipReasonLine(state)
step := protocol.RunStep{
Sequence: sequence,
Stage: key,
Status: "completed",
Outcome: "skipped",
Output: stage.Values{"summary": reason},
Target: def.Edges["skipped"],
Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads},
}
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
if finishErr != nil {
finish("failed", "error", finishErr.Error())
return
}
if terminal {
finish(status, outcome, diagnostic)
return
}
state.key = step.Target
continue
}
var tools []string
explicitTools := false
var toolErr error
if stage.EffectiveType(def.Type) == stage.ModelType {
tools, explicitTools, toolErr = stage.ToolNames(def)
}
if toolErr != nil {
finish("failed", "error", toolErr.Error())
return
}
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
// Every agent visit gets a per-run/step directory for the files it
// produces. Creation is best-effort: an unusable directory must never
// fail the run, and it is only attempted for model stages.
artifactsDir := ""
if stage.EffectiveType(def.Type) == stage.ModelType {
candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence))
if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil {
log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr)
} else {
artifactsDir = candidate
def.ArtifactsDir = candidate
}
}
operations := &stage.OperationLog{}
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools}
client.OnSession = func(session string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
}
var result stage.Result
var piClient *stage.PiClient
var err error
stageCtx := ctx
var modelConfig stage.ModelConfig
if stage.EffectiveType(def.Type) == stage.ModelType {
spec, specErr := (stage.ModelStage{}).Spec(def)
if specErr != nil {
finish("failed", "error", specErr.Error())
return
}
modelConfig = spec.Config
if budgets[key] == nil {
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
}
stageCtx = stage.WithBudget(ctx, budgets[key])
}
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
before := filediff.Capture(snapshotCtx, workspace)
snapshotCancel()
var stageSession string
panicked := false
func() {
defer func() {
if recover() != nil {
panicked = true
err = fmt.Errorf("stage execution panicked")
}
}()
if r.execute != nil {
timeout, _ := stage.StageTimeout(def)
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
defer cancel()
result, err = r.execute(executeCtx, def, input)
} else if modelConfig.Harness == "pi" {
configured, configErr := harness.Load()
if configErr != nil {
err = configErr
return
}
profile, ok := configured.Profiles["pi"]
if !ok {
err = fmt.Errorf("Pi is not configured on execution server")
return
}
piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir}
piClient.OnOwnership = func(session, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
}
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
stageSession = piClient.SessionID
client.Transcript = piClient.Transcript
} else if useAgent {
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
stageSession = client.SessionID
} else if stage.EffectiveType(def.Type) == stage.ModelType {
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if active {
stageSession = session
}
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
}
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
client.Transcript = plain.FormatAudit
} else {
result, err = stage.DefaultRegistry().Run(ctx, def, input)
}
}()
// Capture after cleanup, even when the stage's context was canceled.
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
after := filediff.Capture(snapshotCtx, workspace, before)
snapshotCancel()
changes := filediff.Compare(before, after)
if errors.Is(err, stage.ErrStageCleanup) {
changes.Status = "partial"
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
}
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir)
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
// Stamp the effective provider/model this visit actually ran with for every
// model stage, including failures, so the app/TUI evidence reflects any
// override. Non-model stages keep empty values.
if stage.EffectiveType(def.Type) == stage.ModelType {
step.Provider = modelConfig.Provider
step.Model = modelConfig.Model
}
if budget := budgets[key]; budget != nil {
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
}
if err != nil {
step.Status = "failed"
step.Diagnostic = err.Error()
step.Outcome = "error"
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
if len(result.Output) > 0 {
step.Output["salvage"] = result.Output
}
}
// A failed or timed-out step carries the session pointer even when the
// bounded transcript would otherwise be only its stage_limits envelope.
if err != nil && stageSession != "" {
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
}
step.Target = def.Edges[step.Outcome]
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
for _, outcome := range selected.Limits.On {
if outcome == step.Outcome {
step.Control.LimitReached = true
step.Control.ModelTarget = step.Target
step.Target = protocol.FailureEnd
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
}
}
}
if panicked {
step.Target = protocol.FailureEnd
}
// Generate the one-line verdict reason once, at completion. It is
// best-effort: a summariser failure or timeout returns NoSummary and must
// never fail the stage or the run, so every error is swallowed. A failed
// step still gets a verdict from its error envelope.
if r.verdict != nil {
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
}
reasonCancel()
}
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
if finishErr != nil {
finish("failed", "error", finishErr.Error())
return
}
// Advance the in-memory UI-surface evidence only after the step is
// persisted, so the skip guard agrees with what the store holds.
state.recordChanges(step.FileChanges, step.Artifacts)
if step.Status == "completed" {
state.completed++
}
// Step finished: report its outcome, routing target, diagnostic and the
// artifacts recorded for the visit, plus the run's running step count.
r.emitRun(protocol.RunEventFrame{
RunID: id,
Workspace: workspace,
Status: protocol.RunStatusRunning,
CurrentStage: key,
CompletedSteps: state.completed,
Artifacts: len(step.Artifacts),
Step: &protocol.RunStepDelta{
Sequence: step.Sequence,
Stage: step.Stage,
Status: step.Status,
Outcome: step.Outcome,
Target: step.Target,
Diagnostic: step.Diagnostic,
Artifacts: len(step.Artifacts),
},
}, false)
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
if checkErr != nil || len(sessions) > 0 {
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
return
}
}
if ctx.Err() != nil {
finish("canceled", "", ctx.Err().Error())
return
}
if terminal {
finish(status, outcome, diagnostic)
return
}
state.key = step.Target
}
finish("failed", "error", "total stage budget exceeded")
}
// setTitle renames a run: it replaces the stored operator-facing title without
// touching the run's input, steps or execution, then returns the updated detail.
func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) {
explicit, ok := summarise.ExplicitTitle(title)
if !ok {
return nil, fmt.Errorf("a title with usable text is required")
}
if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil {
return nil, err
}
return r.getRun(id)
}