98713e55f5
The committer staged only the implementer's self-declared changed_files, so an incomplete declaration silently left part of the approved change uncommitted. The runner already records the run's real changed paths by comparing the working tree around each step; publish that set to the committer. - runState gains an unbounded, deduplicated, first-seen-ordered captured path accumulator (capturedPaths/capturedSeen), distinct from the bounded changedPaths sample used only by the UI-surface skip reason. - recordChanges folds each captured FileChange path into it in the existing single pass, so no second walk and no new I/O. - publishServerChangedFiles writes the set into the input memory value as input.server_changed_files, called before memoryInput so the binding always resolves (an empty list when nothing was captured) and only stages that select it receive it. rebuildRunState rebuilds it from persisted completed steps, so a resumed run hands the committer the same evidence. - Tests prove the committer receives the captured set distinct from and additive to the implementer's declared changed_files, that an omitted path is still present, and that the field is an empty list when nothing was captured. - control.md and orchestration.md document the reserved field and instruct the committer to select input.server_changed_files alongside the declared changed_files and stage their union. Commands run: go build ./...; go vet ./...; go test -count=1 ./internal/server/... ./internal/stage/...; go test -count=1 ./internal/author/...; go test -count=1 -run TestCommitterReceivesServerCapturedChangedPaths ./internal/server/. All passed. go.mod/go.sum unchanged.
867 lines
32 KiB
Go
867 lines
32 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/filediff"
|
|
"git.a-wan.net/alan/pipelines/internal/harness"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
// activeRun records one in-flight run's cancellation seam and the resolved
|
|
// workspace it is changing. A single map is the only active-run state, so the
|
|
// workspace cannot drift from the cancel function.
|
|
type activeRun struct {
|
|
cancel context.CancelFunc
|
|
workspace string
|
|
}
|
|
|
|
type runner struct {
|
|
ctx context.Context
|
|
store *pipeline.Store
|
|
artifactsRoot string
|
|
mu sync.Mutex
|
|
active map[int64]activeRun
|
|
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
|
|
// title generates the operator-facing run title from the run prompt. When nil
|
|
// the store derives a fallback from the prompt with no model call, so tests
|
|
// and unconfigured servers stay deterministic.
|
|
title func(context.Context, string) summarise.Title
|
|
// verdict generates the bounded one-line reason for a completed stage visit
|
|
// from its output map. When nil the store derives the reason at read time with
|
|
// no model call. It is best-effort: any failure must never fail the stage.
|
|
verdict func(context.Context, map[string]any) string
|
|
wg sync.WaitGroup
|
|
schedulerWG sync.WaitGroup
|
|
}
|
|
|
|
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
|
sessions, err := s.ActiveSessionOwners(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, owner := range sessions {
|
|
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
var err error
|
|
if owner.Backend == "pi" {
|
|
err = stage.InterruptPi(stop, owner.Reference)
|
|
} else {
|
|
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
|
|
}
|
|
cancel()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
|
}
|
|
}
|
|
if err := s.RecoverRuns(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
root := filepath.Join(s.Directory(), "artifacts")
|
|
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}}, nil
|
|
}
|
|
|
|
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
|
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
|
return nil, fmt.Errorf("an explicit version and request key are required")
|
|
}
|
|
workspace, err := os.Getwd()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
workspace, err = filepath.EvalSymlinks(workspace)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if request.Workspace != "" {
|
|
if !filepath.IsAbs(request.Workspace) {
|
|
return nil, fmt.Errorf("run workspace must be an absolute directory")
|
|
}
|
|
selected, err := filepath.EvalSymlinks(request.Workspace)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err)
|
|
}
|
|
info, err := os.Stat(selected)
|
|
if err != nil || !info.IsDir() {
|
|
return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace)
|
|
}
|
|
workspace = selected
|
|
}
|
|
if len(request.WritePaths) > 20 {
|
|
return nil, fmt.Errorf("too many write paths")
|
|
}
|
|
for _, path := range request.WritePaths {
|
|
if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") {
|
|
return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths")
|
|
}
|
|
parent := filepath.Join(workspace, path)
|
|
for {
|
|
_, err := os.Lstat(parent)
|
|
if err == nil {
|
|
break
|
|
}
|
|
if !os.IsNotExist(err) {
|
|
return nil, err
|
|
}
|
|
next := filepath.Dir(parent)
|
|
if next == parent {
|
|
return nil, fmt.Errorf("write path is unavailable")
|
|
}
|
|
parent = next
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(parent)
|
|
if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) {
|
|
return nil, fmt.Errorf("write path must stay within the run directory")
|
|
}
|
|
}
|
|
request.Input = maps.Clone(request.Input)
|
|
if request.Input == nil {
|
|
request.Input = map[string]any{}
|
|
}
|
|
request.Input["pwd"] = workspace
|
|
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
|
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
|
}
|
|
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["harness"] == "pi" && r.execute == nil {
|
|
configured, err := harness.Load()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, ok := configured.Profiles["pi"]; !ok {
|
|
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
|
|
}
|
|
}
|
|
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !explicit {
|
|
request.AllowTools = true
|
|
}
|
|
for _, tool := range tools {
|
|
if tool == "edit" || tool == "shell" {
|
|
request.AllowTools = true
|
|
}
|
|
}
|
|
}
|
|
// Decide admission and idempotency under the lock, but generate the title
|
|
// only for a request that will actually start a run: a rejected or replayed
|
|
// request must not pay for a bounded model generation, and a live one must
|
|
// not hold the lock (stalling listings and cancellation) while a model is
|
|
// called. An unavailable or slow model falls back to the prompt's first line.
|
|
r.mu.Lock()
|
|
if r.ctx.Err() != nil {
|
|
r.mu.Unlock()
|
|
return nil, r.ctx.Err()
|
|
}
|
|
owners, err := r.store.ActiveSessionOwners(r.ctx)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
// Refuse only an orphaned session: one whose owning run is not active now.
|
|
// A session owned by a currently active run (another run in another
|
|
// workspace) must not block new work.
|
|
for _, owner := range owners {
|
|
if _, ok := r.active[owner.RunID]; !ok {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
|
}
|
|
}
|
|
// Idempotent repeats are accepted even while the original run is active, and
|
|
// a replay reuses the stored title instead of generating a new one.
|
|
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
var runTitle summarise.Title
|
|
if existing == 0 {
|
|
// An explicit operator title skips the model call; otherwise the
|
|
// configured generator names the run, falling back to the prompt's
|
|
// first line when it is unavailable or slow.
|
|
if explicit, ok := summarise.ExplicitTitle(request.Title); ok {
|
|
runTitle = explicit
|
|
} else if r.title != nil {
|
|
prompt, _ := request.Input["prompt"].(string)
|
|
runTitle = r.title(r.ctx, prompt)
|
|
}
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
// Another run may have reserved this workspace while the title was
|
|
// generated, so re-check before reserving it; a race on the request key is
|
|
// handled by the store's idempotent insert.
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, runTitle, retryOf...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if created {
|
|
r.launch(id, workspace, func(ctx context.Context) {
|
|
r.run(ctx, id, d, workspace, request)
|
|
})
|
|
}
|
|
detail, err := r.store.GetRun(r.ctx, id)
|
|
if detail != nil {
|
|
_, detail.Executing = r.active[id]
|
|
}
|
|
return detail, err
|
|
}
|
|
|
|
// launch registers an active run's cancellation seam and workspace, adds it to
|
|
// the wait group and starts fn on a derived cancellable context that cleans up
|
|
// the registration when it returns. A fresh run and a resumed decision both go
|
|
// through it, so registration and teardown live once. Callers must hold r.mu.
|
|
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
|
ctx, cancel := context.WithCancel(r.ctx)
|
|
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
|
|
r.wg.Add(1)
|
|
go func() {
|
|
defer r.wg.Done()
|
|
defer cancel()
|
|
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
|
|
fn(ctx)
|
|
}()
|
|
}
|
|
|
|
// checkWorkspaceFree refuses a run when an active run is already changing the
|
|
// same resolved workspace. The workspace is the only scope a run reserves, and
|
|
// one active run per workspace keeps this scan finding at most one match, so the
|
|
// message is deterministic. Callers must hold r.mu.
|
|
func (r *runner) checkWorkspaceFree(workspace string) error {
|
|
for id, entry := range r.active {
|
|
if entry.workspace == workspace {
|
|
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
d, err := r.store.GetRun(r.ctx, id)
|
|
if d != nil {
|
|
_, d.Executing = r.active[id]
|
|
for _, s := range d.Steps {
|
|
if s.AgentActive {
|
|
d.Executing = true
|
|
}
|
|
}
|
|
}
|
|
return d, err
|
|
}
|
|
|
|
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
items, err := r.store.ListRuns(r.ctx)
|
|
for i := range items {
|
|
_, items[i].Executing = r.active[items[i].ID]
|
|
if items[i].Status == "failed" && !items[i].Executing {
|
|
d, err := r.store.GetRun(r.ctx, items[i].ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, step := range d.Steps {
|
|
if step.AgentActive {
|
|
items[i].Executing = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items, err
|
|
}
|
|
|
|
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("retry requires a positive run ID")
|
|
}
|
|
original, err := r.getRun(request.RunID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if original.Executing {
|
|
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
|
|
}
|
|
if original.Status != "failed" || original.FinishedAt == "" {
|
|
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
|
}
|
|
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID)
|
|
}
|
|
|
|
func (r *runner) cancel(id int64) error {
|
|
r.mu.Lock()
|
|
entry, ok := r.active[id]
|
|
r.mu.Unlock()
|
|
if ok {
|
|
entry.cancel()
|
|
return nil
|
|
}
|
|
// A run that is waiting on a person has no live goroutine, so cancel it at
|
|
// the persistence layer instead. This keeps cancel working across a restart.
|
|
return r.store.CancelWaiting(r.ctx, id)
|
|
}
|
|
|
|
// decide records an operator's choice for a waiting human stage and resumes the
|
|
// run along the chosen edge. ResolveDecision owns validation and idempotency, so
|
|
// decide does no option parsing of its own. A resumed run is registered and
|
|
// launched exactly like a fresh one but starts from the persisted state.
|
|
func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("decide requires a positive run ID")
|
|
}
|
|
r.mu.Lock()
|
|
detail, err := r.store.GetRun(r.ctx, request.RunID)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if _, busy := r.active[detail.ID]; busy {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
|
|
}
|
|
if detail.Status == protocol.RunStatusWaiting {
|
|
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if applied {
|
|
for i := range detail.Steps {
|
|
if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage {
|
|
detail.Steps[i] = step
|
|
break
|
|
}
|
|
}
|
|
r.launch(detail.ID, detail.Workspace, func(ctx context.Context) {
|
|
r.resume(ctx, detail.ID, detail, detail.Workspace, step)
|
|
})
|
|
}
|
|
r.mu.Unlock()
|
|
return r.getRun(request.RunID)
|
|
}
|
|
|
|
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
|
|
original, copyErr := copyValues(stage.Values(request.Input))
|
|
if copyErr != nil {
|
|
r.finishRunState(id, "failed", "error", copyErr.Error(), stage.Values(request.Input))
|
|
return
|
|
}
|
|
input := stage.Values(request.Input)
|
|
state := newRunState(d.Graph.Stages[0].Key, input)
|
|
state.memory["input"] = memoryValue{output: original}
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
|
|
}
|
|
|
|
// terminalFinish classifies a stage's resolved target. A terminal key yields the
|
|
// run's final status, outcome and diagnostic; an empty target means the stage
|
|
// named an undeclared outcome; any other value is a stage key and is not
|
|
// terminal. resume and the loop tail share it, so terminal routing lives once.
|
|
func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) {
|
|
switch target {
|
|
case protocol.SuccessEnd:
|
|
return "succeeded", "success", "", true
|
|
case protocol.FailureEnd:
|
|
if diagnostic == "" {
|
|
diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome)
|
|
}
|
|
return "failed", "error", diagnostic, true
|
|
case "":
|
|
return "failed", "error", "stage returned an undeclared outcome", true
|
|
}
|
|
return "", "", "", false
|
|
}
|
|
|
|
// resume continues a decided run from the state already persisted for its run
|
|
// detail: memory from the run input and the completed steps, visits counted per
|
|
// stage, and the next sequence above the highest recorded one. It then runs the
|
|
// shared loop from the decision's target; no full-table scan and no extra
|
|
// per-step query are introduced.
|
|
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
|
|
// The decision may route straight to a terminal, which has no stage to run.
|
|
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
|
|
r.finishRunState(id, status, outcome, diagnostic, stage.Values(step.Output))
|
|
return
|
|
}
|
|
state := rebuildRunState(d, step)
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools}, state)
|
|
}
|
|
|
|
// runState is the loop's whole mutable state, extracted so a fresh run and a
|
|
// resumed run share one implementation. nextSequence is set before each visit so
|
|
// a wait that returns to the caller records where the run will continue.
|
|
type runState struct {
|
|
key string
|
|
nextSequence int
|
|
visits map[string]int
|
|
memory map[string]memoryValue
|
|
input stage.Values
|
|
// changedPaths retains a bounded sample of the run's recorded changed paths
|
|
// and changedCount the full total, so the skip guard reads only in-memory
|
|
// evidence. uiSurface is latched once any recorded change touched a UI
|
|
// surface. Both are folded in by recordChanges (ui_surface.go).
|
|
changedPaths []string
|
|
changedCount int
|
|
uiSurface bool
|
|
// capturedPaths is the full, deduplicated, first-seen-ordered set of the
|
|
// run's server-captured changed paths, distinct from the bounded
|
|
// changedPaths sample used only by the skip reason. capturedSeen is the
|
|
// single dedupe set. Both are folded in by recordChanges (ui_surface.go) and
|
|
// published to a selecting stage as input.server_changed_files.
|
|
capturedPaths []string
|
|
capturedSeen map[string]bool
|
|
}
|
|
|
|
func newRunState(key string, input stage.Values) *runState {
|
|
return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input}
|
|
}
|
|
|
|
func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState {
|
|
state := newRunState(decided.Target, stage.Values(decided.Output))
|
|
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
|
state.memory["input"] = memoryValue{output: original}
|
|
}
|
|
for _, step := range detail.Steps {
|
|
state.visits[step.Stage]++
|
|
if step.Sequence >= state.nextSequence {
|
|
state.nextSequence = step.Sequence + 1
|
|
}
|
|
if step.Status == "completed" {
|
|
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
}
|
|
// Seed the UI-surface accumulator from the persisted evidence so a
|
|
// resumed run reaches the same skip decision with no extra query.
|
|
state.recordChanges(step.FileChanges, step.Artifacts)
|
|
}
|
|
return state
|
|
}
|
|
|
|
// serverChangedFilesField is the reserved run-input memory field under which the
|
|
// runner publishes the server-captured changed paths. A stage selects it with
|
|
// `"server_changed_files":"input.server_changed_files"`, alongside another
|
|
// stage's declared list, so it sees the verified evidence, not just a model's
|
|
// claim of authorship.
|
|
const serverChangedFilesField = "server_changed_files"
|
|
|
|
// capturedList returns the full, first-seen-ordered set of server-captured
|
|
// changed paths, or a non-nil empty list when nothing was captured so the field
|
|
// still resolves as a list. It copies the accumulator so the retained evidence
|
|
// cannot be aliased by a published value. O(captured paths).
|
|
func (s *runState) capturedList() []string {
|
|
if len(s.capturedPaths) == 0 {
|
|
return []string{}
|
|
}
|
|
return append([]string(nil), s.capturedPaths...)
|
|
}
|
|
|
|
// publishServerChangedFiles writes the run's captured changed-path set into the
|
|
// run input memory value under serverChangedFilesField, creating the entry and
|
|
// its output map when absent. Called before memoryInput so a stage selecting
|
|
// input.server_changed_files always resolves; only that field is added and only
|
|
// a selecting stage copies it out, so state.input and other stages' input are
|
|
// unchanged. O(captured paths).
|
|
func (s *runState) publishServerChangedFiles() {
|
|
entry, ok := s.memory["input"]
|
|
if !ok || entry.output == nil {
|
|
entry = memoryValue{output: stage.Values{}}
|
|
}
|
|
entry.output[serverChangedFilesField] = s.capturedList()
|
|
s.memory["input"] = entry
|
|
}
|
|
|
|
func (r *runner) finishRunState(id int64, status, outcome, diagnostic string, input stage.Values) {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
|
|
log.Printf("finish run %d: %v", id, err)
|
|
}
|
|
}
|
|
|
|
// finishStage persists one step, refreshes the shared memory for a completed
|
|
// step, notifies and classifies the step's target. Both the executed path and
|
|
// the skipped path use it, so persistence, notification and terminal routing
|
|
// cannot drift between them. A persist or memory-copy failure is returned for
|
|
// the caller to fail the run; the caller still owns the model-specific
|
|
// verdict, quiescence and cancellation handling around it.
|
|
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
persistErr := r.store.FinishStep(save, id, step)
|
|
cancel()
|
|
if persistErr != nil {
|
|
return "", "", "", false, persistErr
|
|
}
|
|
if step.Status == "completed" {
|
|
values, copyErr := copyValues(stage.Values(step.Output))
|
|
if copyErr != nil {
|
|
return "", "", "", false, copyErr
|
|
}
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
r.notifyStep(id, name, workspace, selected.Notify, step)
|
|
state.input = stage.Values(step.Output)
|
|
status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
return status, outcome, diagnostic, terminal, nil
|
|
}
|
|
|
|
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
|
|
finish := func(status, outcome, diagnostic string) {
|
|
r.finishRunState(id, status, outcome, diagnostic, state.input)
|
|
}
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
finish("failed", "error", "stage execution panicked")
|
|
}
|
|
}()
|
|
budgets := map[string]*stage.Budget{}
|
|
if state.key == "" {
|
|
finish("failed", "error", "stage returned an undeclared outcome")
|
|
return
|
|
}
|
|
for sequence := state.nextSequence; sequence <= 30; sequence++ {
|
|
state.nextSequence = sequence + 1
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
key := state.key
|
|
var def stage.Definition
|
|
var selected protocol.Stage
|
|
found := false
|
|
for _, s := range graph.Stages {
|
|
if s.Key == key {
|
|
selected = s
|
|
def = pipeline.StageDefinition(s)
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
finish("failed", "error", "missing stage")
|
|
return
|
|
}
|
|
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
|
|
state.visits[key]++
|
|
maxVisits := 5
|
|
if selected.Limits != nil {
|
|
maxVisits = selected.Limits.MaxVisits
|
|
}
|
|
// A human stage pauses for a person; neither the global visit budget nor
|
|
// the per-stage max_visits may kill it. The shared 30-step run budget
|
|
// still bounds every run.
|
|
if !isHuman {
|
|
if state.visits[key] > 5 {
|
|
finish("failed", "error", "stage visit budget exceeded")
|
|
return
|
|
}
|
|
if state.visits[key] > maxVisits {
|
|
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
|
|
return
|
|
}
|
|
}
|
|
// Publish the run's server-captured changed paths into the input memory
|
|
// value before resolving bindings, so a stage selecting
|
|
// input.server_changed_files always resolves (an empty list when nothing
|
|
// was captured). Only a selecting stage copies it into its input.
|
|
state.publishServerChangedFiles()
|
|
var reads map[string]protocol.MemoryRead
|
|
input, reads, copyErr := memoryInput(state.input, selected.Memory, state.memory)
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
if isHuman {
|
|
// A human step is O(1): one BeginStep plus one MarkWaiting, no
|
|
// filediff capture, no artifacts directory and no workspace mutation.
|
|
evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges))
|
|
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence}
|
|
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
|
|
return
|
|
}
|
|
// A stage that declares a `skipped` route is short-circuited when the run
|
|
// has recorded no UI surface: the stage would answer exactly that, so skip
|
|
// the model call and its capture work, record the step as completed and
|
|
// route along the declared edge. Only a declared route is honoured, and no
|
|
// other stage or outcome routing changes.
|
|
if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface {
|
|
reason := skipReasonLine(state)
|
|
step := protocol.RunStep{
|
|
Sequence: sequence,
|
|
Stage: key,
|
|
Status: "completed",
|
|
Outcome: "skipped",
|
|
Output: stage.Values{"summary": reason},
|
|
Target: def.Edges["skipped"],
|
|
Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads},
|
|
}
|
|
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
|
|
if finishErr != nil {
|
|
finish("failed", "error", finishErr.Error())
|
|
return
|
|
}
|
|
if terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
continue
|
|
}
|
|
var tools []string
|
|
explicitTools := false
|
|
var toolErr error
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
tools, explicitTools, toolErr = stage.ToolNames(def)
|
|
}
|
|
if toolErr != nil {
|
|
finish("failed", "error", toolErr.Error())
|
|
return
|
|
}
|
|
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
|
|
// Every agent visit gets a per-run/step directory for the files it
|
|
// produces. Creation is best-effort: an unusable directory must never
|
|
// fail the run, and it is only attempted for model stages.
|
|
artifactsDir := ""
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence))
|
|
if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil {
|
|
log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr)
|
|
} else {
|
|
artifactsDir = candidate
|
|
def.ArtifactsDir = candidate
|
|
}
|
|
}
|
|
operations := &stage.OperationLog{}
|
|
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools}
|
|
client.OnSession = func(session string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
|
|
}
|
|
var result stage.Result
|
|
var piClient *stage.PiClient
|
|
var err error
|
|
stageCtx := ctx
|
|
var modelConfig stage.ModelConfig
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
spec, specErr := (stage.ModelStage{}).Spec(def)
|
|
if specErr != nil {
|
|
finish("failed", "error", specErr.Error())
|
|
return
|
|
}
|
|
modelConfig = spec.Config
|
|
if budgets[key] == nil {
|
|
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
|
}
|
|
stageCtx = stage.WithBudget(ctx, budgets[key])
|
|
}
|
|
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
|
before := filediff.Capture(snapshotCtx, workspace)
|
|
snapshotCancel()
|
|
var stageSession string
|
|
panicked := false
|
|
func() {
|
|
defer func() {
|
|
if recover() != nil {
|
|
panicked = true
|
|
err = fmt.Errorf("stage execution panicked")
|
|
}
|
|
}()
|
|
if r.execute != nil {
|
|
timeout, _ := stage.StageTimeout(def)
|
|
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
|
|
defer cancel()
|
|
result, err = r.execute(executeCtx, def, input)
|
|
} else if modelConfig.Harness == "pi" {
|
|
configured, configErr := harness.Load()
|
|
if configErr != nil {
|
|
err = configErr
|
|
return
|
|
}
|
|
profile, ok := configured.Profiles["pi"]
|
|
if !ok {
|
|
err = fmt.Errorf("Pi is not configured on execution server")
|
|
return
|
|
}
|
|
piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir}
|
|
piClient.OnOwnership = func(session, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
|
|
stageSession = piClient.SessionID
|
|
client.Transcript = piClient.Transcript
|
|
} else if useAgent {
|
|
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
|
|
stageSession = client.SessionID
|
|
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
|
|
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if active {
|
|
stageSession = session
|
|
}
|
|
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
|
|
client.Transcript = plain.FormatAudit
|
|
} else {
|
|
result, err = stage.DefaultRegistry().Run(ctx, def, input)
|
|
}
|
|
}()
|
|
// Capture after cleanup, even when the stage's context was canceled.
|
|
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
after := filediff.Capture(snapshotCtx, workspace, before)
|
|
snapshotCancel()
|
|
changes := filediff.Compare(before, after)
|
|
if errors.Is(err, stage.ErrStageCleanup) {
|
|
changes.Status = "partial"
|
|
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
|
|
}
|
|
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir)
|
|
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
if len(result.Output) > 0 {
|
|
step.Output["salvage"] = result.Output
|
|
}
|
|
}
|
|
// A failed or timed-out step carries the session pointer even when the
|
|
// bounded transcript would otherwise be only its stage_limits envelope.
|
|
if err != nil && stageSession != "" {
|
|
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
|
|
}
|
|
step.Target = def.Edges[step.Outcome]
|
|
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
|
|
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
|
|
for _, outcome := range selected.Limits.On {
|
|
if outcome == step.Outcome {
|
|
step.Control.LimitReached = true
|
|
step.Control.ModelTarget = step.Target
|
|
step.Target = protocol.FailureEnd
|
|
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
|
|
}
|
|
}
|
|
}
|
|
if panicked {
|
|
step.Target = protocol.FailureEnd
|
|
}
|
|
// Generate the one-line verdict reason once, at completion. It is
|
|
// best-effort: a summariser failure or timeout returns NoSummary and must
|
|
// never fail the stage or the run, so every error is swallowed. A failed
|
|
// step still gets a verdict from its error envelope.
|
|
if r.verdict != nil {
|
|
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
|
|
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
|
|
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
|
|
}
|
|
reasonCancel()
|
|
}
|
|
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
|
|
if finishErr != nil {
|
|
finish("failed", "error", finishErr.Error())
|
|
return
|
|
}
|
|
// Advance the in-memory UI-surface evidence only after the step is
|
|
// persisted, so the skip guard agrees with what the store holds.
|
|
state.recordChanges(step.FileChanges, step.Artifacts)
|
|
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
|
|
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
|
|
if checkErr != nil || len(sessions) > 0 {
|
|
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
|
|
return
|
|
}
|
|
}
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
if terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
}
|
|
finish("failed", "error", "total stage budget exceeded")
|
|
}
|
|
|
|
// setTitle renames a run: it replaces the stored operator-facing title without
|
|
// touching the run's input, steps or execution, then returns the updated detail.
|
|
func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) {
|
|
explicit, ok := summarise.ExplicitTitle(title)
|
|
if !ok {
|
|
return nil, fmt.Errorf("a title with usable text is required")
|
|
}
|
|
if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil {
|
|
return nil, err
|
|
}
|
|
return r.getRun(id)
|
|
}
|