b05277a57d
Introduce a short, markdown-ish result format whose first non-empty line is
the declared outcome and whose "# output" section carries the routing data,
so stage output stops carrying multi-kilobyte prose inside a JSON envelope.
- New internal/stage/line_result.go: resultFormat() gates legacy JSON ('{'
first) vs the new line format; decodeLineResult() parses outcome-from-first
line, requires a JSON-object "# output" section, stores extra sections
verbatim under normalised keys, and records a reserved result_parse status
(parsed/partial/failed). Failures return a salvage Result plus
ErrInvalidModelResult, and never route an undeclared outcome.
- internal/stage/model.go: write the raw response to raw-output.md (0600,
best-effort) before parsing, and route the line format through
decodeLineResult while leaving the JSON path (stageResultText ->
decodeModelResult -> repairModelResult) unchanged.
- internal/stage/agent.go: agentResultText returns line-format text verbatim
so the "# output" body is not stripped by the JSON-object extractor.
- internal/server/runner.go: merge a non-empty salvage Result into the error
envelope under "salvage" on failure (a no-op for the legacy JSON path).
- internal/stage/line_result_test.go: raw-text fixtures covering happy path,
missing/undeclared outcome, missing "# output", best-effort extras,
adversarial prose, legacy JSON unchanged, >=64KB responses, and
raw-output.md written before parsing.
767 lines
28 KiB
Go
767 lines
28 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/filediff"
|
|
"git.a-wan.net/alan/pipelines/internal/harness"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
// activeRun records one in-flight run's cancellation seam and the resolved
|
|
// workspace it is changing. A single map is the only active-run state, so the
|
|
// workspace cannot drift from the cancel function.
|
|
type activeRun struct {
|
|
cancel context.CancelFunc
|
|
workspace string
|
|
}
|
|
|
|
type runner struct {
|
|
ctx context.Context
|
|
store *pipeline.Store
|
|
artifactsRoot string
|
|
mu sync.Mutex
|
|
active map[int64]activeRun
|
|
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
|
|
// title generates the operator-facing run title from the run prompt. When nil
|
|
// the store derives a fallback from the prompt with no model call, so tests
|
|
// and unconfigured servers stay deterministic.
|
|
title func(context.Context, string) summarise.Title
|
|
// verdict generates the bounded one-line reason for a completed stage visit
|
|
// from its output map. When nil the store derives the reason at read time with
|
|
// no model call. It is best-effort: any failure must never fail the stage.
|
|
verdict func(context.Context, map[string]any) string
|
|
wg sync.WaitGroup
|
|
schedulerWG sync.WaitGroup
|
|
}
|
|
|
|
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
|
sessions, err := s.ActiveSessionOwners(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, owner := range sessions {
|
|
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
var err error
|
|
if owner.Backend == "pi" {
|
|
err = stage.InterruptPi(stop, owner.Reference)
|
|
} else {
|
|
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
|
|
}
|
|
cancel()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
|
}
|
|
}
|
|
if err := s.RecoverRuns(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
root := filepath.Join(s.Directory(), "artifacts")
|
|
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}}, nil
|
|
}
|
|
|
|
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
|
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
|
return nil, fmt.Errorf("an explicit version and request key are required")
|
|
}
|
|
workspace, err := os.Getwd()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
workspace, err = filepath.EvalSymlinks(workspace)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if request.Workspace != "" {
|
|
if !filepath.IsAbs(request.Workspace) {
|
|
return nil, fmt.Errorf("run workspace must be an absolute directory")
|
|
}
|
|
selected, err := filepath.EvalSymlinks(request.Workspace)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err)
|
|
}
|
|
info, err := os.Stat(selected)
|
|
if err != nil || !info.IsDir() {
|
|
return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace)
|
|
}
|
|
workspace = selected
|
|
}
|
|
if len(request.WritePaths) > 20 {
|
|
return nil, fmt.Errorf("too many write paths")
|
|
}
|
|
for _, path := range request.WritePaths {
|
|
if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") {
|
|
return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths")
|
|
}
|
|
parent := filepath.Join(workspace, path)
|
|
for {
|
|
_, err := os.Lstat(parent)
|
|
if err == nil {
|
|
break
|
|
}
|
|
if !os.IsNotExist(err) {
|
|
return nil, err
|
|
}
|
|
next := filepath.Dir(parent)
|
|
if next == parent {
|
|
return nil, fmt.Errorf("write path is unavailable")
|
|
}
|
|
parent = next
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(parent)
|
|
if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) {
|
|
return nil, fmt.Errorf("write path must stay within the run directory")
|
|
}
|
|
}
|
|
request.Input = maps.Clone(request.Input)
|
|
if request.Input == nil {
|
|
request.Input = map[string]any{}
|
|
}
|
|
request.Input["pwd"] = workspace
|
|
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
|
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
|
}
|
|
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["harness"] == "pi" && r.execute == nil {
|
|
configured, err := harness.Load()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, ok := configured.Profiles["pi"]; !ok {
|
|
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
|
|
}
|
|
}
|
|
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !explicit {
|
|
request.AllowTools = true
|
|
}
|
|
for _, tool := range tools {
|
|
if tool == "edit" || tool == "shell" {
|
|
request.AllowTools = true
|
|
}
|
|
}
|
|
}
|
|
// Decide admission and idempotency under the lock, but generate the title
|
|
// only for a request that will actually start a run: a rejected or replayed
|
|
// request must not pay for a bounded model generation, and a live one must
|
|
// not hold the lock (stalling listings and cancellation) while a model is
|
|
// called. An unavailable or slow model falls back to the prompt's first line.
|
|
r.mu.Lock()
|
|
if r.ctx.Err() != nil {
|
|
r.mu.Unlock()
|
|
return nil, r.ctx.Err()
|
|
}
|
|
owners, err := r.store.ActiveSessionOwners(r.ctx)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
// Refuse only an orphaned session: one whose owning run is not active now.
|
|
// A session owned by a currently active run (another run in another
|
|
// workspace) must not block new work.
|
|
for _, owner := range owners {
|
|
if _, ok := r.active[owner.RunID]; !ok {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
|
}
|
|
}
|
|
// Idempotent repeats are accepted even while the original run is active, and
|
|
// a replay reuses the stored title instead of generating a new one.
|
|
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
var runTitle summarise.Title
|
|
if existing == 0 {
|
|
// An explicit operator title skips the model call; otherwise the
|
|
// configured generator names the run, falling back to the prompt's
|
|
// first line when it is unavailable or slow.
|
|
if explicit, ok := summarise.ExplicitTitle(request.Title); ok {
|
|
runTitle = explicit
|
|
} else if r.title != nil {
|
|
prompt, _ := request.Input["prompt"].(string)
|
|
runTitle = r.title(r.ctx, prompt)
|
|
}
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
// Another run may have reserved this workspace while the title was
|
|
// generated, so re-check before reserving it; a race on the request key is
|
|
// handled by the store's idempotent insert.
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, runTitle, retryOf...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if created {
|
|
r.launch(id, workspace, func(ctx context.Context) {
|
|
r.run(ctx, id, d, workspace, request)
|
|
})
|
|
}
|
|
detail, err := r.store.GetRun(r.ctx, id)
|
|
if detail != nil {
|
|
_, detail.Executing = r.active[id]
|
|
}
|
|
return detail, err
|
|
}
|
|
|
|
// launch registers an active run's cancellation seam and workspace, adds it to
|
|
// the wait group and starts fn on a derived cancellable context that cleans up
|
|
// the registration when it returns. A fresh run and a resumed decision both go
|
|
// through it, so registration and teardown live once. Callers must hold r.mu.
|
|
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
|
ctx, cancel := context.WithCancel(r.ctx)
|
|
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
|
|
r.wg.Add(1)
|
|
go func() {
|
|
defer r.wg.Done()
|
|
defer cancel()
|
|
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
|
|
fn(ctx)
|
|
}()
|
|
}
|
|
|
|
// checkWorkspaceFree refuses a run when an active run is already changing the
|
|
// same resolved workspace. The workspace is the only scope a run reserves, and
|
|
// one active run per workspace keeps this scan finding at most one match, so the
|
|
// message is deterministic. Callers must hold r.mu.
|
|
func (r *runner) checkWorkspaceFree(workspace string) error {
|
|
for id, entry := range r.active {
|
|
if entry.workspace == workspace {
|
|
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
d, err := r.store.GetRun(r.ctx, id)
|
|
if d != nil {
|
|
_, d.Executing = r.active[id]
|
|
for _, s := range d.Steps {
|
|
if s.AgentActive {
|
|
d.Executing = true
|
|
}
|
|
}
|
|
}
|
|
return d, err
|
|
}
|
|
|
|
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
items, err := r.store.ListRuns(r.ctx)
|
|
for i := range items {
|
|
_, items[i].Executing = r.active[items[i].ID]
|
|
if items[i].Status == "failed" && !items[i].Executing {
|
|
d, err := r.store.GetRun(r.ctx, items[i].ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, step := range d.Steps {
|
|
if step.AgentActive {
|
|
items[i].Executing = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items, err
|
|
}
|
|
|
|
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("retry requires a positive run ID")
|
|
}
|
|
original, err := r.getRun(request.RunID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if original.Executing {
|
|
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
|
|
}
|
|
if original.Status != "failed" || original.FinishedAt == "" {
|
|
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
|
}
|
|
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID)
|
|
}
|
|
|
|
func (r *runner) cancel(id int64) error {
|
|
r.mu.Lock()
|
|
entry, ok := r.active[id]
|
|
r.mu.Unlock()
|
|
if ok {
|
|
entry.cancel()
|
|
return nil
|
|
}
|
|
// A run that is waiting on a person has no live goroutine, so cancel it at
|
|
// the persistence layer instead. This keeps cancel working across a restart.
|
|
return r.store.CancelWaiting(r.ctx, id)
|
|
}
|
|
|
|
// decide records an operator's choice for a waiting human stage and resumes the
|
|
// run along the chosen edge. ResolveDecision owns validation and idempotency, so
|
|
// decide does no option parsing of its own. A resumed run is registered and
|
|
// launched exactly like a fresh one but starts from the persisted state.
|
|
func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("decide requires a positive run ID")
|
|
}
|
|
r.mu.Lock()
|
|
detail, err := r.store.GetRun(r.ctx, request.RunID)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if _, busy := r.active[detail.ID]; busy {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
|
|
}
|
|
if detail.Status == protocol.RunStatusWaiting {
|
|
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if applied {
|
|
for i := range detail.Steps {
|
|
if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage {
|
|
detail.Steps[i] = step
|
|
break
|
|
}
|
|
}
|
|
r.launch(detail.ID, detail.Workspace, func(ctx context.Context) {
|
|
r.resume(ctx, detail.ID, detail, detail.Workspace, step)
|
|
})
|
|
}
|
|
r.mu.Unlock()
|
|
return r.getRun(request.RunID)
|
|
}
|
|
|
|
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
|
|
original, copyErr := copyValues(stage.Values(request.Input))
|
|
if copyErr != nil {
|
|
r.finishRunState(id, "failed", "error", copyErr.Error(), stage.Values(request.Input))
|
|
return
|
|
}
|
|
input := stage.Values(request.Input)
|
|
state := newRunState(d.Graph.Stages[0].Key, input)
|
|
state.memory["input"] = memoryValue{output: original}
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
|
|
}
|
|
|
|
// terminalFinish classifies a stage's resolved target. A terminal key yields the
|
|
// run's final status, outcome and diagnostic; an empty target means the stage
|
|
// named an undeclared outcome; any other value is a stage key and is not
|
|
// terminal. resume and the loop tail share it, so terminal routing lives once.
|
|
func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) {
|
|
switch target {
|
|
case protocol.SuccessEnd:
|
|
return "succeeded", "success", "", true
|
|
case protocol.FailureEnd:
|
|
if diagnostic == "" {
|
|
diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome)
|
|
}
|
|
return "failed", "error", diagnostic, true
|
|
case "":
|
|
return "failed", "error", "stage returned an undeclared outcome", true
|
|
}
|
|
return "", "", "", false
|
|
}
|
|
|
|
// resume continues a decided run from the state already persisted for its run
|
|
// detail: memory from the run input and the completed steps, visits counted per
|
|
// stage, and the next sequence above the highest recorded one. It then runs the
|
|
// shared loop from the decision's target; no full-table scan and no extra
|
|
// per-step query are introduced.
|
|
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
|
|
// The decision may route straight to a terminal, which has no stage to run.
|
|
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
|
|
r.finishRunState(id, status, outcome, diagnostic, stage.Values(step.Output))
|
|
return
|
|
}
|
|
state := rebuildRunState(d, step)
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools}, state)
|
|
}
|
|
|
|
// runState is the loop's whole mutable state, extracted so a fresh run and a
|
|
// resumed run share one implementation. nextSequence is set before each visit so
|
|
// a wait that returns to the caller records where the run will continue.
|
|
type runState struct {
|
|
key string
|
|
nextSequence int
|
|
visits map[string]int
|
|
memory map[string]memoryValue
|
|
input stage.Values
|
|
}
|
|
|
|
func newRunState(key string, input stage.Values) *runState {
|
|
return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input}
|
|
}
|
|
|
|
func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState {
|
|
state := newRunState(decided.Target, stage.Values(decided.Output))
|
|
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
|
state.memory["input"] = memoryValue{output: original}
|
|
}
|
|
for _, step := range detail.Steps {
|
|
state.visits[step.Stage]++
|
|
if step.Sequence >= state.nextSequence {
|
|
state.nextSequence = step.Sequence + 1
|
|
}
|
|
if step.Status == "completed" {
|
|
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
}
|
|
}
|
|
return state
|
|
}
|
|
|
|
func (r *runner) finishRunState(id int64, status, outcome, diagnostic string, input stage.Values) {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
|
|
log.Printf("finish run %d: %v", id, err)
|
|
}
|
|
}
|
|
|
|
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
|
|
finish := func(status, outcome, diagnostic string) {
|
|
r.finishRunState(id, status, outcome, diagnostic, state.input)
|
|
}
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
finish("failed", "error", "stage execution panicked")
|
|
}
|
|
}()
|
|
budgets := map[string]*stage.Budget{}
|
|
if state.key == "" {
|
|
finish("failed", "error", "stage returned an undeclared outcome")
|
|
return
|
|
}
|
|
for sequence := state.nextSequence; sequence <= 30; sequence++ {
|
|
state.nextSequence = sequence + 1
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
key := state.key
|
|
var def stage.Definition
|
|
var selected protocol.Stage
|
|
found := false
|
|
for _, s := range graph.Stages {
|
|
if s.Key == key {
|
|
selected = s
|
|
def = pipeline.StageDefinition(s)
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
finish("failed", "error", "missing stage")
|
|
return
|
|
}
|
|
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
|
|
state.visits[key]++
|
|
maxVisits := 5
|
|
if selected.Limits != nil {
|
|
maxVisits = selected.Limits.MaxVisits
|
|
}
|
|
// A human stage pauses for a person; neither the global visit budget nor
|
|
// the per-stage max_visits may kill it. The shared 30-step run budget
|
|
// still bounds every run.
|
|
if !isHuman {
|
|
if state.visits[key] > 5 {
|
|
finish("failed", "error", "stage visit budget exceeded")
|
|
return
|
|
}
|
|
if state.visits[key] > maxVisits {
|
|
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
|
|
return
|
|
}
|
|
}
|
|
var reads map[string]protocol.MemoryRead
|
|
input, reads, copyErr := memoryInput(state.input, selected.Memory, state.memory)
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
if isHuman {
|
|
// A human step is O(1): one BeginStep plus one MarkWaiting, no
|
|
// filediff capture, no artifacts directory and no workspace mutation.
|
|
evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges))
|
|
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence}
|
|
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
|
|
return
|
|
}
|
|
var tools []string
|
|
explicitTools := false
|
|
var toolErr error
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
tools, explicitTools, toolErr = stage.ToolNames(def)
|
|
}
|
|
if toolErr != nil {
|
|
finish("failed", "error", toolErr.Error())
|
|
return
|
|
}
|
|
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
|
|
// Every agent visit gets a per-run/step directory for the files it
|
|
// produces. Creation is best-effort: an unusable directory must never
|
|
// fail the run, and it is only attempted for model stages.
|
|
artifactsDir := ""
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence))
|
|
if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil {
|
|
log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr)
|
|
} else {
|
|
artifactsDir = candidate
|
|
def.ArtifactsDir = candidate
|
|
}
|
|
}
|
|
operations := &stage.OperationLog{}
|
|
client := &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: request.WritePaths, Tools: tools}
|
|
client.OnSession = func(session string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
|
|
}
|
|
var result stage.Result
|
|
var piClient *stage.PiClient
|
|
var err error
|
|
stageCtx := ctx
|
|
var modelConfig stage.ModelConfig
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
spec, specErr := (stage.ModelStage{}).Spec(def)
|
|
if specErr != nil {
|
|
finish("failed", "error", specErr.Error())
|
|
return
|
|
}
|
|
modelConfig = spec.Config
|
|
if budgets[key] == nil {
|
|
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
|
}
|
|
stageCtx = stage.WithBudget(ctx, budgets[key])
|
|
}
|
|
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
|
before := filediff.Capture(snapshotCtx, workspace)
|
|
snapshotCancel()
|
|
var stageSession string
|
|
panicked := false
|
|
func() {
|
|
defer func() {
|
|
if recover() != nil {
|
|
panicked = true
|
|
err = fmt.Errorf("stage execution panicked")
|
|
}
|
|
}()
|
|
if r.execute != nil {
|
|
timeout, _ := stage.StageTimeout(def)
|
|
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
|
|
defer cancel()
|
|
result, err = r.execute(executeCtx, def, input)
|
|
} else if modelConfig.Harness == "pi" {
|
|
configured, configErr := harness.Load()
|
|
if configErr != nil {
|
|
err = configErr
|
|
return
|
|
}
|
|
profile, ok := configured.Profiles["pi"]
|
|
if !ok {
|
|
err = fmt.Errorf("Pi is not configured on execution server")
|
|
return
|
|
}
|
|
piClient = &stage.PiClient{Executable: profile.Executable, Directory: workspace, Tools: tools, WritePaths: request.WritePaths, ArtifactsDir: artifactsDir}
|
|
piClient.OnOwnership = func(session, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
|
|
stageSession = piClient.SessionID
|
|
client.Transcript = piClient.Transcript
|
|
} else if useAgent {
|
|
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
|
|
stageSession = client.SessionID
|
|
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
|
|
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if active {
|
|
stageSession = session
|
|
}
|
|
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
|
|
client.Transcript = plain.FormatAudit
|
|
} else {
|
|
result, err = stage.DefaultRegistry().Run(ctx, def, input)
|
|
}
|
|
}()
|
|
// Capture after cleanup, even when the stage's context was canceled.
|
|
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
after := filediff.Capture(snapshotCtx, workspace, before)
|
|
snapshotCancel()
|
|
changes := filediff.Compare(before, after)
|
|
if errors.Is(err, stage.ErrStageCleanup) {
|
|
changes.Status = "partial"
|
|
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
|
|
}
|
|
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir)
|
|
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
if len(result.Output) > 0 {
|
|
step.Output["salvage"] = result.Output
|
|
}
|
|
}
|
|
// A failed or timed-out step carries the session pointer even when the
|
|
// bounded transcript would otherwise be only its stage_limits envelope.
|
|
if err != nil && stageSession != "" {
|
|
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
|
|
}
|
|
step.Target = def.Edges[step.Outcome]
|
|
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
|
|
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
|
|
for _, outcome := range selected.Limits.On {
|
|
if outcome == step.Outcome {
|
|
step.Control.LimitReached = true
|
|
step.Control.ModelTarget = step.Target
|
|
step.Target = protocol.FailureEnd
|
|
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
|
|
}
|
|
}
|
|
}
|
|
if panicked {
|
|
step.Target = protocol.FailureEnd
|
|
}
|
|
// Generate the one-line verdict reason once, at completion. It is
|
|
// best-effort: a summariser failure or timeout returns NoSummary and must
|
|
// never fail the stage or the run, so every error is swallowed. A failed
|
|
// step still gets a verdict from its error envelope.
|
|
if r.verdict != nil {
|
|
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
|
|
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
|
|
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
|
|
}
|
|
reasonCancel()
|
|
}
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
persistErr := r.store.FinishStep(save, id, step)
|
|
cancel()
|
|
if persistErr != nil {
|
|
finish("failed", "error", persistErr.Error())
|
|
return
|
|
}
|
|
if step.Status == "completed" {
|
|
values, err := copyValues(stage.Values(step.Output))
|
|
if err != nil {
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
state.memory[key] = memoryValue{output: values, sequence: sequence}
|
|
}
|
|
r.notifyStep(id, name, workspace, selected.Notify, step)
|
|
state.input = stage.Values(step.Output)
|
|
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
|
|
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
|
|
if checkErr != nil || len(sessions) > 0 {
|
|
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
|
|
return
|
|
}
|
|
}
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
}
|
|
finish("failed", "error", "total stage budget exceeded")
|
|
}
|
|
|
|
// setTitle renames a run: it replaces the stored operator-facing title without
|
|
// touching the run's input, steps or execution, then returns the updated detail.
|
|
func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) {
|
|
explicit, ok := summarise.ExplicitTitle(title)
|
|
if !ok {
|
|
return nil, fmt.Errorf("a title with usable text is required")
|
|
}
|
|
if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil {
|
|
return nil, err
|
|
}
|
|
return r.getRun(id)
|
|
}
|