31aec05f99
Derive validated same-run visit artifact capabilities in the runner and pass them to OpenCode and Pi clients. Permit supported file tools to read completed prior visits and write only the current visit, preserving workspace, secret, database, and shell restrictions.\n\nChecked packaged templates and authoring skill references: neither needs updating in this item; docs/notes/scoped-artifact-access.md records the shared documentation staleness for the docs-fold writer.
1309 lines
51 KiB
Go
1309 lines
51 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/filediff"
|
|
"git.a-wan.net/alan/pipelines/internal/harness"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
// activeRun records one in-flight run's cancellation seam and the resolved
|
|
// workspace it is changing. A single map is the only active-run state, so the
|
|
// workspace cannot drift from the cancel function.
|
|
type activeRun struct {
|
|
cancel context.CancelFunc
|
|
workspace string
|
|
}
|
|
|
|
type runner struct {
|
|
ctx context.Context
|
|
store *pipeline.Store
|
|
artifactsRoot string
|
|
mu sync.Mutex
|
|
active map[int64]activeRun
|
|
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
|
|
// title generates the operator-facing run title from the run prompt. When nil
|
|
// the store derives a fallback from the prompt with no model call, so tests
|
|
// and unconfigured servers stay deterministic.
|
|
title func(context.Context, string) summarise.Title
|
|
// verdict generates the bounded one-line reason for a completed stage visit
|
|
// from its output map. When nil the store derives the reason at read time with
|
|
// no model call. It is best-effort: any failure must never fail the stage.
|
|
verdict func(context.Context, map[string]any) string
|
|
// events pushes run state deltas to subscribed clients. When nil (tests that
|
|
// build a runner directly) emits are dropped, so nothing else changes.
|
|
events *runEventHub
|
|
// identMu guards idents: the pipeline identity and stored title (see
|
|
// runIdentity) that every emitted frame of a run carries, so a client with no
|
|
// cached row renders a brand-new run without re-reading the list. Populated
|
|
// where a run is created, decided or canceled; a run with no remembered
|
|
// identity keeps blank identity fields and the client resyncs once instead.
|
|
identMu sync.Mutex
|
|
idents map[int64]runIdentity
|
|
// exportSession writes a finished stage's session into its own artifacts
|
|
// directory. It defaults to stage.ExportSession; a failure is best-effort and
|
|
// never changes the stage's status, outcome, target or result validation.
|
|
exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error)
|
|
wg sync.WaitGroup
|
|
schedulerWG sync.WaitGroup
|
|
// attentionMu guards attentionSent, the per-run set of run-attention events
|
|
// already delivered — by a stage's own notify policy or by the run-level
|
|
// default — so neither republishes the same genuine event.
|
|
attentionMu sync.Mutex
|
|
attentionSent map[int64]map[string]bool
|
|
}
|
|
|
|
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
|
sessions, err := s.ActiveSessionOwners(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, owner := range sessions {
|
|
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
var err error
|
|
if owner.Backend == "pi" {
|
|
err = stage.InterruptPi(stop, owner.Reference)
|
|
} else {
|
|
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).Interrupt(stop, owner.ID)
|
|
}
|
|
cancel()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
|
}
|
|
}
|
|
if err := s.RecoverRuns(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
root := filepath.Join(s.Directory(), "artifacts")
|
|
return &runner{ctx: ctx, store: s, artifactsRoot: root, active: map[int64]activeRun{}, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, exportSession: stage.ExportSession}, nil
|
|
}
|
|
|
|
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
|
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
|
return nil, fmt.Errorf("an explicit version and request key are required")
|
|
}
|
|
workspace, err := os.Getwd()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
workspace, err = filepath.EvalSymlinks(workspace)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if request.Workspace != "" {
|
|
if !filepath.IsAbs(request.Workspace) {
|
|
return nil, fmt.Errorf("run workspace must be an absolute directory")
|
|
}
|
|
selected, err := filepath.EvalSymlinks(request.Workspace)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("run workspace is unavailable on the execution server: %q: %w; for Docker, mount this host directory at the same absolute path and use its owner's UID/GID", request.Workspace, err)
|
|
}
|
|
info, err := os.Stat(selected)
|
|
if err != nil || !info.IsDir() {
|
|
return nil, fmt.Errorf("run workspace %q must be an accessible directory on the execution server", request.Workspace)
|
|
}
|
|
workspace = selected
|
|
}
|
|
if len(request.WritePaths) > 20 {
|
|
return nil, fmt.Errorf("too many write paths")
|
|
}
|
|
for _, path := range request.WritePaths {
|
|
if filepath.IsAbs(path) || filepath.Clean(path) != path || path == ".." || strings.HasPrefix(path, ".."+string(os.PathSeparator)) || strings.ContainsAny(path, "*?\\") {
|
|
return nil, fmt.Errorf("optional write paths must be exact run-directory-relative paths")
|
|
}
|
|
parent := filepath.Join(workspace, path)
|
|
for {
|
|
_, err := os.Lstat(parent)
|
|
if err == nil {
|
|
break
|
|
}
|
|
if !os.IsNotExist(err) {
|
|
return nil, err
|
|
}
|
|
next := filepath.Dir(parent)
|
|
if next == parent {
|
|
return nil, fmt.Errorf("write path is unavailable")
|
|
}
|
|
parent = next
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(parent)
|
|
if err != nil || (resolved != workspace && !strings.HasPrefix(resolved, workspace+string(os.PathSeparator))) {
|
|
return nil, fmt.Errorf("write path must stay within the run directory")
|
|
}
|
|
}
|
|
request.Input = maps.Clone(request.Input)
|
|
if request.Input == nil {
|
|
request.Input = map[string]any{}
|
|
}
|
|
request.Input["pwd"] = workspace
|
|
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
|
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
|
}
|
|
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
|
|
return nil, err
|
|
}
|
|
// Validate the launch-time model overrides against the pinned graph before
|
|
// admission: an unknown stage key or a malformed provider/model must never
|
|
// insert a run row. The resolution is O(#overrides + #stages) with no model or
|
|
// network call, and the same helper is the authoritative precedence rule.
|
|
if _, err := pipeline.EffectiveModels(d.Graph, request.Model, request.StageModels); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["harness"] == "pi" && r.execute == nil {
|
|
configured, err := harness.Load()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, ok := configured.Profiles["pi"]; !ok {
|
|
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
|
|
}
|
|
}
|
|
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !explicit {
|
|
request.AllowTools = true
|
|
}
|
|
for _, tool := range tools {
|
|
if tool == "edit" || tool == "shell" {
|
|
request.AllowTools = true
|
|
}
|
|
}
|
|
}
|
|
// Decide admission and idempotency under the lock, but generate the title
|
|
// only for a request that will actually start a run: a rejected or replayed
|
|
// request must not pay for a bounded model generation, and a live one must
|
|
// not hold the lock (stalling listings and cancellation) while a model is
|
|
// called. An unavailable or slow model falls back to the prompt's first line.
|
|
r.mu.Lock()
|
|
if r.ctx.Err() != nil {
|
|
r.mu.Unlock()
|
|
return nil, r.ctx.Err()
|
|
}
|
|
owners, err := r.store.ActiveSessionOwners(r.ctx)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
// Refuse only an orphaned session: one whose owning run is not active now.
|
|
// A session owned by a currently active run (another run in another
|
|
// workspace) must not block new work.
|
|
for _, owner := range owners {
|
|
if _, ok := r.active[owner.RunID]; !ok {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
|
}
|
|
}
|
|
// Idempotent repeats are accepted even while the original run is active, and
|
|
// a replay reuses the stored title instead of generating a new one.
|
|
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
var runTitle summarise.Title
|
|
if existing == 0 {
|
|
// An explicit operator title skips the model call; otherwise the
|
|
// configured generator names the run, falling back to the prompt's
|
|
// first line when it is unavailable or slow.
|
|
if explicit, ok := summarise.ExplicitTitle(request.Title); ok {
|
|
runTitle = explicit
|
|
} else if r.title != nil {
|
|
prompt, _ := request.Input["prompt"].(string)
|
|
runTitle = r.title(r.ctx, prompt)
|
|
}
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
// Another run may have reserved this workspace while the title was
|
|
// generated, so re-check before reserving it; a race on the request key is
|
|
// handled by the store's idempotent insert.
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Model: request.Model, StageModels: request.StageModels}, runTitle, retryOf...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// The run row now exists: read it once for its resolved identity (pipeline
|
|
// id/name, the store-derived title, the pinned version), so the queued frame
|
|
// and every later frame carry what a client needs to render a brand-new run
|
|
// without re-reading the list. A failed read only costs that seed: the run is
|
|
// still launched below and its frames carry no identity, so the client falls
|
|
// back to its one resync.
|
|
detail, readErr := r.store.GetRun(r.ctx, id)
|
|
if readErr != nil {
|
|
detail = nil
|
|
} else if detail != nil {
|
|
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
|
}
|
|
if created {
|
|
// Emit the created delta before launching: the queued frame reaches
|
|
// subscribers before the running/step-started transition, so a client
|
|
// sees the run appear and then advance.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusQueued,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}, false)
|
|
r.launch(id, workspace, func(ctx context.Context) {
|
|
r.run(ctx, id, d, workspace, request)
|
|
})
|
|
}
|
|
if detail != nil {
|
|
_, detail.Executing = r.active[id]
|
|
r.overlayEffectiveModels(detail)
|
|
}
|
|
if readErr != nil {
|
|
return nil, readErr
|
|
}
|
|
return detail, nil
|
|
}
|
|
|
|
// launch registers an active run's cancellation seam and workspace, adds it to
|
|
// the wait group and starts fn on a derived cancellable context that cleans up
|
|
// the registration when it returns. A fresh run and a resumed decision both go
|
|
// through it, so registration and teardown live once. Callers must hold r.mu.
|
|
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
|
ctx, cancel := context.WithCancel(r.ctx)
|
|
r.active[id] = activeRun{cancel: cancel, workspace: workspace}
|
|
r.wg.Add(1)
|
|
go func() {
|
|
defer r.wg.Done()
|
|
defer cancel()
|
|
defer func() { r.mu.Lock(); delete(r.active, id); r.mu.Unlock() }()
|
|
fn(ctx)
|
|
}()
|
|
}
|
|
|
|
// checkWorkspaceFree refuses a run when an active run is already changing the
|
|
// same resolved workspace. The workspace is the only scope a run reserves, and
|
|
// one active run per workspace keeps this scan finding at most one match, so the
|
|
// message is deterministic. Callers must hold r.mu.
|
|
func (r *runner) checkWorkspaceFree(workspace string) error {
|
|
for id, entry := range r.active {
|
|
if entry.workspace == workspace {
|
|
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
d, err := r.store.GetRun(r.ctx, id)
|
|
if d != nil {
|
|
_, d.Executing = r.active[id]
|
|
for _, s := range d.Steps {
|
|
if s.AgentActive {
|
|
d.Executing = true
|
|
}
|
|
}
|
|
r.overlayEffectiveModels(d)
|
|
}
|
|
return d, err
|
|
}
|
|
|
|
// overlayEffectiveModels replaces each model stage's provider/model in a returned
|
|
// run's graph with the effective pair the run actually uses, so the app and TUI
|
|
// stage-definition surfaces show the override with no client change. It is
|
|
// presentation only: the persisted graph_json and the pinned source returned by
|
|
// `show` are never rewritten. A run with no overrides is left untouched.
|
|
func (r *runner) overlayEffectiveModels(d *protocol.RunDetail) {
|
|
if d == nil || (d.Model == "" && len(d.StageModels) == 0) {
|
|
return
|
|
}
|
|
effective, err := pipeline.EffectiveModels(d.Graph, d.Model, d.StageModels)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for i := range d.Graph.Stages {
|
|
s := &d.Graph.Stages[i]
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
override, ok := effective[s.Key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if s.Configs == nil {
|
|
s.Configs = map[string]string{}
|
|
}
|
|
s.Configs["provider"] = override.Provider
|
|
s.Configs["model"] = override.Model
|
|
}
|
|
}
|
|
|
|
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
items, err := r.store.ListRuns(r.ctx)
|
|
for i := range items {
|
|
_, items[i].Executing = r.active[items[i].ID]
|
|
if items[i].Status == "failed" && !items[i].Executing {
|
|
d, err := r.store.GetRun(r.ctx, items[i].ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, step := range d.Steps {
|
|
if step.AgentActive {
|
|
items[i].Executing = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items, err
|
|
}
|
|
|
|
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("retry requires a positive run ID")
|
|
}
|
|
original, err := r.getRun(request.RunID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if original.Executing {
|
|
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
|
|
}
|
|
if original.Status != "failed" || original.FinishedAt == "" {
|
|
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
|
}
|
|
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels}, original.ID)
|
|
}
|
|
|
|
func (r *runner) cancel(id int64) error {
|
|
r.mu.Lock()
|
|
entry, ok := r.active[id]
|
|
r.mu.Unlock()
|
|
if ok {
|
|
entry.cancel()
|
|
return nil
|
|
}
|
|
// A run that is waiting on a person has no live goroutine, so cancel it at
|
|
// the persistence layer instead. This keeps cancel working across a restart.
|
|
// The single follow-up read seeds the terminal delta; cancel is a cold
|
|
// operator action, not a per-transition hot path.
|
|
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
|
|
return err
|
|
}
|
|
// A canceled run is terminal: nothing more can notify for it.
|
|
r.forgetAttention(id)
|
|
if r.events != nil {
|
|
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
|
|
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: detail.Workspace,
|
|
Status: detail.Status,
|
|
Outcome: detail.Outcome,
|
|
CurrentStage: detail.CurrentStage,
|
|
CompletedSteps: detail.CompletedSteps,
|
|
FinishedAt: detail.FinishedAt,
|
|
}, true)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// decide records an operator's choice for a waiting human stage and resumes the
|
|
// run along the chosen edge. ResolveDecision owns validation and idempotency, so
|
|
// decide does no option parsing of its own. A resumed run is registered and
|
|
// launched exactly like a fresh one but starts from the persisted state.
|
|
func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("decide requires a positive run ID")
|
|
}
|
|
r.mu.Lock()
|
|
detail, err := r.store.GetRun(r.ctx, request.RunID)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if _, busy := r.active[detail.ID]; busy {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
|
|
}
|
|
if detail.Status == protocol.RunStatusWaiting {
|
|
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if applied {
|
|
// Re-seed the identity so the resumed frames (this one and the resumed
|
|
// loop's) carry it even after a restart, when no creation was observed.
|
|
r.rememberRunIdentity(detail.ID, identityOfRunSummary(detail.RunSummary))
|
|
for i := range detail.Steps {
|
|
if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage {
|
|
detail.Steps[i] = step
|
|
break
|
|
}
|
|
}
|
|
// The decision closes the waiting step and returns the run to running;
|
|
// report that before the resumed loop emits its own deltas.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: detail.ID,
|
|
Workspace: detail.Workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: step.Stage,
|
|
CompletedSteps: completedSteps(detail.Steps),
|
|
Step: &protocol.RunStepDelta{
|
|
Sequence: step.Sequence,
|
|
Stage: step.Stage,
|
|
Status: step.Status,
|
|
Outcome: step.Outcome,
|
|
Target: step.Target,
|
|
Diagnostic: step.Diagnostic,
|
|
},
|
|
}, false)
|
|
r.launch(detail.ID, detail.Workspace, func(ctx context.Context) {
|
|
r.resume(ctx, detail.ID, detail, detail.Workspace, step)
|
|
})
|
|
}
|
|
r.mu.Unlock()
|
|
return r.getRun(request.RunID)
|
|
}
|
|
|
|
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
|
|
original, copyErr := copyValues(stage.Values(request.Input))
|
|
if copyErr != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
|
|
return
|
|
}
|
|
input := stage.Values(request.Input)
|
|
state := newRunState(d.Graph.Stages[0].Key, input)
|
|
state.memory["input"] = memoryValue{output: original}
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
|
|
}
|
|
|
|
// terminalFinish classifies a stage's resolved target. A terminal key yields the
|
|
// run's final status, outcome and diagnostic; an empty target means the stage
|
|
// named an undeclared outcome; any other value is a stage key and is not
|
|
// terminal. resume and the loop tail share it, so terminal routing lives once.
|
|
func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) {
|
|
switch target {
|
|
case protocol.SuccessEnd:
|
|
return "succeeded", "success", "", true
|
|
case protocol.FailureEnd:
|
|
if diagnostic == "" {
|
|
diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome)
|
|
}
|
|
return "failed", "error", diagnostic, true
|
|
case "":
|
|
return "failed", "error", "stage returned an undeclared outcome", true
|
|
}
|
|
return "", "", "", false
|
|
}
|
|
|
|
// resume continues a decided run from the state already persisted for its run
|
|
// detail: memory from the run input and the completed steps, visits counted per
|
|
// stage, and the next sequence above the highest recorded one. It then runs the
|
|
// shared loop from the decision's target; no full-table scan and no extra
|
|
// per-step query are introduced.
|
|
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
|
|
// A resumed run's frames carry the same identity as its creation, so re-seed
|
|
// it here too: resume is also reachable from a store read after a restart.
|
|
r.rememberRunIdentity(id, identityOfRunSummary(d.RunSummary))
|
|
// The decision may route straight to a terminal, which has no stage to run.
|
|
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
|
|
r.finishRunState(id, d.Name, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Sequence, step.Stage, stage.Values(step.Output))
|
|
return
|
|
}
|
|
state := rebuildRunState(d, step)
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools, Model: d.Model, StageModels: d.StageModels}, state)
|
|
}
|
|
|
|
// runState is the loop's whole mutable state, extracted so a fresh run and a
|
|
// resumed run share one implementation. nextSequence is set before each visit so
|
|
// a wait that returns to the caller records where the run will continue.
|
|
type runState struct {
|
|
key string
|
|
nextSequence int
|
|
visits map[string]int
|
|
memory map[string]memoryValue
|
|
input stage.Values
|
|
// changedPaths retains a bounded sample of the run's recorded changed paths
|
|
// and changedCount the full total, so the skip guard reads only in-memory
|
|
// evidence. uiSurface is latched once any recorded change touched a UI
|
|
// surface. Both are folded in by recordChanges (ui_surface.go).
|
|
changedPaths []string
|
|
changedCount int
|
|
uiSurface bool
|
|
// capturedPaths is the full, deduplicated, first-seen-ordered set of the
|
|
// run's server-captured changed paths, distinct from the bounded
|
|
// changedPaths sample used only by the skip reason. capturedSeen is the
|
|
// single dedupe set. Both are folded in by recordChanges (ui_surface.go) and
|
|
// published to a selecting stage as input.server_changed_files.
|
|
capturedPaths []string
|
|
capturedSeen map[string]bool
|
|
// completed counts the run's finished steps, so a delta can report the
|
|
// running total without re-reading the store.
|
|
completed int
|
|
artifactDirs []string
|
|
}
|
|
|
|
func newRunState(key string, input stage.Values) *runState {
|
|
return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input}
|
|
}
|
|
|
|
func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState {
|
|
state := newRunState(decided.Target, stage.Values(decided.Output))
|
|
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
|
state.memory["input"] = memoryValue{output: original}
|
|
}
|
|
for _, step := range detail.Steps {
|
|
state.visits[step.Stage]++
|
|
if step.Sequence >= state.nextSequence {
|
|
state.nextSequence = step.Sequence + 1
|
|
}
|
|
if step.Status == "completed" {
|
|
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
state.completed++
|
|
}
|
|
// Seed the UI-surface accumulator from the persisted evidence so a
|
|
// resumed run reaches the same skip decision with no extra query.
|
|
state.recordChanges(step.FileChanges, step.Artifacts)
|
|
// Only completed, recorded visits are eligible for a later stage's
|
|
// review capability. The path is reconstructed by the runner below.
|
|
if step.Status == "completed" {
|
|
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
|
}
|
|
}
|
|
return state
|
|
}
|
|
|
|
// artifactAccess derives exact visit directories; it never consumes a path from
|
|
// a definition or model result. An unreadable or symlinked root fails closed.
|
|
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
|
|
root, err := filepath.EvalSymlinks(r.artifactsRoot)
|
|
if err != nil {
|
|
return stage.ArtifactAccess{}
|
|
}
|
|
runRoot := filepath.Join(root, strconv.FormatInt(runID, 10))
|
|
visitDirectory := func(visit int) (string, bool) {
|
|
if visit <= 0 {
|
|
return "", false
|
|
}
|
|
directory := filepath.Join(runRoot, strconv.Itoa(visit))
|
|
resolved, err := filepath.EvalSymlinks(directory)
|
|
if err != nil || resolved != directory {
|
|
return "", false
|
|
}
|
|
info, err := os.Stat(resolved)
|
|
if err != nil || !info.IsDir() {
|
|
return "", false
|
|
}
|
|
return resolved, true
|
|
}
|
|
access := stage.ArtifactAccess{}
|
|
if directory, ok := visitDirectory(sequence); ok {
|
|
resolved, err := filepath.EvalSymlinks(current)
|
|
if err != nil || resolved != directory {
|
|
return access
|
|
}
|
|
access.Write = directory
|
|
access.Read = append(access.Read, directory)
|
|
}
|
|
for _, priorSequence := range prior {
|
|
visit, err := strconv.Atoi(priorSequence)
|
|
if err != nil || visit <= 0 || visit >= sequence {
|
|
continue
|
|
}
|
|
if directory, ok := visitDirectory(visit); ok {
|
|
access.Read = append(access.Read, directory)
|
|
}
|
|
}
|
|
return access
|
|
}
|
|
|
|
func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient {
|
|
return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
|
|
}
|
|
|
|
func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient {
|
|
return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
|
|
}
|
|
|
|
// serverChangedFilesField is the reserved run-input memory field under which the
|
|
// runner publishes the server-captured changed paths. A stage selects it with
|
|
// `"server_changed_files":"input.server_changed_files"`, alongside another
|
|
// stage's declared list, so it sees the verified evidence, not just a model's
|
|
// claim of authorship.
|
|
const serverChangedFilesField = "server_changed_files"
|
|
|
|
// capturedList returns the full, first-seen-ordered set of server-captured
|
|
// changed paths, or a non-nil empty list when nothing was captured so the field
|
|
// still resolves as a list. It copies the accumulator so the retained evidence
|
|
// cannot be aliased by a published value. O(captured paths).
|
|
func (s *runState) capturedList() []string {
|
|
if len(s.capturedPaths) == 0 {
|
|
return []string{}
|
|
}
|
|
return append([]string(nil), s.capturedPaths...)
|
|
}
|
|
|
|
// publishServerChangedFiles writes the run's captured changed-path set into the
|
|
// run input memory value under serverChangedFilesField, creating the entry and
|
|
// its output map when absent. Called before memoryInput so a stage selecting
|
|
// input.server_changed_files always resolves; only that field is added and only
|
|
// a selecting stage copies it out, so state.input and other stages' input are
|
|
// unchanged. O(captured paths).
|
|
func (s *runState) publishServerChangedFiles() {
|
|
entry, ok := s.memory["input"]
|
|
if !ok || entry.output == nil {
|
|
entry = memoryValue{output: stage.Values{}}
|
|
}
|
|
entry.output[serverChangedFilesField] = s.capturedList()
|
|
s.memory["input"] = entry
|
|
}
|
|
|
|
// runIdentity is the pipeline identity and stored title a run's pushed frames
|
|
// carry (RunEventFrame.PipelineID/Name/Title/Version). It is remembered where a
|
|
// run is created, decided or canceled and stamped onto every frame that leaves
|
|
// emitRun, so a client that has never seen the run can build its list row from
|
|
// the frame without re-reading the whole list.
|
|
type runIdentity struct {
|
|
pipelineID int64
|
|
name string
|
|
title string
|
|
version int
|
|
}
|
|
|
|
// identityOfRunSummary is the one projection of a run row onto the identity its
|
|
// pushed frames carry, so the frame and the list row cannot disagree.
|
|
func identityOfRunSummary(s protocol.RunSummary) runIdentity {
|
|
return runIdentity{pipelineID: s.PipelineID, name: s.Name, title: s.Title, version: s.Version}
|
|
}
|
|
|
|
// rememberRunIdentity records the identity every later frame of run id carries.
|
|
// It is safe from any goroutine and takes its own lock (never r.mu), so it is
|
|
// callable from a path that already holds r.mu.
|
|
func (r *runner) rememberRunIdentity(id int64, ident runIdentity) {
|
|
if r.idents == nil {
|
|
return
|
|
}
|
|
r.identMu.Lock()
|
|
r.idents[id] = ident
|
|
r.identMu.Unlock()
|
|
}
|
|
|
|
// stampRunIdentity fills a frame's blank identity fields from the run's
|
|
// remembered identity, leaving any the emit site set untouched. A run with no
|
|
// remembered identity keeps them blank, so the client falls back to its resync.
|
|
func (r *runner) stampRunIdentity(frame *protocol.RunEventFrame) {
|
|
r.identMu.Lock()
|
|
ident, ok := r.idents[frame.RunID]
|
|
r.identMu.Unlock()
|
|
if !ok {
|
|
return
|
|
}
|
|
if frame.PipelineID == 0 {
|
|
frame.PipelineID = ident.pipelineID
|
|
}
|
|
if frame.Name == "" {
|
|
frame.Name = ident.name
|
|
}
|
|
if frame.Title == "" {
|
|
frame.Title = ident.title
|
|
}
|
|
if frame.Version == 0 {
|
|
frame.Version = ident.version
|
|
}
|
|
}
|
|
|
|
// emitRun pushes one run delta to the subscribed-client hub when one is wired,
|
|
// and does nothing otherwise (a runner built directly in a test has no hub).
|
|
// terminal marks a final transition, which the hub flushes without coalescing.
|
|
func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
|
|
if r.events == nil {
|
|
return
|
|
}
|
|
r.stampRunIdentity(&frame)
|
|
r.events.emit(frame, terminal)
|
|
}
|
|
|
|
func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
|
|
log.Printf("finish run %d: %v", id, err)
|
|
}
|
|
// Push the terminal delta before the attention notification, so a slow ntfy
|
|
// never delays the live update the app folds.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: status,
|
|
Outcome: outcome,
|
|
CurrentStage: currentStage,
|
|
CompletedSteps: completed,
|
|
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}, true)
|
|
r.notifyRun(id, name, status, sequence, currentStage, diagnostic)
|
|
// The run is terminal: nothing more can notify for it.
|
|
r.forgetAttention(id)
|
|
}
|
|
|
|
// completedSteps counts the completed steps of a run detail; resume uses it to
|
|
// seed the terminal delta without another store read.
|
|
func completedSteps(steps []protocol.RunStep) int {
|
|
n := 0
|
|
for _, step := range steps {
|
|
if step.Status == "completed" {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// stageUsage carries the usage a model stage already measured for budget
|
|
// accounting onto the run record. It reports a cost only when the provider
|
|
// reported one and an in/out split only when the harness reported it, so an
|
|
// unknown cost or split is never presented as zero. It returns nil when no usage
|
|
// was observed, so the step stays absent rather than zero.
|
|
func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage {
|
|
if !snapshot.TokensKnown && !snapshot.USDKnown {
|
|
return nil
|
|
}
|
|
usage := &protocol.StageUsage{TokensKnown: snapshot.TokensKnown, CostKnown: snapshot.USDKnown}
|
|
if snapshot.TokensKnown {
|
|
usage.TokensTotal = snapshot.UsedTokens
|
|
if snapshot.SplitKnown {
|
|
usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true
|
|
}
|
|
}
|
|
if snapshot.USDKnown {
|
|
usage.CostUSD = snapshot.UsedUSD
|
|
}
|
|
return usage
|
|
}
|
|
|
|
// finishStage persists one step, refreshes the shared memory for a completed
|
|
// step, notifies and classifies the step's target. Both the executed path and
|
|
// the skipped path use it, so persistence, notification and terminal routing
|
|
// cannot drift between them. A persist or memory-copy failure is returned for
|
|
// the caller to fail the run; the caller still owns the model-specific
|
|
// verdict, quiescence and cancellation handling around it.
|
|
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
persistErr := r.store.FinishStep(save, id, step)
|
|
cancel()
|
|
if persistErr != nil {
|
|
return "", "", "", false, persistErr
|
|
}
|
|
if step.Status == "completed" {
|
|
values, copyErr := copyValues(stage.Values(step.Output))
|
|
if copyErr != nil {
|
|
return "", "", "", false, copyErr
|
|
}
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
r.notifyStep(id, name, workspace, selected.Notify, step)
|
|
state.input = stage.Values(step.Output)
|
|
status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
return status, outcome, diagnostic, terminal, nil
|
|
}
|
|
|
|
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
|
|
finish := func(status, outcome, diagnostic string) {
|
|
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
|
|
}
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
finish("failed", "error", "stage execution panicked")
|
|
}
|
|
}()
|
|
budgets := map[string]*stage.Budget{}
|
|
if state.key == "" {
|
|
finish("failed", "error", "stage returned an undeclared outcome")
|
|
return
|
|
}
|
|
// Resolve the effective provider/model for every model stage once, before the
|
|
// loop, so applying an override is an in-memory O(1) map lookup per visit and
|
|
// never re-parses the graph or re-resolves the whole definition. A run with no
|
|
// overrides skips the resolution entirely. Launch validation already ran this
|
|
// helper, so a failure here only guards a resumed run whose overrides drifted.
|
|
var effectiveModels map[string]pipeline.ModelOverride
|
|
if request.Model != "" || len(request.StageModels) > 0 {
|
|
resolved, err := pipeline.EffectiveModels(graph, request.Model, request.StageModels)
|
|
if err != nil {
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
effectiveModels = resolved
|
|
}
|
|
for sequence := state.nextSequence; sequence <= 30; sequence++ {
|
|
state.nextSequence = sequence + 1
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
key := state.key
|
|
var def stage.Definition
|
|
var selected protocol.Stage
|
|
found := false
|
|
for _, s := range graph.Stages {
|
|
if s.Key == key {
|
|
selected = s
|
|
def = pipeline.StageDefinition(s)
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
finish("failed", "error", "missing stage")
|
|
return
|
|
}
|
|
// StageDefinition clones Configs, so setting the effective pair on the
|
|
// clone is safe and the pinned graph is never mutated. Because
|
|
// ModelStage.Spec reads def.Configs and passes spec.Config into the repair
|
|
// path, the same effective model governs the stage and its repair.
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
if override, ok := effectiveModels[key]; ok {
|
|
def.Configs["provider"] = override.Provider
|
|
def.Configs["model"] = override.Model
|
|
}
|
|
}
|
|
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
|
|
state.visits[key]++
|
|
maxVisits := 5
|
|
if selected.Limits != nil {
|
|
maxVisits = selected.Limits.MaxVisits
|
|
}
|
|
// A human stage pauses for a person; neither the global visit budget nor
|
|
// the per-stage max_visits may kill it. The shared 30-step run budget
|
|
// still bounds every run.
|
|
if !isHuman {
|
|
if state.visits[key] > 5 {
|
|
finish("failed", "error", "stage visit budget exceeded")
|
|
return
|
|
}
|
|
if state.visits[key] > maxVisits {
|
|
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
|
|
return
|
|
}
|
|
}
|
|
// Publish the run's server-captured changed paths into the input memory
|
|
// value before resolving bindings, so a stage selecting
|
|
// input.server_changed_files always resolves (an empty list when nothing
|
|
// was captured). Only a selecting stage copies it into its input.
|
|
state.publishServerChangedFiles()
|
|
var reads map[string]protocol.MemoryRead
|
|
input, reads, copyErr := memoryInput(state.input, selected.Memory, state.memory)
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
// queued -> running: report the new current stage and the step start.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
|
|
}, false)
|
|
if isHuman {
|
|
// A human step is O(1): one BeginStep plus one MarkWaiting, no
|
|
// filediff capture, no artifacts directory and no workspace mutation.
|
|
evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges))
|
|
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence}
|
|
// A human step parks the run without a live goroutine; the delta is
|
|
// coalesced by the hub's timer, which keeps running after this returns.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusWaiting,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
|
|
}, false)
|
|
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
|
|
r.notifyRun(id, name, protocol.RunStatusWaiting, sequence, key, "")
|
|
return
|
|
}
|
|
// A stage that declares a `skipped` route is short-circuited when the run
|
|
// has recorded no UI surface: the stage would answer exactly that, so skip
|
|
// the model call and its capture work, record the step as completed and
|
|
// route along the declared edge. Only a declared route is honoured, and no
|
|
// other stage or outcome routing changes.
|
|
if _, declaresSkip := def.Edges["skipped"]; declaresSkip && !state.uiSurface {
|
|
reason := skipReasonLine(state)
|
|
step := protocol.RunStep{
|
|
Sequence: sequence,
|
|
Stage: key,
|
|
Status: "completed",
|
|
Outcome: "skipped",
|
|
Output: stage.Values{"summary": reason},
|
|
Target: def.Edges["skipped"],
|
|
Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads},
|
|
}
|
|
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
|
|
if finishErr != nil {
|
|
finish("failed", "error", finishErr.Error())
|
|
return
|
|
}
|
|
if terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
continue
|
|
}
|
|
var tools []string
|
|
explicitTools := false
|
|
var toolErr error
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
tools, explicitTools, toolErr = stage.ToolNames(def)
|
|
}
|
|
if toolErr != nil {
|
|
finish("failed", "error", toolErr.Error())
|
|
return
|
|
}
|
|
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
|
|
// Every agent visit gets a per-run/step directory for the files it
|
|
// produces. Creation is best-effort: an unusable directory must never
|
|
// fail the run, and it is only attempted for model stages.
|
|
artifactsDir := ""
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
candidate := filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10), strconv.Itoa(sequence))
|
|
if mkErr := os.MkdirAll(candidate, 0700); mkErr != nil {
|
|
log.Printf("stage artifacts directory run=%d step=%d: %v", id, sequence, mkErr)
|
|
} else {
|
|
artifactsDir = candidate
|
|
def.ArtifactsDir = candidate
|
|
}
|
|
}
|
|
operations := &stage.OperationLog{}
|
|
client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools)
|
|
client.OnSession = func(session string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
|
|
}
|
|
var result stage.Result
|
|
var piClient *stage.PiClient
|
|
var err error
|
|
stageCtx := ctx
|
|
var modelConfig stage.ModelConfig
|
|
var budgetBefore stage.BudgetSnapshot
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
spec, specErr := (stage.ModelStage{}).Spec(def)
|
|
if specErr != nil {
|
|
finish("failed", "error", specErr.Error())
|
|
return
|
|
}
|
|
modelConfig = spec.Config
|
|
if budgets[key] == nil {
|
|
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
|
}
|
|
// The budget is shared across every visit to this stage key, so capture
|
|
// its cumulative usage before the visit and record only this visit's
|
|
// increment on the step, never the cumulative total again.
|
|
budgetBefore = budgets[key].Snapshot()
|
|
stageCtx = stage.WithBudget(ctx, budgets[key])
|
|
}
|
|
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
|
before := filediff.Capture(snapshotCtx, workspace)
|
|
snapshotCancel()
|
|
var stageSession string
|
|
panicked := false
|
|
func() {
|
|
defer func() {
|
|
if recover() != nil {
|
|
panicked = true
|
|
err = fmt.Errorf("stage execution panicked")
|
|
}
|
|
}()
|
|
if r.execute != nil {
|
|
timeout, _ := stage.StageTimeout(def)
|
|
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
|
|
defer cancel()
|
|
result, err = r.execute(executeCtx, def, input)
|
|
} else if modelConfig.Harness == "pi" {
|
|
configured, configErr := harness.Load()
|
|
if configErr != nil {
|
|
err = configErr
|
|
return
|
|
}
|
|
profile, ok := configured.Profiles["pi"]
|
|
if !ok {
|
|
err = fmt.Errorf("Pi is not configured on execution server")
|
|
return
|
|
}
|
|
piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools)
|
|
piClient.OnOwnership = func(session, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
|
|
stageSession = piClient.SessionID
|
|
client.Transcript = piClient.Transcript
|
|
} else if useAgent {
|
|
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
|
|
stageSession = client.SessionID
|
|
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
|
|
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if active {
|
|
stageSession = session
|
|
}
|
|
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
|
|
client.Transcript = plain.FormatAudit
|
|
} else {
|
|
result, err = stage.DefaultRegistry().Run(ctx, def, input)
|
|
}
|
|
}()
|
|
// Capture after cleanup, even when the stage's context was canceled.
|
|
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
after := filediff.Capture(snapshotCtx, workspace, before)
|
|
snapshotCancel()
|
|
changes := filediff.Compare(before, after)
|
|
if errors.Is(err, stage.ErrStageCleanup) {
|
|
changes.Status = "partial"
|
|
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
|
|
}
|
|
// Export the stage's session into its own artifacts directory before the
|
|
// bounded scan, so the existing listing and serving paths find the files
|
|
// with no special casing. This is bounded, best-effort evidence: it never
|
|
// waits for session idleness and a failure never changes the stage's
|
|
// status, outcome, target or result validation.
|
|
var sessionExport *protocol.SessionExport
|
|
if artifactsDir != "" {
|
|
if owner, ownerErr := r.store.RunSessionOwner(ctx, id, sequence); ownerErr == nil && owner.ID != "" {
|
|
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
|
|
}
|
|
}
|
|
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir)
|
|
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, SessionExport: sessionExport, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
|
|
// Stamp the effective provider/model this visit actually ran with for every
|
|
// model stage, including failures, so the app/TUI evidence reflects any
|
|
// override. Non-model stages keep empty values. The usage the stage already
|
|
// measured for budget accounting is carried onto the same step, never
|
|
// re-read from the harness or parsed from the transcript.
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
step.Provider = modelConfig.Provider
|
|
step.Model = modelConfig.Model
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Usage = stageUsage(budget.Snapshot().Since(budgetBefore))
|
|
}
|
|
}
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
if len(result.Output) > 0 {
|
|
step.Output["salvage"] = result.Output
|
|
}
|
|
}
|
|
// A failed or timed-out step carries the session pointer even when the
|
|
// bounded transcript would otherwise be only its stage_limits envelope.
|
|
if err != nil && stageSession != "" {
|
|
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
|
|
}
|
|
step.Target = def.Edges[step.Outcome]
|
|
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
|
|
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
|
|
for _, outcome := range selected.Limits.On {
|
|
if outcome == step.Outcome {
|
|
step.Control.LimitReached = true
|
|
step.Control.ModelTarget = step.Target
|
|
step.Target = protocol.FailureEnd
|
|
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
|
|
}
|
|
}
|
|
}
|
|
if panicked {
|
|
step.Target = protocol.FailureEnd
|
|
}
|
|
// Generate the one-line verdict reason once, at completion. It is
|
|
// best-effort: a summariser failure or timeout returns NoSummary and must
|
|
// never fail the stage or the run, so every error is swallowed. A failed
|
|
// step still gets a verdict from its error envelope.
|
|
if r.verdict != nil {
|
|
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
|
|
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
|
|
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
|
|
}
|
|
reasonCancel()
|
|
}
|
|
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
|
|
if finishErr != nil {
|
|
finish("failed", "error", finishErr.Error())
|
|
return
|
|
}
|
|
// Advance the in-memory UI-surface evidence only after the step is
|
|
// persisted, so the skip guard agrees with what the store holds.
|
|
state.recordChanges(step.FileChanges, step.Artifacts)
|
|
if step.Status == "completed" {
|
|
state.completed++
|
|
if artifactsDir != "" {
|
|
state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence))
|
|
}
|
|
}
|
|
// Step finished: report its outcome, routing target, diagnostic and the
|
|
// artifacts recorded for the visit, plus the run's running step count.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Artifacts: len(step.Artifacts),
|
|
Step: &protocol.RunStepDelta{
|
|
Sequence: step.Sequence,
|
|
Stage: step.Stage,
|
|
Status: step.Status,
|
|
Outcome: step.Outcome,
|
|
Target: step.Target,
|
|
Diagnostic: step.Diagnostic,
|
|
Artifacts: len(step.Artifacts),
|
|
},
|
|
}, false)
|
|
if useAgent || errors.Is(err, stage.ErrStageCleanup) {
|
|
sessions, checkErr := r.store.ActiveAgentSessionsForRun(context.Background(), id)
|
|
if checkErr != nil || len(sessions) > 0 {
|
|
finish("failed", "error", "agent quiescence not confirmed; inspect retained session before further work")
|
|
return
|
|
}
|
|
}
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
if terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
}
|
|
finish("failed", "error", "total stage budget exceeded")
|
|
}
|
|
|
|
// exportStageSession writes the finished stage's session into its own artifacts
|
|
// directory and returns the evidence for the step. A non-OpenCode harness (pi)
|
|
// has no native export path, so it records that limitation and writes no files.
|
|
// Any export error is recorded as a sanitized diagnostic and never returned to
|
|
// the stage: the export is read-only, bounded and best-effort.
|
|
func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport {
|
|
evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend}
|
|
if owner.Backend != "v1" && owner.Backend != "v2" {
|
|
evidence.Source = "unavailable"
|
|
evidence.Diagnostic = "session export unavailable: harness has no native export path"
|
|
return evidence
|
|
}
|
|
export := r.exportSession
|
|
if export == nil {
|
|
export = stage.ExportSession
|
|
}
|
|
// A canceled stage still gets one bounded, non-blocking attempt; the export
|
|
// never consumes the stage timeout and never waits for the session.
|
|
exportCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stage.SessionExportTimeout)
|
|
defer cancel()
|
|
result, err := export(exportCtx, stage.SessionExportRequest{
|
|
SessionID: owner.ID,
|
|
Backend: owner.Backend,
|
|
Directory: owner.Directory,
|
|
ArtifactsDir: artifactsDir,
|
|
RunID: runID,
|
|
Sequence: sequence,
|
|
Stage: key,
|
|
Provider: modelConfig.Provider,
|
|
Model: modelConfig.Model,
|
|
})
|
|
if err != nil {
|
|
evidence.Source = "native_export"
|
|
evidence.Diagnostic = stage.RedactEvidence(err.Error())
|
|
return evidence
|
|
}
|
|
evidence.Source = result.Source
|
|
evidence.HTML = result.HTML
|
|
evidence.HTMLBytes = result.HTMLBytes
|
|
evidence.Messages = result.Messages
|
|
evidence.Truncated = result.Truncated
|
|
evidence.Diagnostic = result.Diagnostic
|
|
return evidence
|
|
}
|
|
|
|
// setTitle renames a run: it replaces the stored operator-facing title without
|
|
// touching the run's input, steps or execution, then returns the updated detail.
|
|
func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) {
|
|
explicit, ok := summarise.ExplicitTitle(title)
|
|
if !ok {
|
|
return nil, fmt.Errorf("a title with usable text is required")
|
|
}
|
|
if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil {
|
|
return nil, err
|
|
}
|
|
return r.getRun(id)
|
|
}
|