cedfc22e45
Resolve the run workspace's environment with workenv (direnv, parent .envrc, mise; same account rule as Claude chats) at run start and resume, and apply it to Pi stage processes, a per-environment Pipelines-owned OpenCode stage service (keyed by a hash of the change, reused across runs, idle ones stopped after PIPELINES_OPENCODE_STAGE_IDLE_TIMEOUT, revived per registration), delivery-gate checks and merges, plain-stage opencode CLI calls and file-change capture git reads. Record the directory, .envrc files, sources and variable names (never values) on the run detail; a blocked .envrc becomes a run warning.
2251 lines
89 KiB
Go
2251 lines
89 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/filediff"
|
|
"git.a-wan.net/alan/pipelines/internal/harness"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
"git.a-wan.net/alan/pipelines/internal/turn"
|
|
"git.a-wan.net/alan/pipelines/internal/workenv"
|
|
)
|
|
|
|
// activeRun records an in-flight or cleanup-unconfirmed run's cancellation seam
|
|
// and resolved workspace reservation. A single map is the active-run state, so the
|
|
// workspace cannot drift from the cancel function.
|
|
type activeRun struct {
|
|
cancel context.CancelFunc
|
|
workspace string
|
|
generation *byte
|
|
}
|
|
|
|
type runner struct {
|
|
ctx context.Context
|
|
store *pipeline.Store
|
|
artifactsRoot string
|
|
mu sync.Mutex
|
|
active map[int64]activeRun
|
|
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
|
|
// dirEnv resolves the run workspace's own environment (direnv/mise) at
|
|
// run start; nil uses workenv.Default.
|
|
dirEnv *workenv.Resolver
|
|
|
|
exportRepairMu sync.Mutex
|
|
|
|
// title generates the operator-facing run title from the run prompt. When nil
|
|
// the store derives a fallback from the prompt with no model call, so tests
|
|
// and unconfigured servers stay deterministic.
|
|
title func(context.Context, string) summarise.Title
|
|
// verdict generates the bounded one-line reason for a completed stage visit
|
|
// from its output map. When nil the store derives the reason at read time with
|
|
// no model call. It is best-effort: any failure must never fail the stage.
|
|
verdict func(context.Context, map[string]any) string
|
|
// events pushes run state deltas to subscribed clients. When nil (tests that
|
|
// build a runner directly) emits are dropped, so nothing else changes.
|
|
events *runEventHub
|
|
// notes records and streams app notifications beside every ntfy
|
|
// publication. Nil (a runner built directly in a test) records nothing.
|
|
notes *appNotifier
|
|
// identMu guards idents: the pipeline identity and stored title (see
|
|
// runIdentity) that every emitted frame of a run carries, so a client with no
|
|
// cached row renders a brand-new run without re-reading the list. Populated
|
|
// where a run is created, decided or canceled; a run with no remembered
|
|
// identity keeps blank identity fields and the client resyncs once instead.
|
|
identMu sync.Mutex
|
|
idents map[int64]runIdentity
|
|
// exportSession writes a finished stage's session into its own artifacts
|
|
// directory. It defaults to stage.ExportSession; a failure is best-effort and
|
|
// never changes the stage's status, outcome, target or result validation.
|
|
exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error)
|
|
// verifyTarget is installed only by server wiring for a controlled fixture
|
|
// workflow. Leaving it nil fails closed for target-aware definitions.
|
|
verifyTarget stage.VerificationRunner
|
|
wg sync.WaitGroup
|
|
schedulerWG sync.WaitGroup
|
|
// attentionMu guards attentionSent, the per-run set of run-attention events
|
|
// already delivered — by a stage's own notify policy or by the run-level
|
|
// default — so neither republishes the same genuine event.
|
|
attentionMu sync.Mutex
|
|
attentionSent map[int64]map[string]bool
|
|
// notifyMu guards notifyModes: the per-run auto/on/off override recorded at
|
|
// admission. It is separate from the stage's own notify policy.
|
|
notifyMu sync.Mutex
|
|
notifyModes map[int64]string
|
|
// sizeKick wakes the background run-size backfill (sizeLoop); nil when the
|
|
// loop is not running (a runner built directly in a test).
|
|
sizeKick chan struct{}
|
|
// detach leaves harness sessions running when the server stops; reattach
|
|
// holds the runs start-up recovery kept for reattachRuns to continue.
|
|
detach bool
|
|
reattach []reattachPlan
|
|
}
|
|
|
|
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
|
return newRunnerAt(ctx, s, "")
|
|
}
|
|
|
|
// newRunnerAt builds a runner whose stage artifacts live under artifactsRoot
|
|
// ("" means <store directory>/artifacts). artifactsRoot is fixed before any
|
|
// background goroutine (session export jobs, notifications, the size backfill)
|
|
// starts and is never written afterwards, so those goroutines read it without
|
|
// locking.
|
|
func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) {
|
|
return newRunnerReattaching(ctx, s, artifactsRoot, false)
|
|
}
|
|
|
|
// newRunnerReattaching builds the runner. With detach set, a server shutdown
|
|
// leaves stage sessions running (see stage.ErrServerShutdown) and start-up
|
|
// recovery re-adopts the runs a previous process left in flight instead of
|
|
// interrupting them; reattachRuns then continues them.
|
|
func newRunnerReattaching(ctx context.Context, s *pipeline.Store, artifactsRoot string, detach bool) (*runner, error) {
|
|
if err := s.AcquireRunnerLock(); err != nil {
|
|
return nil, err
|
|
}
|
|
// Store.Open is also used by offline tools, whose cwd is not execution
|
|
// authority. Bind imported legacy defaults only here, before serving requests
|
|
// or starting scheduler workers, and never rebind an explicit saved folder.
|
|
workspace, err := pipeline.ResolveRunWorkspace("", false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err = s.BindLegacyScheduleWorkspace(ctx, workspace); err != nil {
|
|
return nil, err
|
|
}
|
|
var plans []reattachPlan
|
|
keep, reasons, gone := map[int64]bool{}, map[int64]string{}, map[int64]bool{}
|
|
if detach {
|
|
if plans, reasons, gone, err = planReattach(ctx, s); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, plan := range plans {
|
|
keep[plan.runID] = true
|
|
}
|
|
}
|
|
sessions, err := s.ActiveSessionOwners(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, owner := range sessions {
|
|
if keep[owner.RunID] {
|
|
continue // Re-adopted below: never interrupt work that survived.
|
|
}
|
|
var err error
|
|
if !gone[owner.RunID] {
|
|
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
err = interruptRecoveredSession(stop, owner)
|
|
cancel()
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
|
}
|
|
if err := s.ConfirmRecoveredSession(ctx, owner); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if err := s.RecoverRunsKeeping(ctx, keep, reasons); err != nil {
|
|
return nil, err
|
|
}
|
|
root := artifactsRoot
|
|
if root == "" {
|
|
root = filepath.Join(s.Directory(), "artifacts")
|
|
}
|
|
active := map[int64]activeRun{}
|
|
rows, err := s.ListRuns(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, row := range rows {
|
|
if row.CleanupUnconfirmed {
|
|
active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace}
|
|
}
|
|
}
|
|
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession, detach: detach, reattach: plans}
|
|
if err := r.startSessionExportJobs(); err != nil {
|
|
return nil, err
|
|
}
|
|
r.schedulerWG.Add(1)
|
|
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
|
|
return r, nil
|
|
}
|
|
|
|
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
|
return r.startWithSchedule(request, nil, retryOf...)
|
|
}
|
|
|
|
func (r *runner) startWithSchedule(request protocol.Message, origin *pipeline.ScheduleAdmission, retryOf ...int64) (*protocol.RunDetail, error) {
|
|
if err := protocol.ValidateOutputFormat(request.OutputFormat); err != nil {
|
|
return nil, err
|
|
}
|
|
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
|
return nil, fmt.Errorf("an explicit version and request key are required")
|
|
}
|
|
workspace, err := pipeline.ResolveRunWorkspace(request.Workspace, origin != nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err = pipeline.ValidateRunWritePaths(workspace, request.WritePaths); err != nil {
|
|
return nil, err
|
|
}
|
|
request.Input = maps.Clone(request.Input)
|
|
if request.Input == nil {
|
|
request.Input = map[string]any{}
|
|
}
|
|
request.Input["pwd"] = workspace
|
|
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
|
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
|
}
|
|
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
|
|
return nil, err
|
|
}
|
|
// Validate the launch-time model overrides against the pinned graph before
|
|
// admission: an unknown stage key or a malformed provider/model must never
|
|
// insert a run row. The resolution is O(#overrides + #stages) with no model or
|
|
// network call, and the same helper is the authoritative precedence rule.
|
|
if _, err := pipeline.EffectiveModels(d.Graph, request.Model, request.StageModels); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["harness"] == "pi" && r.execute == nil {
|
|
configured, err := harness.Load()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, ok := configured.Profiles["pi"]; !ok {
|
|
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
|
|
}
|
|
}
|
|
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !explicit {
|
|
request.AllowTools = true
|
|
}
|
|
for _, tool := range tools {
|
|
if tool == "edit" || tool == "shell" {
|
|
request.AllowTools = true
|
|
}
|
|
}
|
|
}
|
|
// Decide admission and idempotency under the lock, but generate the title
|
|
// only for a request that will actually start a run: a rejected or replayed
|
|
// request must not pay for a bounded model generation, and a live one must
|
|
// not hold the lock (stalling listings and cancellation) while a model is
|
|
// called. An unavailable or slow model falls back to the prompt's first line.
|
|
r.mu.Lock()
|
|
if r.ctx.Err() != nil {
|
|
r.mu.Unlock()
|
|
return nil, r.ctx.Err()
|
|
}
|
|
owners, err := r.store.ActiveSessionOwners(r.ctx)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
// Refuse only an orphaned session: one whose owning run is not active now.
|
|
// A session owned by a currently active run (another run in another
|
|
// workspace) must not block new work.
|
|
for _, owner := range owners {
|
|
if _, ok := r.active[owner.RunID]; !ok {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
|
}
|
|
}
|
|
// Idempotent repeats are accepted even while the original run is active, and
|
|
// a replay reuses the stored title instead of generating a new one.
|
|
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
var runTitle summarise.Title
|
|
if existing == 0 {
|
|
// An explicit operator title skips the model call; otherwise the
|
|
// configured generator names the run, falling back to the prompt's
|
|
// first line when it is unavailable or slow.
|
|
if explicit, ok := summarise.ExplicitTitle(request.Title); ok {
|
|
runTitle = explicit
|
|
} else if r.title != nil {
|
|
prompt, _ := request.Input["prompt"].(string)
|
|
runTitle = r.title(r.ctx, prompt)
|
|
}
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
// Another run may have reserved this workspace while the title was
|
|
// generated, so re-check before reserving it; a race on the request key is
|
|
// handled by the store's idempotent insert.
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Schedule: origin, Model: request.Model, StageModels: request.StageModels, NotifyMode: request.NotifyMode, OutputFormat: request.OutputFormat}, runTitle, retryOf...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r.setNotifyMode(id, request.NotifyMode)
|
|
// The run row now exists: read it once for its resolved identity (pipeline
|
|
// id/name, the store-derived title, the pinned version), so the queued frame
|
|
// and every later frame carry what a client needs to render a brand-new run
|
|
// without re-reading the list. A failed read only costs that seed: the run is
|
|
// still launched below and its frames carry no identity, so the client falls
|
|
// back to its one resync.
|
|
detail, readErr := r.store.GetRun(r.ctx, id)
|
|
if readErr != nil {
|
|
detail = nil
|
|
} else if detail != nil {
|
|
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
|
}
|
|
if created {
|
|
if request.RetrySource > 0 && request.RetryMode != "restart" {
|
|
checkpoint, checkpointErr := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput)
|
|
if checkpointErr != nil {
|
|
return nil, checkpointErr
|
|
}
|
|
if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
// Emit the created delta before launching: the queued frame reaches
|
|
// subscribers before the running/step-started transition, so a client
|
|
// sees the run appear and then advance.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusQueued,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}, false)
|
|
r.launch(id, workspace, func(ctx context.Context) {
|
|
r.run(ctx, id, d, workspace, request)
|
|
})
|
|
}
|
|
if detail != nil {
|
|
_, detail.Executing = r.active[id]
|
|
if detail.ExportPendingSequence > 0 {
|
|
detail.Executing = false
|
|
}
|
|
r.overlayEffectiveModels(detail)
|
|
}
|
|
if readErr != nil {
|
|
return nil, readErr
|
|
}
|
|
return detail, nil
|
|
}
|
|
|
|
// launch registers an active run's cancellation seam and workspace, adds it to
|
|
// the wait group and starts fn on a derived cancellable context that cleans up
|
|
// the registration when it returns. A fresh run and a resumed decision both go
|
|
// through it, so registration and teardown live once. Callers must hold r.mu.
|
|
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
|
ctx, cancel := r.runContext()
|
|
generation := new(byte)
|
|
r.active[id] = activeRun{cancel: cancel, workspace: workspace, generation: generation}
|
|
r.wg.Add(1)
|
|
go func() {
|
|
defer r.wg.Done()
|
|
defer cancel()
|
|
defer func() {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if entry, ok := r.active[id]; !ok || entry.generation != generation {
|
|
return // A parked launch must not remove the resumed reservation.
|
|
}
|
|
check, done := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer done()
|
|
detail, err := r.store.GetRun(check, id)
|
|
if err != nil || detail.CleanupUnconfirmed {
|
|
return // Fail closed: execution/cleanup has not settled durably.
|
|
}
|
|
if detail.Status == protocol.RunStatusWaiting && detail.CancelRequested {
|
|
if err := r.store.CancelWaiting(check, id); err != nil {
|
|
return
|
|
}
|
|
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: detail.Workspace, Status: protocol.RunStatusCanceled, CancelRequested: true, CurrentStage: detail.CurrentStage, FinishedAt: time.Now().UTC().Format(time.RFC3339)}, true)
|
|
}
|
|
if detail.FinishedAt != "" || detail.Status == protocol.RunStatusWaiting {
|
|
delete(r.active, id)
|
|
}
|
|
}()
|
|
fn(ctx)
|
|
}()
|
|
}
|
|
|
|
// checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run
|
|
// reserves an overlapping resolved workspace. Callers must hold r.mu.
|
|
func (r *runner) checkWorkspaceFree(workspace string) error {
|
|
return r.checkWorkspaceFreeExcept(workspace, 0)
|
|
}
|
|
|
|
func (r *runner) checkWorkspaceFreeExcept(workspace string, ownGate int64) error {
|
|
for id, entry := range r.active {
|
|
if workspacesOverlap(entry.workspace, workspace) {
|
|
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
|
|
}
|
|
}
|
|
check, cancel := context.WithTimeout(context.WithoutCancel(r.ctx), 5*time.Second)
|
|
defer cancel()
|
|
reservations, err := r.store.SessionExportReservations(check)
|
|
if err != nil {
|
|
return fmt.Errorf("read pending export workspace reservations: %w", err)
|
|
}
|
|
for id, reserved := range reservations {
|
|
if id != ownGate && workspacesOverlap(reserved, workspace) {
|
|
return fmt.Errorf("run %d is awaiting its session export; workspace %s remains reserved", id, reserved)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func workspacesOverlap(a, b string) bool {
|
|
within := func(parent, child string) bool {
|
|
rel, err := filepath.Rel(parent, child)
|
|
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator))
|
|
}
|
|
return within(a, b) || within(b, a)
|
|
}
|
|
|
|
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
d, err := r.store.GetRun(r.ctx, id)
|
|
if d != nil {
|
|
_, d.Executing = r.active[id]
|
|
if d.ExportPendingSequence > 0 {
|
|
d.Executing = false
|
|
}
|
|
for _, s := range d.Steps {
|
|
if s.AgentActive {
|
|
d.Executing = true
|
|
}
|
|
}
|
|
r.overlayEffectiveModels(d)
|
|
}
|
|
return d, err
|
|
}
|
|
|
|
// overlayEffectiveModels replaces each model stage's provider/model in a returned
|
|
// run's graph with the effective pair the run actually uses, so the app and TUI
|
|
// stage-definition surfaces show the override with no client change. It is
|
|
// presentation only: the persisted graph_json and the pinned source returned by
|
|
// `show` are never rewritten. A run with no overrides is left untouched.
|
|
func (r *runner) overlayEffectiveModels(d *protocol.RunDetail) {
|
|
if d == nil || (d.Model == "" && len(d.StageModels) == 0) {
|
|
return
|
|
}
|
|
effective, err := pipeline.EffectiveModels(d.Graph, d.Model, d.StageModels)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for i := range d.Graph.Stages {
|
|
s := &d.Graph.Stages[i]
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
override, ok := effective[s.Key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if s.Configs == nil {
|
|
s.Configs = map[string]string{}
|
|
}
|
|
s.Configs["provider"] = override.Provider
|
|
s.Configs["model"] = override.Model
|
|
}
|
|
}
|
|
|
|
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
items, err := r.store.ListRuns(r.ctx)
|
|
for i := range items {
|
|
_, items[i].Executing = r.active[items[i].ID]
|
|
if items[i].ExportPendingSequence > 0 {
|
|
items[i].Executing = false
|
|
}
|
|
if items[i].Status == "failed" && !items[i].Executing {
|
|
d, err := r.store.GetRun(r.ctx, items[i].ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, step := range d.Steps {
|
|
if step.AgentActive {
|
|
items[i].Executing = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items, err
|
|
}
|
|
|
|
type retryCheckpoint struct {
|
|
detail *protocol.RunDetail
|
|
failed protocol.RunStep
|
|
handoff map[string]any
|
|
}
|
|
|
|
type retryPrefixRecord struct {
|
|
stage string
|
|
sequence int
|
|
public []string
|
|
publicKnown bool
|
|
}
|
|
|
|
const maxRetryInputBytes = 64 * 1024
|
|
|
|
func retryInputOverride(base map[string]any, override map[string]any) (map[string]any, error) {
|
|
if override == nil {
|
|
return base, nil
|
|
}
|
|
encoded, err := json.Marshal(override)
|
|
if err != nil || len(encoded) > maxRetryInputBytes {
|
|
return nil, fmt.Errorf("retry input must be a JSON object no larger than %d bytes", maxRetryInputBytes)
|
|
}
|
|
reserved := map[string]bool{"pwd": true, "workspace": true, "write_paths": true, "allow_tools": true, "pipeline_id": true, "pipeline_version": true, "request_key": true, "retry_mode": true, "retry_source": true, "retry_stage": true, "retry_visit": true, serverChangedFilesField: true}
|
|
merged := map[string]any{}
|
|
for key, value := range base {
|
|
merged[key] = value
|
|
}
|
|
for key, value := range override {
|
|
if reserved[key] {
|
|
return nil, fmt.Errorf("retry input field %q is reserved", key)
|
|
}
|
|
merged[key] = value
|
|
}
|
|
return merged, nil
|
|
}
|
|
|
|
// retryStageContractEqual compares the complete persisted stage contract. A
|
|
// retry may carry completed output across a published version only when the
|
|
// stages that produced that output are unchanged; comparing only effective
|
|
// types would allow prompt, config, edge, or limit changes through.
|
|
func retryStageContractEqual(left, right protocol.Stage) bool {
|
|
a, err := json.Marshal(left)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
b, err := json.Marshal(right)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return string(a) == string(b)
|
|
}
|
|
|
|
func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) {
|
|
return r.retrySourceCheckpointSelected(sourceID, pinned, "", 0, nil)
|
|
}
|
|
|
|
func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.PipelineDetail, selectedStage string, selectedVisit int, override map[string]any) (retryCheckpoint, error) {
|
|
source, err := r.store.GetRun(r.ctx, sourceID)
|
|
if err != nil {
|
|
return retryCheckpoint{}, err
|
|
}
|
|
if source.Status != "failed" || source.FinishedAt == "" {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID)
|
|
}
|
|
if source.PipelineID != pinned.ID {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d belongs to a different pipeline", sourceID)
|
|
}
|
|
if len(source.Steps) == 0 {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID)
|
|
}
|
|
failed := source.Steps[len(source.Steps)-1]
|
|
if selectedStage != "" {
|
|
failed = protocol.RunStep{}
|
|
visits := 0
|
|
for _, candidate := range source.Steps {
|
|
if candidate.Stage != selectedStage {
|
|
continue
|
|
}
|
|
visits++
|
|
if selectedVisit == 0 && visits == 1 {
|
|
failed = candidate
|
|
}
|
|
if selectedVisit > 0 && visits == selectedVisit {
|
|
failed = candidate
|
|
}
|
|
}
|
|
if failed.Stage == "" {
|
|
return retryCheckpoint{}, fmt.Errorf("retry stage %q was not recorded in run %d", selectedStage, sourceID)
|
|
}
|
|
if selectedVisit == 0 {
|
|
count := 0
|
|
for _, candidate := range source.Steps {
|
|
if candidate.Stage == selectedStage {
|
|
count++
|
|
}
|
|
}
|
|
if count > 1 {
|
|
return retryCheckpoint{}, fmt.Errorf("retry stage %q was visited %d times; --visit is required", selectedStage, count)
|
|
}
|
|
}
|
|
if selectedVisit > visits {
|
|
return retryCheckpoint{}, fmt.Errorf("retry stage %q has no visit %d", selectedStage, selectedVisit)
|
|
}
|
|
}
|
|
if (failed.Status != "completed" && failed.Status != "failed") || (selectedStage == "" && failed.Target != protocol.FailureEnd) {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID)
|
|
}
|
|
var handoff map[string]any
|
|
if failed.Input == nil {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID)
|
|
}
|
|
if _, err := retryInputOverride(failed.Input, override); err != nil {
|
|
return retryCheckpoint{}, err
|
|
}
|
|
// Keep the source-stage input immutable in the checkpoint. The selected
|
|
// stage's memory bindings must be resolved from the original prefix before
|
|
// this override is applied at invocation time.
|
|
handoff = map[string]any{"input": failed.Input, "override": override, "sequence": failed.Sequence, "stage": failed.Stage, "visit": failed.Sequence}
|
|
prefix, prefixRecords, prefixErr := r.retryPrefix(sourceID, source, failed.Sequence)
|
|
if prefixErr != nil {
|
|
return retryCheckpoint{}, prefixErr
|
|
}
|
|
handoff["prefix"] = prefix
|
|
serializedPrefix := make([]map[string]any, 0, len(prefixRecords))
|
|
for _, record := range prefixRecords {
|
|
entry := map[string]any{"stage": record.stage, "sequence": record.sequence}
|
|
if record.publicKnown {
|
|
// Persist the exact additions so a later retry can rebuild memory
|
|
// and carry public/ without reading this attempt's rows.
|
|
entry["public"] = record.public
|
|
}
|
|
serializedPrefix = append(serializedPrefix, entry)
|
|
}
|
|
handoff["prefix_provenance"] = serializedPrefix
|
|
// A newer explicitly selected version may continue only when every recorded
|
|
// prefix stage still exists with the same kind. Its outputs are evidence from
|
|
// the source attempt and cannot be safely handed to a changed stage contract.
|
|
sourceStages := map[string]protocol.Stage{}
|
|
for _, declared := range source.Graph.Stages {
|
|
sourceStages[declared.Key] = declared
|
|
}
|
|
for _, record := range prefixRecords {
|
|
_, found := retryPrefixStep(source, record)
|
|
if !found {
|
|
continue
|
|
}
|
|
var pinnedStage *protocol.Stage
|
|
for i := range pinned.Graph.Stages {
|
|
if pinned.Graph.Stages[i].Key == record.stage {
|
|
pinnedStage = &pinned.Graph.Stages[i]
|
|
break
|
|
}
|
|
}
|
|
previous, ok := sourceStages[record.stage]
|
|
if pinnedStage == nil || !ok || !retryStageContractEqual(previous, *pinnedStage) {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, record.stage)
|
|
}
|
|
}
|
|
var sourceSelected protocol.Stage
|
|
var sourceSelectedOK bool
|
|
if sourceSelected, sourceSelectedOK = sourceStages[failed.Stage]; !sourceSelectedOK {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in its recorded graph", sourceID)
|
|
}
|
|
pinnedStages := map[string]protocol.Stage{}
|
|
for _, declared := range pinned.Graph.Stages {
|
|
pinnedStages[declared.Key] = declared
|
|
}
|
|
// The retry will continue through every stage that the source execution
|
|
// already recorded after the selected checkpoint. Validate those stages
|
|
// before admitting the new run as well: otherwise an explicitly selected
|
|
// version can pass prefix/selected-stage validation and fail later in the
|
|
// loop with the less useful "missing stage" error.
|
|
for _, step := range source.Steps {
|
|
if step.Sequence <= failed.Sequence {
|
|
continue
|
|
}
|
|
if _, ok := pinnedStages[step.Stage]; !ok {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d selected version is missing later stage %q", sourceID, step.Stage)
|
|
}
|
|
}
|
|
for _, declared := range pinned.Graph.Stages {
|
|
if declared.Key != failed.Stage {
|
|
continue
|
|
}
|
|
if stage.EffectiveType(sourceSelected.Type) != stage.EffectiveType(declared.Type) {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible selected stage %q", sourceID, failed.Stage)
|
|
}
|
|
typ := stage.EffectiveType(declared.Type)
|
|
if typ == stage.HumanType || typ == stage.NtfyType {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage)
|
|
}
|
|
if typ == stage.ModelType {
|
|
if _, _, toolErr := stage.ToolNames(pipeline.StageDefinition(declared)); toolErr != nil {
|
|
return retryCheckpoint{}, toolErr
|
|
}
|
|
}
|
|
return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil
|
|
}
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID)
|
|
}
|
|
|
|
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("retry requires a positive run ID")
|
|
}
|
|
original, err := r.getRun(request.RunID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if original.Executing {
|
|
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
|
|
}
|
|
if original.Status != "failed" || original.FinishedAt == "" {
|
|
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
|
}
|
|
if request.RequestKey == "" {
|
|
return nil, fmt.Errorf("retry requires an idempotency request key")
|
|
}
|
|
// Resolve an already admitted request before validating optional retry
|
|
// selectors. This makes replay after a restart genuinely idempotent even
|
|
// when the original request used an input override that the replay omits.
|
|
if existing, err := r.store.RunByKey(r.ctx, request.RequestKey); err != nil {
|
|
return nil, err
|
|
} else if existing > 0 {
|
|
return r.store.GetRun(r.ctx, existing)
|
|
}
|
|
if blocked, err := r.store.SessionExportBlocked(r.ctx, original.ID); err != nil {
|
|
return nil, err
|
|
} else if blocked {
|
|
return nil, fmt.Errorf("run %d has a retained computation awaiting export-only recovery; model retry refused", original.ID)
|
|
}
|
|
if request.RetryVisit < 0 || (request.RetryVisit > 0 && request.RetryStage == "") {
|
|
return nil, fmt.Errorf("retry visit requires a positive --stage visit selection")
|
|
}
|
|
mode := request.RetryMode
|
|
if mode == "" {
|
|
mode = "resume"
|
|
}
|
|
if mode != "resume" && mode != "restart" {
|
|
return nil, fmt.Errorf("retry mode must be resume or restart")
|
|
}
|
|
version := original.Version
|
|
if request.RetryVersion > 0 {
|
|
version = request.RetryVersion
|
|
}
|
|
d, err := r.store.Get(r.ctx, original.PipelineID, version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if mode == "resume" {
|
|
if _, err := r.retrySourceCheckpointSelected(original.ID, d, request.RetryStage, request.RetryVisit, request.RetryInput); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
notifyMode := request.NotifyMode
|
|
if notifyMode == "" {
|
|
notifyMode, _ = r.store.RunNotifyMode(r.ctx, original.ID)
|
|
}
|
|
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, RetryStage: request.RetryStage, RetryVisit: request.RetryVisit, RetryInput: request.RetryInput, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels, NotifyMode: notifyMode, OutputFormat: original.OutputFormat}, original.ID)
|
|
}
|
|
|
|
func (r *runner) cancel(id int64) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
entry, ok := r.active[id]
|
|
if err := r.store.Cancel(r.ctx, id); err != nil {
|
|
return err
|
|
}
|
|
if ok {
|
|
entry.cancel()
|
|
} else {
|
|
// Only a parked human run is known to have no live executor.
|
|
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
r.forgetAttention(id)
|
|
if r.events != nil {
|
|
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
|
|
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: detail.Workspace,
|
|
Status: detail.Status,
|
|
Outcome: detail.Outcome,
|
|
CurrentStage: detail.CurrentStage,
|
|
CompletedSteps: detail.CompletedSteps,
|
|
FinishedAt: detail.FinishedAt,
|
|
CancelRequested: detail.CancelRequested,
|
|
CleanupUnconfirmed: detail.CleanupUnconfirmed,
|
|
}, !ok)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// decide records an operator's choice for a waiting human stage and resumes the
|
|
// run along the chosen edge. ResolveDecision owns validation and idempotency, so
|
|
// decide does no option parsing of its own. A resumed run is registered and
|
|
// launched exactly like a fresh one but starts from the persisted state.
|
|
func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("decide requires a positive run ID")
|
|
}
|
|
r.mu.Lock()
|
|
detail, err := r.store.GetRun(r.ctx, request.RunID)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
// finishRunState persists the terminal status under r.mu, but the run stays
|
|
// in r.active until its goroutine finishes notifying and the launch teardown
|
|
// runs. A run whose terminal state is already durable (FinishedAt is only set
|
|
// once cleanup is confirmed) has no executor left to race with a decision, so
|
|
// ResolveDecision answers it ("already decided" or "not waiting") and never
|
|
// resumes a terminal run. A run that is still executing is refused.
|
|
if _, busy := r.active[detail.ID]; busy && detail.FinishedAt == "" {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
|
|
}
|
|
if detail.ExportPendingSequence > 0 {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("run %d is awaiting session export, not a human decision", detail.ID)
|
|
}
|
|
if detail.Status == protocol.RunStatusWaiting {
|
|
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if applied {
|
|
// Re-seed the identity so the resumed frames (this one and the resumed
|
|
// loop's) carry it even after a restart, when no creation was observed.
|
|
r.rememberRunIdentity(detail.ID, identityOfRunSummary(detail.RunSummary))
|
|
for i := range detail.Steps {
|
|
if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage {
|
|
detail.Steps[i] = step
|
|
break
|
|
}
|
|
}
|
|
// The decision closes the waiting step and returns the run to running;
|
|
// report that before the resumed loop emits its own deltas.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: detail.ID,
|
|
Workspace: detail.Workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: step.Stage,
|
|
CompletedSteps: completedSteps(detail.Steps),
|
|
Step: &protocol.RunStepDelta{
|
|
Sequence: step.Sequence,
|
|
Stage: step.Stage,
|
|
Status: step.Status,
|
|
Outcome: step.Outcome,
|
|
Target: step.Target,
|
|
Diagnostic: step.Diagnostic,
|
|
},
|
|
}, false)
|
|
r.launch(detail.ID, detail.Workspace, func(ctx context.Context) {
|
|
r.resume(ctx, detail.ID, detail, detail.Workspace, step)
|
|
})
|
|
}
|
|
r.mu.Unlock()
|
|
return r.getRun(request.RunID)
|
|
}
|
|
|
|
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
|
|
if request.RetrySource > 0 && request.RetryMode != "restart" {
|
|
source, err := r.store.GetRun(ctx, request.RetrySource)
|
|
if err != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
|
return
|
|
}
|
|
checkpoint, err := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput)
|
|
if err != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
|
return
|
|
}
|
|
storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id)
|
|
if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil {
|
|
if checkpointErr == nil {
|
|
checkpointErr = fmt.Errorf("retry checkpoint is incomplete")
|
|
}
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil)
|
|
return
|
|
}
|
|
if input, ok := storedHandoff["input"].(map[string]any); ok {
|
|
checkpoint.failed.Input = input
|
|
} else {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
|
|
return
|
|
}
|
|
state := rebuildRetryState(source, checkpoint.failed, storedHandoff)
|
|
if override, ok := storedHandoff["override"].(map[string]any); ok {
|
|
state.retryOverride = stage.Values(override)
|
|
}
|
|
// The retry is a new run with a new artifact root. Carry the completed
|
|
// prefix's recorded public files before the first visit, so memory does
|
|
// not name files this attempt cannot read.
|
|
if err := r.carryRetryPublic(id, request.RetrySource, checkpoint.failed.Sequence, storedHandoff, source); err != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
|
return
|
|
}
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
|
|
return
|
|
}
|
|
original, copyErr := copyValues(stage.Values(request.Input))
|
|
if copyErr != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
|
|
return
|
|
}
|
|
input := stage.Values(request.Input)
|
|
state := newRunState(d.Graph.Stages[0].Key, input)
|
|
state.memory["input"] = memoryValue{output: original}
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
|
|
}
|
|
|
|
// terminalFinish classifies a stage's resolved target. A terminal key yields the
|
|
// run's final status, outcome and diagnostic; an empty target means the stage
|
|
// named an undeclared outcome; any other value is a stage key and is not
|
|
// terminal. resume and the loop tail share it, so terminal routing lives once.
|
|
func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) {
|
|
switch target {
|
|
case protocol.SuccessEnd:
|
|
return "succeeded", "success", "", true
|
|
case protocol.FailureEnd:
|
|
if diagnostic == "" {
|
|
diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome)
|
|
}
|
|
return "failed", "error", diagnostic, true
|
|
case "":
|
|
return "failed", "error", "stage returned an undeclared outcome", true
|
|
}
|
|
return "", "", "", false
|
|
}
|
|
|
|
// resume continues a decided run from the state already persisted for its run
|
|
// detail: memory from the run input and the completed steps, visits counted per
|
|
// stage, and the next sequence above the highest recorded one. It then runs the
|
|
// shared loop from the decision's target; no full-table scan and no extra
|
|
// per-step query are introduced.
|
|
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
|
|
// A resumed run's frames carry the same identity as its creation, so re-seed
|
|
// it here too: resume is also reachable from a store read after a restart.
|
|
r.rememberRunIdentity(id, identityOfRunSummary(d.RunSummary))
|
|
// The decision may route straight to a terminal, which has no stage to run.
|
|
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
|
|
r.finishRunState(id, d.Name, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Sequence, step.Stage, stage.Values(step.Output))
|
|
return
|
|
}
|
|
state := rebuildRunState(d, step)
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools, Model: d.Model, StageModels: d.StageModels}, state)
|
|
}
|
|
|
|
// runState is the loop's whole mutable state, extracted so a fresh run and a
|
|
// resumed run share one implementation. nextSequence is set before each visit so
|
|
// a wait that returns to the caller records where the run will continue.
|
|
type runState struct {
|
|
key string
|
|
nextSequence int
|
|
visits map[string]int
|
|
memory map[string]memoryValue
|
|
input stage.Values
|
|
// changedPaths retains a bounded sample of the run's recorded changed paths
|
|
// and changedCount the full total, so the skip guard reads only in-memory
|
|
// evidence. uiSurface is latched once any visit captured a rendered (image)
|
|
// artifact. captureIncomplete counts visits whose file-change capture was
|
|
// missing, partial or unavailable; any such visit forbids the server skip
|
|
// fallback. All are folded in by recordChanges (ui_surface.go).
|
|
changedPaths []string
|
|
changedCount int
|
|
uiSurface bool
|
|
captureIncomplete int
|
|
// capturedPaths is the full, deduplicated, first-seen-ordered set of the
|
|
// run's server-captured changed paths, distinct from the bounded
|
|
// changedPaths sample used only by the skip reason. capturedSeen is the
|
|
// single dedupe set. Both are folded in by recordChanges (ui_surface.go) and
|
|
// published to a selecting stage as input.server_changed_files.
|
|
capturedPaths []string
|
|
capturedSeen map[string]bool
|
|
// completed counts the run's finished steps, so a delta can report the
|
|
// running total without re-reading the store.
|
|
completed int
|
|
artifactDirs []string
|
|
// prompt is the original run prompt; lastOutput is the previous visit's
|
|
// markdown and turns is every completed turn in order. Together they render
|
|
// the next model visit's input.md, memory.md and history.md.
|
|
prompt string
|
|
lastOutput string
|
|
turns []turn.MemoryEntry
|
|
// retryOverride is applied once, after the selected stage's memory bindings
|
|
// have been resolved. It is intentionally not part of run input or memory.
|
|
retryOverride stage.Values
|
|
priorBudgets map[string]stage.BudgetSnapshot
|
|
// adopt names the visit to re-adopt from a previous server process instead
|
|
// of starting it; it is consumed by the loop's first iteration.
|
|
adopt *adoptedVisit
|
|
}
|
|
|
|
func newRunState(key string, input stage.Values) *runState {
|
|
prompt, _ := input["prompt"].(string)
|
|
return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input, prompt: prompt}
|
|
}
|
|
|
|
func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState {
|
|
state := newRunState(decided.Target, stage.Values(decided.Output))
|
|
if prompt, ok := detail.Input["prompt"].(string); ok {
|
|
state.prompt = prompt
|
|
}
|
|
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
|
state.memory["input"] = memoryValue{output: original}
|
|
}
|
|
publicSeen := map[string]bool{}
|
|
state.priorBudgets = map[string]stage.BudgetSnapshot{}
|
|
for _, step := range detail.Steps {
|
|
// The runtime stamps this cumulative snapshot on every executed model
|
|
// visit. Reusing it preserves per-stage limits across export recovery.
|
|
var audit struct {
|
|
Limits struct {
|
|
Budget stage.BudgetSnapshot `json:"budget"`
|
|
} `json:"stage_limits"`
|
|
}
|
|
if json.Unmarshal([]byte(step.Transcript), &audit) == nil {
|
|
state.priorBudgets[step.Stage] = audit.Limits.Budget
|
|
}
|
|
state.visits[step.Stage]++
|
|
if step.Sequence >= state.nextSequence {
|
|
state.nextSequence = step.Sequence + 1
|
|
}
|
|
added := publicArtifacts(step.Artifacts, publicSeen)
|
|
if step.Status == "completed" {
|
|
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
state.completed++
|
|
state.turns = append(state.turns, turn.MemoryEntry{Stage: step.Stage, Visit: state.visits[step.Stage], Outcome: step.Outcome, Output: turnOutput(step.Output), PublicFiles: added})
|
|
state.lastOutput = turnOutput(step.Output)
|
|
}
|
|
// Seed the UI-surface accumulator from the persisted evidence so a
|
|
// resumed run reaches the same skip decision with no extra query.
|
|
state.recordPersistedStep(step)
|
|
// Only completed, recorded visits are eligible for a later stage's
|
|
// review capability. The path is reconstructed by the runner below.
|
|
if step.Status == "completed" {
|
|
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
|
}
|
|
}
|
|
return state
|
|
}
|
|
|
|
// rebuildRetryState restores only the completed prefix. The failed visit is
|
|
// deliberately not counted and its step/evidence is not copied into the new
|
|
// attempt, so retries cannot duplicate execution or usage aggregates.
|
|
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep, handoff map[string]any) *runState {
|
|
state := newRunState(failed.Stage, stage.Values(failed.Input))
|
|
// Resume at the failed sequence. The retry is a new run with a new run ID
|
|
// and artifact root, so this sequence's turn directory does not overlap the
|
|
// source attempt's files.
|
|
state.nextSequence = failed.Sequence
|
|
if prompt, ok := detail.Input["prompt"].(string); ok {
|
|
state.prompt = prompt
|
|
}
|
|
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
|
state.memory["input"] = memoryValue{output: original}
|
|
}
|
|
publicSeen := map[string]bool{}
|
|
_, hasCheckpointPrefix := handoff["prefix"]
|
|
if !hasCheckpointPrefix {
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence >= failed.Sequence {
|
|
break
|
|
}
|
|
state.visits[step.Stage]++
|
|
added := publicArtifacts(step.Artifacts, publicSeen)
|
|
if step.Status == "completed" {
|
|
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
state.completed++
|
|
state.turns = append(state.turns, turn.MemoryEntry{Stage: step.Stage, Visit: state.visits[step.Stage], Outcome: step.Outcome, Output: turnOutput(step.Output), PublicFiles: added})
|
|
state.lastOutput = turnOutput(step.Output)
|
|
}
|
|
state.recordPersistedStep(step)
|
|
if step.Status == "completed" {
|
|
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
|
}
|
|
}
|
|
}
|
|
// A selected-stage retry has no persisted rows for its inherited prefix.
|
|
// Restore it from the immutable checkpoint instead of guessing from the
|
|
// retry source's immediate steps (which is what made retry-of-retry fail
|
|
// closed with "memory source ... has not completed").
|
|
if prefix, ok := handoff["prefix"].(map[string]any); ok {
|
|
// The checkpoint carries outputs, not file-change captures, so an inherited
|
|
// prefix is unobserved evidence: the server skip fallback must not apply.
|
|
state.captureIncomplete++
|
|
for _, record := range retryPrefixRecords(handoff["prefix_provenance"]) {
|
|
output, exists := prefix[record.stage].(map[string]any)
|
|
if !exists {
|
|
continue
|
|
}
|
|
values, err := copyValues(stage.Values(output))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
state.memory[record.stage] = memoryValue{output: values, sequence: record.sequence}
|
|
state.visits[record.stage]++
|
|
state.completed++
|
|
state.turns = append(state.turns, turn.MemoryEntry{Stage: record.stage, Visit: state.visits[record.stage], Output: turnOutput(values), PublicFiles: recordPublicFiles(record, detail, publicSeen)})
|
|
state.lastOutput = turnOutput(values)
|
|
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(record.sequence)))
|
|
}
|
|
}
|
|
return state
|
|
}
|
|
|
|
// retryPrefix returns the completed prefix from the source attempt plus the
|
|
// immutable prefix carried by that attempt's checkpoint. A retry attempt only
|
|
// persists the selected stage onward, so immediate source steps are not a
|
|
// complete representation of its memory.
|
|
func (r *runner) retryPrefix(sourceID int64, source *protocol.RunDetail, failedSequence int) (map[string]any, []retryPrefixRecord, error) {
|
|
prefix := map[string]any{}
|
|
records := []retryPrefixRecord{}
|
|
inheritedPublic := map[string]bool{}
|
|
if source.RetryMode == "resume" && source.RetrySource > 0 {
|
|
checkpointSource, mode, _, inherited, err := r.store.RetryCheckpoint(r.ctx, sourceID)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("retry source %d has no immutable prefix checkpoint: %w", sourceID, err)
|
|
}
|
|
if checkpointSource != source.RetrySource || mode != "resume" {
|
|
return nil, nil, fmt.Errorf("retry source %d has an invalid prefix checkpoint", sourceID)
|
|
}
|
|
if values, ok := inherited["prefix"].(map[string]any); ok {
|
|
for key, value := range values {
|
|
prefix[key] = value
|
|
}
|
|
}
|
|
for _, record := range retryPrefixRecords(inherited["prefix_provenance"]) {
|
|
records = append(records, record)
|
|
for _, name := range record.public {
|
|
inheritedPublic[name] = true
|
|
}
|
|
}
|
|
}
|
|
publicBySequence := retryPublicBySequence(source, failedSequence)
|
|
for _, step := range source.Steps {
|
|
if step.Sequence >= failedSequence || step.Status != "completed" {
|
|
continue
|
|
}
|
|
prefix[step.Stage] = step.Output
|
|
added := make([]string, 0, len(publicBySequence[step.Sequence]))
|
|
for _, name := range publicBySequence[step.Sequence] {
|
|
// Carried prefix files reappear in this attempt's inventory but were
|
|
// added by an earlier turn, not this one.
|
|
if !inheritedPublic[name] {
|
|
added = append(added, name)
|
|
}
|
|
}
|
|
records = append(records, retryPrefixRecord{stage: step.Stage, sequence: step.Sequence, public: added, publicKnown: true})
|
|
}
|
|
return prefix, records, nil
|
|
}
|
|
|
|
func retryPrefixRecords(raw any) []retryPrefixRecord {
|
|
items, ok := raw.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
result := make([]retryPrefixRecord, 0, len(items))
|
|
for _, item := range items {
|
|
entry, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
stageName, ok := entry["stage"].(string)
|
|
if !ok || stageName == "" {
|
|
continue
|
|
}
|
|
sequence, ok := entry["sequence"].(float64)
|
|
if !ok || sequence < 1 {
|
|
continue
|
|
}
|
|
record := retryPrefixRecord{stage: stageName, sequence: int(sequence)}
|
|
if raw, ok := entry["public"]; ok {
|
|
record.public = stringValues(raw)
|
|
record.publicKnown = true
|
|
}
|
|
result = append(result, record)
|
|
}
|
|
return result
|
|
}
|
|
|
|
// stringValues accepts the JSON-decoded forms of a string list in a retry
|
|
// checkpoint and returns a copy; unknown element types are ignored.
|
|
func stringValues(raw any) []string {
|
|
switch values := raw.(type) {
|
|
case []string:
|
|
return append([]string(nil), values...)
|
|
case []any:
|
|
result := make([]string, 0, len(values))
|
|
for _, value := range values {
|
|
if text, ok := value.(string); ok {
|
|
result = append(result, text)
|
|
}
|
|
}
|
|
return result
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func retryPrefixStep(detail *protocol.RunDetail, record retryPrefixRecord) (protocol.RunStep, bool) {
|
|
for _, step := range detail.Steps {
|
|
if step.Stage == record.stage && step.Sequence == record.sequence {
|
|
return step, true
|
|
}
|
|
}
|
|
// Inherited rows intentionally are not present in a selected retry. The
|
|
// contract check still uses the stage key; this marker keeps that check
|
|
// independent from model-claimed output.
|
|
return protocol.RunStep{Stage: record.stage, Sequence: record.sequence}, true
|
|
}
|
|
|
|
// artifactAccess derives exact turn directories; it never consumes a path from
|
|
// a definition or model result. The current visit may read and write its own
|
|
// turn and the shared public directory. Earlier visits are not readable: their
|
|
// public files are already shared and their private/ files are never handed to
|
|
// a later stage. An unreadable or symlinked root fails closed.
|
|
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
|
|
_ = prior // prior private directories remain deliberately inaccessible
|
|
root, err := filepath.EvalSymlinks(r.artifactsRoot)
|
|
if err != nil {
|
|
return stage.ArtifactAccess{}
|
|
}
|
|
runRoot := filepath.Join(root, strconv.FormatInt(runID, 10))
|
|
directory := func(path string) (string, bool) {
|
|
if path == "" {
|
|
return "", false
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(path)
|
|
if err != nil || !strings.HasPrefix(resolved, runRoot+string(os.PathSeparator)) {
|
|
return "", false
|
|
}
|
|
info, err := os.Stat(resolved)
|
|
if err != nil || !info.IsDir() {
|
|
return "", false
|
|
}
|
|
return resolved, true
|
|
}
|
|
access := stage.ArtifactAccess{}
|
|
currentDir, ok := directory(current)
|
|
if !ok {
|
|
return access
|
|
}
|
|
// The current directory must be this visit's turn, not another stage's turn
|
|
// in the same run.
|
|
expectedPrefix := fmt.Sprintf("%02d-", sequence)
|
|
if filepath.Dir(currentDir) != filepath.Join(runRoot, "stages") || !strings.HasPrefix(filepath.Base(currentDir), expectedPrefix) {
|
|
return access
|
|
}
|
|
if private, ok := directory(filepath.Join(currentDir, "private")); ok {
|
|
// The model may write its own private/ files, never the server-owned
|
|
// input.md, memory.md or output.md. A missing private/ grants no write
|
|
// scope; falling back to the turn directory would widen the permission.
|
|
access.Write = private
|
|
}
|
|
access.Read = append(access.Read, currentDir)
|
|
if public, ok := directory(filepath.Join(runRoot, "public")); ok {
|
|
access.WriteShared = public
|
|
access.Read = append(access.Read, public)
|
|
}
|
|
return access
|
|
}
|
|
|
|
func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient {
|
|
return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
|
|
}
|
|
|
|
func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient {
|
|
return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
|
|
}
|
|
|
|
// serverChangedFilesField is the reserved run-input memory field under which the
|
|
// runner publishes the server-captured changed paths. A stage selects it with
|
|
// `"server_changed_files":"input.server_changed_files"`, alongside another
|
|
// stage's declared list, so it sees the verified evidence, not just a model's
|
|
// claim of authorship.
|
|
const serverChangedFilesField = "server_changed_files"
|
|
|
|
// capturedList returns the full, first-seen-ordered set of server-captured
|
|
// changed paths, or a non-nil empty list when nothing was captured so the field
|
|
// still resolves as a list. It copies the accumulator so the retained evidence
|
|
// cannot be aliased by a published value. O(captured paths).
|
|
func (s *runState) capturedList() []string {
|
|
if len(s.capturedPaths) == 0 {
|
|
return []string{}
|
|
}
|
|
return append([]string(nil), s.capturedPaths...)
|
|
}
|
|
|
|
// publishServerChangedFiles writes the run's captured changed-path set into the
|
|
// run input memory value under serverChangedFilesField, creating the entry and
|
|
// its output map when absent. Called before memoryInput so a stage selecting
|
|
// input.server_changed_files always resolves; only that field is added and only
|
|
// a selecting stage copies it out, so state.input and other stages' input are
|
|
// unchanged. O(captured paths).
|
|
func (s *runState) publishServerChangedFiles() {
|
|
entry, ok := s.memory["input"]
|
|
if !ok || entry.output == nil {
|
|
entry = memoryValue{output: stage.Values{}}
|
|
}
|
|
entry.output[serverChangedFilesField] = s.capturedList()
|
|
s.memory["input"] = entry
|
|
}
|
|
|
|
// runIdentity is the pipeline identity and stored title a run's pushed frames
|
|
// carry (RunEventFrame.PipelineID/Name/Title/Version). It is remembered where a
|
|
// run is created, decided or canceled and stamped onto every frame that leaves
|
|
// emitRun, so a client that has never seen the run can build its list row from
|
|
// the frame without re-reading the whole list.
|
|
type runIdentity struct {
|
|
scheduled bool
|
|
scheduleID int64
|
|
scheduleName string
|
|
pipelineID int64
|
|
name string
|
|
title string
|
|
version int
|
|
}
|
|
|
|
// identityOfRunSummary is the one projection of a run row onto the identity its
|
|
// pushed frames carry, so the frame and the list row cannot disagree.
|
|
func identityOfRunSummary(s protocol.RunSummary) runIdentity {
|
|
return runIdentity{scheduled: s.Scheduled, scheduleID: s.ScheduleID, scheduleName: s.ScheduleName, pipelineID: s.PipelineID, name: s.Name, title: s.Title, version: s.Version}
|
|
}
|
|
|
|
// rememberRunIdentity records the identity every later frame of run id carries.
|
|
// It is safe from any goroutine and takes its own lock (never r.mu), so it is
|
|
// callable from a path that already holds r.mu.
|
|
func (r *runner) rememberRunIdentity(id int64, ident runIdentity) {
|
|
r.identMu.Lock()
|
|
if r.idents == nil {
|
|
r.idents = map[int64]runIdentity{}
|
|
}
|
|
r.idents[id] = ident
|
|
r.identMu.Unlock()
|
|
}
|
|
|
|
// stampRunIdentity fills a frame's blank identity fields from the run's
|
|
// remembered identity, leaving any the emit site set untouched. A run with no
|
|
// remembered identity keeps them blank, so the client falls back to its resync.
|
|
func (r *runner) stampRunIdentity(frame *protocol.RunEventFrame) {
|
|
r.identMu.Lock()
|
|
ident, ok := r.idents[frame.RunID]
|
|
r.identMu.Unlock()
|
|
if !ok {
|
|
return
|
|
}
|
|
frame.Scheduled = ident.scheduled
|
|
frame.ScheduleID = ident.scheduleID
|
|
frame.ScheduleName = ident.scheduleName
|
|
if frame.PipelineID == 0 {
|
|
frame.PipelineID = ident.pipelineID
|
|
}
|
|
if frame.Name == "" {
|
|
frame.Name = ident.name
|
|
}
|
|
if frame.Title == "" {
|
|
frame.Title = ident.title
|
|
}
|
|
if frame.Version == 0 {
|
|
frame.Version = ident.version
|
|
}
|
|
}
|
|
|
|
// emitRun pushes one run delta to the subscribed-client hub when one is wired,
|
|
// and does nothing otherwise (a runner built directly in a test has no hub).
|
|
// terminal marks a final transition, which the hub flushes without coalescing.
|
|
func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
|
|
// A run that is no longer waiting has no decision pending: its waiting
|
|
// notifications are cleared on every device.
|
|
if r.notes != nil && frame.Status != "" && frame.Status != protocol.RunStatusWaiting {
|
|
c, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
r.notes.clearFor(c, pipeline.AppNotificationFilter{RunID: frame.RunID, Kinds: []string{protocol.NotificationRunWaiting}})
|
|
cancel()
|
|
}
|
|
if r.events == nil {
|
|
return
|
|
}
|
|
r.stampRunIdentity(&frame)
|
|
frame.NotifyMode = r.notifyMode(frame.RunID)
|
|
r.events.emit(frame, terminal)
|
|
}
|
|
|
|
func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) {
|
|
r.mu.Lock()
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if status == "succeeded" {
|
|
if err := r.finalizeDeliverable(c, id, sequence, input); err != nil {
|
|
status, outcome, diagnostic = "failed", "error", "deliverable contract: "+err.Error()
|
|
}
|
|
}
|
|
// Measure the run's footprint while it is still recorded as active: its
|
|
// files are final, and measuring after the terminal status is written would
|
|
// widen the window in which a client sees the run finished but still owned.
|
|
// The status write re-marks the cache stale, so the background sizer then
|
|
// records the exact value including the final output row.
|
|
size := r.finishedSize(id)
|
|
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
|
|
log.Printf("finish run %d: %v", id, err)
|
|
}
|
|
// The store atomically makes a stop request win over a late success.
|
|
detail, getErr := r.store.GetRun(c, id)
|
|
if getErr != nil || detail == nil {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
status, outcome, diagnostic = detail.Status, detail.Outcome, detail.Diagnostic
|
|
r.mu.Unlock()
|
|
if detail.FinishedAt == "" {
|
|
return // Cleanup is not confirmed: never emit a false terminal event.
|
|
}
|
|
// Push the terminal delta before the attention notification, so a slow ntfy
|
|
// never delays the live update the app folds.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: status,
|
|
Outcome: outcome,
|
|
CurrentStage: currentStage,
|
|
CompletedSteps: completed,
|
|
FinishedAt: detail.FinishedAt,
|
|
SizeBytes: size,
|
|
}, true)
|
|
r.kickSizes()
|
|
if status != protocol.RunStatusCanceled {
|
|
r.notifyTerminalStep(id, status)
|
|
}
|
|
r.deliverRunNotificationBatch(time.Now(), 1)
|
|
// The run is terminal: nothing more can notify for it.
|
|
r.forgetAttention(id)
|
|
r.forgetNotifyMode(id)
|
|
}
|
|
|
|
// completedSteps counts the completed steps of a run detail; resume uses it to
|
|
// seed the terminal delta without another store read.
|
|
func completedSteps(steps []protocol.RunStep) int {
|
|
n := 0
|
|
for _, step := range steps {
|
|
if step.Status == "completed" {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// stageUsage carries the usage a model stage already measured for budget
|
|
// accounting onto the run record. It reports a cost only when the provider
|
|
// reported one and an in/out split only when the harness reported it, so an
|
|
// unknown cost or split is never presented as zero. It returns nil when no usage
|
|
// was observed, so the step stays absent rather than zero.
|
|
func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage {
|
|
if !snapshot.TokensKnown && !snapshot.USDKnown {
|
|
return nil
|
|
}
|
|
usage := &protocol.StageUsage{TokensKnown: snapshot.TokensKnown, CostKnown: snapshot.USDKnown}
|
|
if snapshot.TokensKnown {
|
|
usage.TokensTotal = snapshot.UsedTokens
|
|
if snapshot.SplitKnown {
|
|
usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true
|
|
// The rest of the component split is kept too: for a caching harness
|
|
// most of the total is cache reads, which a client shows as cached
|
|
// rather than fresh tokens.
|
|
usage.TokensReasoning, usage.CacheRead, usage.CacheWrite, usage.CacheKnown = snapshot.TokensReasoning, snapshot.CacheRead, snapshot.CacheWrite, true
|
|
}
|
|
}
|
|
if snapshot.USDKnown {
|
|
usage.CostUSD = snapshot.UsedUSD
|
|
}
|
|
return usage
|
|
}
|
|
|
|
// finishStage persists one step, refreshes the shared memory for a completed
|
|
// step, notifies and classifies the step's target. Both the executed path and
|
|
// the skipped path use it, so persistence, notification and terminal routing
|
|
// cannot drift between them. A persist or memory-copy failure is returned for
|
|
// the caller to fail the run; the caller still owns the model-specific
|
|
// verdict, quiescence and cancellation handling around it.
|
|
// turnOutputOnly reports the turn protocol's single markdown body. The verdict
|
|
// summariser is written for the old field map (summary/option/reason); given a
|
|
// document it treats it as untrusted reference material and invents a reason.
|
|
func turnOutputOnly(output stage.Values) bool {
|
|
if len(output) != 1 {
|
|
return false
|
|
}
|
|
body, ok := output["markdown"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
_, isText := body.(string)
|
|
return isText
|
|
}
|
|
|
|
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
|
|
r.mu.Lock()
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
detail, getErr := r.store.GetRun(save, id)
|
|
if getErr != nil {
|
|
cancel()
|
|
r.mu.Unlock()
|
|
return "", "", "", false, getErr
|
|
}
|
|
if detail.CancelRequested {
|
|
step.Status, step.Outcome, step.Target = "canceled", "", ""
|
|
step.Diagnostic = "Canceled by operator; execution returned"
|
|
}
|
|
persistErr := r.store.FinishStep(save, id, step)
|
|
cancel()
|
|
r.mu.Unlock()
|
|
if persistErr != nil {
|
|
return "", "", "", false, persistErr
|
|
}
|
|
if detail.CancelRequested {
|
|
return "canceled", "", step.Diagnostic, true, nil
|
|
}
|
|
if step.Status == "completed" {
|
|
values, copyErr := copyValues(stage.Values(step.Output))
|
|
if copyErr != nil {
|
|
return "", "", "", false, copyErr
|
|
}
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
state.input = stage.Values(step.Output)
|
|
status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
if !terminal {
|
|
r.notifyStep(id, name, workspace, selected.Notify, step)
|
|
}
|
|
return status, outcome, diagnostic, terminal, nil
|
|
}
|
|
|
|
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
|
|
finish := func(status, outcome, diagnostic string) {
|
|
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
|
|
}
|
|
// canceled settles an operator cancel. A server shutdown is not a cancel:
|
|
// the run is left exactly as persisted for the next process to continue.
|
|
canceled := func() {
|
|
if stage.Detached(ctx) {
|
|
log.Printf("run %d: server stopping; run left in flight for the next start to continue", id)
|
|
return
|
|
}
|
|
finish("canceled", "", ctx.Err().Error())
|
|
}
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
finish("failed", "error", "stage execution panicked")
|
|
}
|
|
}()
|
|
budgets := map[string]*stage.Budget{}
|
|
if state.key == "" {
|
|
finish("failed", "error", "stage returned an undeclared outcome")
|
|
return
|
|
}
|
|
// Every process started for this run (stages, the stage service, checks,
|
|
// captures) gets the workspace's own environment, as a terminal there would.
|
|
ctx = r.withRunEnvironment(ctx, id, workspace)
|
|
// Resolve the effective provider/model for every model stage once, before the
|
|
// loop, so applying an override is an in-memory O(1) map lookup per visit and
|
|
// never re-parses the graph or re-resolves the whole definition. A run with no
|
|
// overrides skips the resolution entirely. Launch validation already ran this
|
|
// helper, so a failure here only guards a resumed run whose overrides drifted.
|
|
var effectiveModels map[string]pipeline.ModelOverride
|
|
if request.Model != "" || len(request.StageModels) > 0 {
|
|
resolved, err := pipeline.EffectiveModels(graph, request.Model, request.StageModels)
|
|
if err != nil {
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
effectiveModels = resolved
|
|
}
|
|
for sequence := state.nextSequence; sequence <= 30; sequence++ {
|
|
if ctx.Err() != nil {
|
|
canceled()
|
|
return
|
|
}
|
|
key := state.key
|
|
var def stage.Definition
|
|
var selected protocol.Stage
|
|
found := false
|
|
for _, s := range graph.Stages {
|
|
if s.Key == key {
|
|
selected = s
|
|
def = pipeline.StageDefinition(s)
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
finish("failed", "error", "missing stage")
|
|
return
|
|
}
|
|
// StageDefinition clones Configs, so setting the effective pair on the
|
|
// clone is safe and the pinned graph is never mutated. Because
|
|
// ModelStage.Spec reads def.Configs and passes spec.Config into the repair
|
|
// path, the same effective model governs the stage and its repair.
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
if override, ok := effectiveModels[key]; ok {
|
|
def.Configs["provider"] = override.Provider
|
|
def.Configs["model"] = override.Model
|
|
}
|
|
}
|
|
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
|
|
maxVisits := 5
|
|
if selected.Limits != nil {
|
|
maxVisits = selected.Limits.MaxVisits
|
|
}
|
|
// Validate the next visit without consuming it. An already-exhausted
|
|
// stage must not begin work it can never execute.
|
|
if !isHuman {
|
|
if state.visits[key]+1 > 5 {
|
|
finish("failed", "error", "stage visit budget exceeded")
|
|
return
|
|
}
|
|
if state.visits[key]+1 > maxVisits {
|
|
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
|
|
return
|
|
}
|
|
}
|
|
// adopted is the visit a previous server process began and left running
|
|
// across its restart: its step row, turn files and session already exist.
|
|
adopted := state.adopt
|
|
if adopted != nil && adopted.sequence != sequence {
|
|
adopted = nil
|
|
}
|
|
state.adopt = nil
|
|
_, declaresSkip := def.Edges["skipped"]
|
|
skip := adopted == nil && !isHuman && declaresSkip && autoSkipEligible(def.Configs, state)
|
|
state.nextSequence = sequence + 1
|
|
state.visits[key]++
|
|
// Publish the run's server-captured changed paths into the input memory
|
|
// value before resolving bindings, so a stage selecting
|
|
// input.server_changed_files always resolves (an empty list when nothing
|
|
// was captured). Only a selecting stage copies it into its input.
|
|
state.publishServerChangedFiles()
|
|
var reads map[string]protocol.MemoryRead
|
|
bindings := selected.Memory
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
// A model stage reads the turn's memory.md and input.md files; the
|
|
// retired per-field memory block no longer selects its input.
|
|
bindings = nil
|
|
}
|
|
input, reads, copyErr := memoryInput(state.input, bindings, state.memory)
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
if changed := state.capturedList(); stage.EffectiveType(def.Type) == stage.ModelType && len(changed) > 0 {
|
|
// The turn files, not a memory binding, carry the stage context. The
|
|
// prompt reads this reserved value so a committer still sees the
|
|
// server-captured paths that distinguish this run from pre-existing
|
|
// dirty work. An empty list adds nothing and leaves the entry input
|
|
// unchanged.
|
|
input[serverChangedFilesField] = changed
|
|
}
|
|
var retryInput stage.Values
|
|
if state.retryOverride != nil {
|
|
input, copyErr = retryInputOverride(input, state.retryOverride)
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
// Keep the merged values for this visit's input.md: the model reads
|
|
// files, not the persisted step row, so the override must be visible
|
|
// there. A selected-stage override belongs only to this invocation,
|
|
// not to a later visit reached through the stage graph.
|
|
retryInput = input
|
|
state.retryOverride = nil
|
|
}
|
|
if adopted == nil {
|
|
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
|
if ctx.Err() != nil {
|
|
canceled()
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
}
|
|
// queued -> running: report the new current stage and the step start.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
|
|
}, false)
|
|
if isHuman {
|
|
// A human step is O(1): one BeginStep plus one MarkWaiting, no
|
|
// filediff capture, no artifacts directory and no workspace mutation.
|
|
evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges))
|
|
r.mu.Lock()
|
|
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
|
|
r.mu.Unlock()
|
|
if ctx.Err() != nil {
|
|
canceled()
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence}
|
|
// A human step parks the run without a live goroutine; the delta is
|
|
// coalesced by the hub's timer, which keeps running after this returns.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusWaiting,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
|
|
}, false)
|
|
// Publishing can be slow. Once waiting is visible, release execution
|
|
// atomically so a decision is not spuriously rejected as "active".
|
|
delete(r.active, id)
|
|
r.mu.Unlock()
|
|
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
|
|
r.deliverRunNotificationBatch(time.Now(), 1)
|
|
return
|
|
}
|
|
// Server skip fallback: a stage that declares a `skipped` route is
|
|
// short-circuited only when the run's evidence is complete and empty
|
|
// (autoSkipEligible, ui_surface.go): every visit recorded a complete
|
|
// capture with no file changes and no rendered artifact. Then skip the
|
|
// model call and its capture work, record the step as completed with a
|
|
// verdict that names the server fallback, and route along the declared
|
|
// edge. Partial, unavailable or missing evidence always runs the model.
|
|
if skip {
|
|
reason := skipReasonLine()
|
|
step := protocol.RunStep{
|
|
Sequence: sequence,
|
|
Stage: key,
|
|
Status: "completed",
|
|
Outcome: "skipped",
|
|
Output: stage.Values{"summary": reason},
|
|
Target: def.Edges["skipped"],
|
|
Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads},
|
|
Verdict: &protocol.StageVerdict{Reason: reason, ReasonSource: protocol.VerdictReasonFallback},
|
|
}
|
|
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
|
|
if finishErr != nil {
|
|
finish("failed", "error", finishErr.Error())
|
|
return
|
|
}
|
|
if terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
continue
|
|
}
|
|
var tools []string
|
|
explicitTools := false
|
|
var toolErr error
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
tools, explicitTools, toolErr = stage.ToolNames(def)
|
|
}
|
|
if toolErr != nil {
|
|
finish("failed", "error", toolErr.Error())
|
|
return
|
|
}
|
|
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
|
|
// Every model visit gets a file-based turn: input.md, memory.md and
|
|
// history.md are written before execution, output.md after it. Creation is part of the
|
|
// protocol, so an unusable turn directory fails the run instead of
|
|
// silently degrading to the retired JSON envelope.
|
|
artifactsDir := ""
|
|
var visitTurn turn.Paths
|
|
var publicBefore map[string]bool
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
visitTurn = r.turnPaths(id, sequence, key)
|
|
if adopted == nil {
|
|
if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil {
|
|
finish("failed", "error", turnErr.Error())
|
|
return
|
|
}
|
|
}
|
|
artifactsDir = visitTurn.TurnDir
|
|
def.ArtifactsDir = artifactsDir
|
|
def.Turn = stageTurnPaths(visitTurn)
|
|
publicBefore = publicFileSet(visitTurn.RunDir)
|
|
}
|
|
operations := &stage.OperationLog{}
|
|
var landing stage.LandingAccess
|
|
delivery, deliveryErr := stage.DeliveryEnabled(def)
|
|
if deliveryErr != nil {
|
|
finish("failed", "error", deliveryErr.Error())
|
|
return
|
|
}
|
|
if delivery {
|
|
access, accessErr := landingAccess(workspace)
|
|
if accessErr != nil {
|
|
finish("failed", "error", "delivery access denied: "+accessErr.Error())
|
|
return
|
|
}
|
|
landing = access
|
|
}
|
|
client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools)
|
|
client.LandingAccess = landing
|
|
client.OnSession = func(session string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
|
|
}
|
|
var result stage.Result
|
|
var piClient *stage.PiClient
|
|
var err error
|
|
stageCtx := ctx
|
|
if adopted != nil {
|
|
// Attach to the surviving session instead of prompting again.
|
|
client.Backend, client.Reference = adopted.owner.Backend, adopted.owner.Reference
|
|
stageCtx = stage.WithResumeSession(stageCtx, adopted.owner.ID)
|
|
}
|
|
var modelConfig stage.ModelConfig
|
|
var budgetBefore stage.BudgetSnapshot
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
spec, specErr := (stage.ModelStage{}).Spec(def)
|
|
if specErr != nil {
|
|
finish("failed", "error", specErr.Error())
|
|
return
|
|
}
|
|
modelConfig = spec.Config
|
|
if budgets[key] == nil {
|
|
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
|
if used, ok := state.priorBudgets[key]; ok {
|
|
budgets[key].RestoreUsage(used)
|
|
}
|
|
}
|
|
// The budget is shared across every visit to this stage key, so capture
|
|
// its cumulative usage before the visit and record only this visit's
|
|
// increment on the step, never the cumulative total again.
|
|
budgetBefore = budgets[key].Snapshot()
|
|
stageCtx = stage.WithBudget(stageCtx, budgets[key])
|
|
}
|
|
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
|
before := filediff.Capture(snapshotCtx, workspace)
|
|
snapshotCancel()
|
|
var stageSession string
|
|
panicked := false
|
|
func() {
|
|
defer func() {
|
|
if recover() != nil {
|
|
panicked = true
|
|
err = fmt.Errorf("stage execution panicked")
|
|
}
|
|
}()
|
|
if ctx.Err() != nil {
|
|
err = ctx.Err()
|
|
return
|
|
}
|
|
if r.execute != nil {
|
|
timeout, _ := stage.StageTimeout(def)
|
|
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
|
|
defer cancel()
|
|
result, err = r.execute(executeCtx, def, input)
|
|
} else if modelConfig.Harness == "pi" {
|
|
configured, configErr := harness.Load()
|
|
if configErr != nil {
|
|
err = configErr
|
|
return
|
|
}
|
|
profile, ok := configured.Profiles["pi"]
|
|
if !ok {
|
|
err = fmt.Errorf("Pi is not configured on execution server")
|
|
return
|
|
}
|
|
piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools)
|
|
piClient.OnOwnership = func(session, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
|
|
stageSession = piClient.SessionID
|
|
client.Transcript = piClient.Transcript
|
|
} else if useAgent {
|
|
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
|
|
stageSession = client.SessionID
|
|
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
|
|
if adopted != nil {
|
|
plain.Backend, plain.Reference = adopted.owner.Backend, adopted.owner.Reference
|
|
}
|
|
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if active {
|
|
stageSession = session
|
|
}
|
|
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
|
|
client.Transcript = plain.FormatAudit
|
|
} else {
|
|
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
|
|
}
|
|
}()
|
|
if r.detach && stage.Detached(ctx) {
|
|
// The server is stopping. The stage left its session running and
|
|
// owned (or never admitted one); nothing about this visit is settled
|
|
// here. The next process re-adopts the session and continues.
|
|
log.Printf("run %d stage %s #%d: server stopping; stage left running for the next start to reattach", id, key, sequence)
|
|
return
|
|
}
|
|
ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id)
|
|
ownershipCancel()
|
|
if ownershipErr != nil || len(activeSessions) > 0 {
|
|
err = errors.Join(stage.ErrStageCleanup, err, ownershipErr)
|
|
}
|
|
if ctx.Err() != nil && !errors.Is(err, stage.ErrStageCleanup) {
|
|
err = ctx.Err() // Ignore a context-unaware executor's late success.
|
|
}
|
|
if stage.EffectiveType(def.Type) == stage.ModelType && err == nil {
|
|
ensureTurnOutput(visitTurn, result)
|
|
}
|
|
// Capture after cleanup, even when the stage's context was canceled.
|
|
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
after := filediff.Capture(snapshotCtx, workspace, before)
|
|
snapshotCancel()
|
|
changes := filediff.Compare(before, after)
|
|
if adopted != nil && changes.Status == "complete" {
|
|
// The pre-visit snapshot belonged to the stopped process; only the
|
|
// changes made after reattaching were compared.
|
|
changes.Status = "partial"
|
|
changes.Diagnostic = "Server restarted during this stage; changes made before the restart are not listed."
|
|
}
|
|
if errors.Is(err, stage.ErrStageCleanup) {
|
|
changes.Status = "partial"
|
|
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
if markErr := r.store.MarkCleanupUnconfirmed(save, id); markErr != nil {
|
|
log.Printf("mark cleanup unconfirmed for run %d: %v", id, markErr)
|
|
}
|
|
cancel()
|
|
}
|
|
// Only settled, exact recorded ownership can satisfy the handoff gate.
|
|
// A canceled execution must not cancel this ownership read silently.
|
|
var sessionExport *protocol.SessionExport
|
|
exportRequired := artifactsDir != "" && r.execute == nil
|
|
if artifactsDir != "" {
|
|
ownerCtx, ownerCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
owner, ownerErr := r.store.RepairSessionOwner(ownerCtx, id, sequence)
|
|
if ownerErr != nil && r.execute != nil {
|
|
// Only genuinely unowned fake executors are exempt. A recorded
|
|
// session with absent ownership metadata must still fail closed.
|
|
if recorded, readErr := r.store.RunSessionOwner(ownerCtx, id, sequence); readErr != nil || recorded.ID != "" {
|
|
exportRequired = true
|
|
if readErr == nil {
|
|
stageSession = recorded.ID
|
|
}
|
|
}
|
|
}
|
|
ownerCancel()
|
|
if ownerErr == nil && owner.ID != "" {
|
|
exportRequired = true // Owned test executors exercise the real gate too.
|
|
stageSession = owner.ID
|
|
if !errors.Is(err, stage.ErrStageCleanup) {
|
|
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
|
|
}
|
|
} else if exportRequired {
|
|
sessionExport = &protocol.SessionExport{Status: "failed", Source: "native_export", Diagnostic: "exact settled session ownership unavailable; handoff refused"}
|
|
}
|
|
}
|
|
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir, visitTurn.PublicDir)
|
|
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, SessionExport: sessionExport, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
|
|
// Stamp the effective provider/model this visit actually ran with for every
|
|
// model stage, including failures, so the app/TUI evidence reflects any
|
|
// override. Non-model stages keep empty values. The usage the stage already
|
|
// measured for budget accounting is carried onto the same step, never
|
|
// re-read from the harness or parsed from the transcript.
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
step.Provider = modelConfig.Provider
|
|
step.Model = modelConfig.Model
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Usage = stageUsage(budget.Snapshot().Since(budgetBefore))
|
|
}
|
|
}
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
if len(result.Output) > 0 {
|
|
step.Output["salvage"] = result.Output
|
|
}
|
|
}
|
|
if err == nil && result.Outcome == "success" {
|
|
target, targetErr := stage.VerificationTargetFor(def)
|
|
if targetErr != nil {
|
|
err = targetErr
|
|
} else if target != nil {
|
|
if r.verifyTarget == nil {
|
|
err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
|
|
} else {
|
|
evidence, verificationErr := r.verifyTarget(stageCtx, *target)
|
|
if verificationErr != nil {
|
|
err = verificationErr
|
|
} else {
|
|
err = stage.VerifyTargetEvidence(*target, evidence)
|
|
}
|
|
}
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
}
|
|
}
|
|
if err == nil && delivery && result.Outcome == "success" {
|
|
checks, checksErr := stage.DeliveryChecks(def)
|
|
if checksErr != nil {
|
|
err = checksErr
|
|
} else {
|
|
gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout)
|
|
var catchUp string
|
|
catchUp, err = verifyDelivery(gateCtx, workspace, landing, checks)
|
|
gateCancel()
|
|
// A failed gate already carries the note in its diagnostic. A step
|
|
// diagnostic on success renders as a stop, so success is logged.
|
|
if catchUp != "" {
|
|
log.Printf("run %d stage %s: %s", id, key, catchUp)
|
|
}
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
}
|
|
}
|
|
// A failed or timed-out step carries the session pointer even when the
|
|
// bounded transcript would otherwise be only its stage_limits envelope.
|
|
if err != nil && stageSession != "" {
|
|
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
|
|
}
|
|
step.Target = def.Edges[step.Outcome]
|
|
if errors.Is(err, stage.ErrStageCleanup) {
|
|
// Retain the partial evidence without pretending that the agent stopped.
|
|
step.Status, step.Outcome, step.Target = "running", "", ""
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
if saveErr := r.store.FinishStep(save, id, step); saveErr != nil {
|
|
log.Printf("save cleanup evidence for run %d: %v", id, saveErr)
|
|
}
|
|
cancel()
|
|
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CleanupUnconfirmed: true}, false)
|
|
return
|
|
}
|
|
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
|
|
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
|
|
for _, outcome := range selected.Limits.On {
|
|
if outcome == step.Outcome {
|
|
step.Control.LimitReached = true
|
|
step.Control.ModelTarget = step.Target
|
|
step.Target = protocol.FailureEnd
|
|
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
|
|
}
|
|
}
|
|
}
|
|
if panicked {
|
|
step.Target = protocol.FailureEnd
|
|
}
|
|
_, _, _, failedTerminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
if exportRequired && !completeSessionExport(sessionExport) && !(err != nil && failedTerminal && step.Target != protocol.SuccessEnd) {
|
|
// Preserve the real computation, files, outcome and target. In
|
|
// particular an export failure is not an agent failure/retry request.
|
|
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
saveErr := r.store.FinishStepExportBlocked(save, id, step)
|
|
cancel()
|
|
if saveErr != nil {
|
|
finish("failed", "error", saveErr.Error())
|
|
return
|
|
}
|
|
// No finishStage, memory publication, recordTurn or next executor.
|
|
check, done := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
detail, readErr := r.store.GetRun(check, id)
|
|
done()
|
|
if readErr != nil {
|
|
log.Printf("read parked export gate for run %d: %v", id, readErr)
|
|
return // The checkpoint is durable; never fabricate a terminal.
|
|
}
|
|
if detail.CancelRequested && detail.ExportPendingSequence == 0 {
|
|
finish("canceled", "", "Canceled by operator; computed visit retained")
|
|
return
|
|
}
|
|
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, ExportPendingSequence: sequence, CurrentStage: key, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts)}}, true)
|
|
return
|
|
}
|
|
// Generate the one-line verdict reason once, at completion. It is
|
|
// best-effort: a summariser failure or timeout returns NoSummary and must
|
|
// never fail the stage or the run, so every error is swallowed. A failed
|
|
// step still gets a verdict from its error envelope. A turn output is one
|
|
// markdown body, which the field-map summariser reads as untrusted data and
|
|
// answers with an invented rationale; those steps get the honest read-time
|
|
// fallback instead.
|
|
if r.verdict != nil && !turnOutputOnly(step.Output) {
|
|
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
|
|
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
|
|
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
|
|
}
|
|
reasonCancel()
|
|
}
|
|
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
|
|
if finishErr != nil {
|
|
finish("failed", "error", finishErr.Error())
|
|
return
|
|
}
|
|
if ctx.Err() != nil {
|
|
canceled()
|
|
return
|
|
}
|
|
// Advance the in-memory UI-surface evidence only after the step is
|
|
// persisted, so the skip guard agrees with what the store holds.
|
|
state.recordChanges(step.FileChanges, step.Artifacts)
|
|
if step.Status == "completed" {
|
|
added := []string(nil)
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
added = addedPublicFiles(publicBefore, publicFileSet(visitTurn.RunDir))
|
|
}
|
|
state.recordTurn(step, added)
|
|
state.completed++
|
|
if artifactsDir != "" {
|
|
state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence))
|
|
}
|
|
}
|
|
// Step finished: report its outcome, routing target, diagnostic and the
|
|
// artifacts recorded for the visit, plus the run's running step count.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Artifacts: len(step.Artifacts),
|
|
Step: &protocol.RunStepDelta{
|
|
Sequence: step.Sequence,
|
|
Stage: step.Stage,
|
|
Status: step.Status,
|
|
Outcome: step.Outcome,
|
|
Target: step.Target,
|
|
Diagnostic: step.Diagnostic,
|
|
Artifacts: len(step.Artifacts),
|
|
},
|
|
}, false)
|
|
if ctx.Err() != nil {
|
|
canceled()
|
|
return
|
|
}
|
|
if terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
}
|
|
finish("failed", "error", "total stage budget exceeded")
|
|
}
|
|
|
|
// exportStageSession writes the finished stage's session into its own artifacts
|
|
// directory and returns verified evidence. The caller enforces the handoff gate;
|
|
// an export failure never replaces the computation's result.
|
|
func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport {
|
|
evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend, Directory: owner.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339)}
|
|
defer classifySessionExport(evidence)
|
|
if owner.Backend != "v1" && owner.Backend != "v2" && !(owner.Backend == "pi" && owner.Reference != "") {
|
|
evidence.Source = "unavailable"
|
|
evidence.Diagnostic = "session export unavailable: harness has no native export path"
|
|
return evidence
|
|
}
|
|
export := r.exportSession
|
|
if export == nil {
|
|
export = stage.ExportSession
|
|
}
|
|
// A canceled stage still gets one bounded, non-blocking attempt; the export
|
|
// never consumes the stage timeout and never waits for the session.
|
|
exportCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stage.SessionExportTimeout)
|
|
defer cancel()
|
|
result, err := export(exportCtx, stage.SessionExportRequest{
|
|
SessionID: owner.ID,
|
|
Backend: owner.Backend,
|
|
Reference: owner.Reference,
|
|
Directory: owner.Directory,
|
|
ArtifactsDir: artifactsDir,
|
|
RunID: runID,
|
|
Sequence: sequence,
|
|
Stage: key,
|
|
Provider: modelConfig.Provider,
|
|
Model: modelConfig.Model,
|
|
})
|
|
evidence.Attempts = result.Attempts
|
|
if err != nil {
|
|
evidence.Source = "native_export"
|
|
evidence.Diagnostic = stage.RedactEvidence(err.Error())
|
|
return evidence
|
|
}
|
|
evidence.Source = result.Source
|
|
evidence.Status = result.Status
|
|
evidence.HTML = result.HTML
|
|
evidence.HTMLBytes = result.HTMLBytes
|
|
evidence.Messages = result.Messages
|
|
evidence.Truncated = result.Truncated
|
|
evidence.Diagnostic = result.Diagnostic
|
|
if result.Status != "" && result.Status != "complete" && evidence.Diagnostic == "" {
|
|
evidence.Diagnostic = "native session export reported " + result.Status
|
|
}
|
|
verifySessionExportFile(evidence, artifactsDir)
|
|
return evidence
|
|
}
|
|
|
|
// setTitle renames a run: it replaces the stored operator-facing title without
|
|
// touching the run's input, steps or execution, then returns the updated detail.
|
|
func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) {
|
|
explicit, ok := summarise.ExplicitTitle(title)
|
|
if !ok {
|
|
return nil, fmt.Errorf("a title with usable text is required")
|
|
}
|
|
if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil {
|
|
return nil, err
|
|
}
|
|
return r.getRun(id)
|
|
}
|