3bfb3d6b44
A run stays in r.active after finishRunState persists its terminal status until its goroutine finishes notifying, so a decision in that window was refused as "already active". decide now refuses only runs whose terminal state is not yet durable; a finished run gets ResolveDecision's already-decided / not-waiting answer, and an executing run is still refused. Tests overwrote runner.artifactsRoot after Handler/newRunner had started background goroutines (sizeLoop, session export jobs) that read it. The root is now fixed at construction through newRunnerAt, which HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it there instead of mutating the field afterwards.
2164 lines
85 KiB
Go
2164 lines
85 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/filediff"
|
|
"git.a-wan.net/alan/pipelines/internal/harness"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
"git.a-wan.net/alan/pipelines/internal/turn"
|
|
)
|
|
|
|
// activeRun records an in-flight or cleanup-unconfirmed run's cancellation seam
|
|
// and resolved workspace reservation. A single map is the active-run state, so the
|
|
// workspace cannot drift from the cancel function.
|
|
type activeRun struct {
|
|
cancel context.CancelFunc
|
|
workspace string
|
|
generation *byte
|
|
}
|
|
|
|
type runner struct {
|
|
ctx context.Context
|
|
store *pipeline.Store
|
|
artifactsRoot string
|
|
mu sync.Mutex
|
|
active map[int64]activeRun
|
|
execute func(context.Context, stage.Definition, stage.Values) (stage.Result, error)
|
|
|
|
exportRepairMu sync.Mutex
|
|
|
|
// title generates the operator-facing run title from the run prompt. When nil
|
|
// the store derives a fallback from the prompt with no model call, so tests
|
|
// and unconfigured servers stay deterministic.
|
|
title func(context.Context, string) summarise.Title
|
|
// verdict generates the bounded one-line reason for a completed stage visit
|
|
// from its output map. When nil the store derives the reason at read time with
|
|
// no model call. It is best-effort: any failure must never fail the stage.
|
|
verdict func(context.Context, map[string]any) string
|
|
// events pushes run state deltas to subscribed clients. When nil (tests that
|
|
// build a runner directly) emits are dropped, so nothing else changes.
|
|
events *runEventHub
|
|
// notes records and streams app notifications beside every ntfy
|
|
// publication. Nil (a runner built directly in a test) records nothing.
|
|
notes *appNotifier
|
|
// identMu guards idents: the pipeline identity and stored title (see
|
|
// runIdentity) that every emitted frame of a run carries, so a client with no
|
|
// cached row renders a brand-new run without re-reading the list. Populated
|
|
// where a run is created, decided or canceled; a run with no remembered
|
|
// identity keeps blank identity fields and the client resyncs once instead.
|
|
identMu sync.Mutex
|
|
idents map[int64]runIdentity
|
|
// exportSession writes a finished stage's session into its own artifacts
|
|
// directory. It defaults to stage.ExportSession; a failure is best-effort and
|
|
// never changes the stage's status, outcome, target or result validation.
|
|
exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error)
|
|
// verifyTarget is installed only by server wiring for a controlled fixture
|
|
// workflow. Leaving it nil fails closed for target-aware definitions.
|
|
verifyTarget stage.VerificationRunner
|
|
wg sync.WaitGroup
|
|
schedulerWG sync.WaitGroup
|
|
// attentionMu guards attentionSent, the per-run set of run-attention events
|
|
// already delivered — by a stage's own notify policy or by the run-level
|
|
// default — so neither republishes the same genuine event.
|
|
attentionMu sync.Mutex
|
|
attentionSent map[int64]map[string]bool
|
|
// notifyMu guards notifyModes: the per-run auto/on/off override recorded at
|
|
// admission. It is separate from the stage's own notify policy.
|
|
notifyMu sync.Mutex
|
|
notifyModes map[int64]string
|
|
// sizeKick wakes the background run-size backfill (sizeLoop); nil when the
|
|
// loop is not running (a runner built directly in a test).
|
|
sizeKick chan struct{}
|
|
}
|
|
|
|
func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
|
return newRunnerAt(ctx, s, "")
|
|
}
|
|
|
|
// newRunnerAt builds a runner whose stage artifacts live under artifactsRoot
|
|
// ("" means <store directory>/artifacts). artifactsRoot is fixed before any
|
|
// background goroutine (session export jobs, notifications, the size backfill)
|
|
// starts and is never written afterwards, so those goroutines read it without
|
|
// locking.
|
|
func newRunnerAt(ctx context.Context, s *pipeline.Store, artifactsRoot string) (*runner, error) {
|
|
if err := s.AcquireRunnerLock(); err != nil {
|
|
return nil, err
|
|
}
|
|
// Store.Open is also used by offline tools, whose cwd is not execution
|
|
// authority. Bind imported legacy defaults only here, before serving requests
|
|
// or starting scheduler workers, and never rebind an explicit saved folder.
|
|
workspace, err := pipeline.ResolveRunWorkspace("", false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err = s.BindLegacyScheduleWorkspace(ctx, workspace); err != nil {
|
|
return nil, err
|
|
}
|
|
sessions, err := s.ActiveSessionOwners(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, owner := range sessions {
|
|
stop, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
var err error
|
|
if owner.Backend == "pi" {
|
|
err = stage.InterruptPi(stop, owner.Reference)
|
|
} else {
|
|
err = (stage.OpenCodeClient{Backend: owner.Backend, Directory: owner.Directory, Reference: owner.Reference}).InterruptAndConfirm(stop, owner.ID)
|
|
}
|
|
cancel()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot interrupt previously active agent %s: %w", owner.ID, err)
|
|
}
|
|
if err := s.ConfirmRecoveredSession(ctx, owner); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if err := s.RecoverRuns(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
root := artifactsRoot
|
|
if root == "" {
|
|
root = filepath.Join(s.Directory(), "artifacts")
|
|
}
|
|
active := map[int64]activeRun{}
|
|
rows, err := s.ListRuns(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, row := range rows {
|
|
if row.CleanupUnconfirmed {
|
|
active[row.ID] = activeRun{cancel: func() {}, workspace: row.Workspace}
|
|
}
|
|
}
|
|
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
|
|
if err := r.startSessionExportJobs(); err != nil {
|
|
return nil, err
|
|
}
|
|
r.schedulerWG.Add(1)
|
|
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
|
|
return r, nil
|
|
}
|
|
|
|
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
|
return r.startWithSchedule(request, nil, retryOf...)
|
|
}
|
|
|
|
func (r *runner) startWithSchedule(request protocol.Message, origin *pipeline.ScheduleAdmission, retryOf ...int64) (*protocol.RunDetail, error) {
|
|
if err := protocol.ValidateOutputFormat(request.OutputFormat); err != nil {
|
|
return nil, err
|
|
}
|
|
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
|
return nil, fmt.Errorf("an explicit version and request key are required")
|
|
}
|
|
workspace, err := pipeline.ResolveRunWorkspace(request.Workspace, origin != nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err = pipeline.ValidateRunWritePaths(workspace, request.WritePaths); err != nil {
|
|
return nil, err
|
|
}
|
|
request.Input = maps.Clone(request.Input)
|
|
if request.Input == nil {
|
|
request.Input = map[string]any{}
|
|
}
|
|
request.Input["pwd"] = workspace
|
|
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
|
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
|
}
|
|
d, err := r.store.Get(r.ctx, request.PipelineID, request.Version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil {
|
|
return nil, err
|
|
}
|
|
// Validate the launch-time model overrides against the pinned graph before
|
|
// admission: an unknown stage key or a malformed provider/model must never
|
|
// insert a run row. The resolution is O(#overrides + #stages) with no model or
|
|
// network call, and the same helper is the authoritative precedence rule.
|
|
if _, err := pipeline.EffectiveModels(d.Graph, request.Model, request.StageModels); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, s := range d.Graph.Stages {
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["harness"] == "pi" && r.execute == nil {
|
|
configured, err := harness.Load()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, ok := configured.Profiles["pi"]; !ok {
|
|
return nil, fmt.Errorf("Pi is not configured on this execution server; run pipelines harness setup pi there")
|
|
}
|
|
}
|
|
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(s))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !explicit {
|
|
request.AllowTools = true
|
|
}
|
|
for _, tool := range tools {
|
|
if tool == "edit" || tool == "shell" {
|
|
request.AllowTools = true
|
|
}
|
|
}
|
|
}
|
|
// Decide admission and idempotency under the lock, but generate the title
|
|
// only for a request that will actually start a run: a rejected or replayed
|
|
// request must not pay for a bounded model generation, and a live one must
|
|
// not hold the lock (stalling listings and cancellation) while a model is
|
|
// called. An unavailable or slow model falls back to the prompt's first line.
|
|
r.mu.Lock()
|
|
if r.ctx.Err() != nil {
|
|
r.mu.Unlock()
|
|
return nil, r.ctx.Err()
|
|
}
|
|
owners, err := r.store.ActiveSessionOwners(r.ctx)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
// Refuse only an orphaned session: one whose owning run is not active now.
|
|
// A session owned by a currently active run (another run in another
|
|
// workspace) must not block new work.
|
|
for _, owner := range owners {
|
|
if _, ok := r.active[owner.RunID]; !ok {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("an agent may still be active; restart recovery must interrupt it before new work")
|
|
}
|
|
}
|
|
// Idempotent repeats are accepted even while the original run is active, and
|
|
// a replay reuses the stored title instead of generating a new one.
|
|
existing, err := r.store.RunByKey(r.ctx, request.RequestKey)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
var runTitle summarise.Title
|
|
if existing == 0 {
|
|
// An explicit operator title skips the model call; otherwise the
|
|
// configured generator names the run, falling back to the prompt's
|
|
// first line when it is unavailable or slow.
|
|
if explicit, ok := summarise.ExplicitTitle(request.Title); ok {
|
|
runTitle = explicit
|
|
} else if r.title != nil {
|
|
prompt, _ := request.Input["prompt"].(string)
|
|
runTitle = r.title(r.ctx, prompt)
|
|
}
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
// Another run may have reserved this workspace while the title was
|
|
// generated, so re-check before reserving it; a race on the request key is
|
|
// handled by the store's idempotent insert.
|
|
if existing == 0 {
|
|
if err := r.checkWorkspaceFree(workspace); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
id, created, err := r.store.CreateRunTitled(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, pipeline.RunOverrides{Schedule: origin, Model: request.Model, StageModels: request.StageModels, NotifyMode: request.NotifyMode, OutputFormat: request.OutputFormat}, runTitle, retryOf...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r.setNotifyMode(id, request.NotifyMode)
|
|
// The run row now exists: read it once for its resolved identity (pipeline
|
|
// id/name, the store-derived title, the pinned version), so the queued frame
|
|
// and every later frame carry what a client needs to render a brand-new run
|
|
// without re-reading the list. A failed read only costs that seed: the run is
|
|
// still launched below and its frames carry no identity, so the client falls
|
|
// back to its one resync.
|
|
detail, readErr := r.store.GetRun(r.ctx, id)
|
|
if readErr != nil {
|
|
detail = nil
|
|
} else if detail != nil {
|
|
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
|
}
|
|
if created {
|
|
if request.RetrySource > 0 && request.RetryMode != "restart" {
|
|
checkpoint, checkpointErr := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput)
|
|
if checkpointErr != nil {
|
|
return nil, checkpointErr
|
|
}
|
|
if err := r.store.SaveRetryCheckpoint(r.ctx, id, request.RetrySource, "resume", checkpoint.failed.Sequence, checkpoint.detail, checkpoint.handoff); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
// Emit the created delta before launching: the queued frame reaches
|
|
// subscribers before the running/step-started transition, so a client
|
|
// sees the run appear and then advance.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusQueued,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}, false)
|
|
r.launch(id, workspace, func(ctx context.Context) {
|
|
r.run(ctx, id, d, workspace, request)
|
|
})
|
|
}
|
|
if detail != nil {
|
|
_, detail.Executing = r.active[id]
|
|
if detail.ExportPendingSequence > 0 {
|
|
detail.Executing = false
|
|
}
|
|
r.overlayEffectiveModels(detail)
|
|
}
|
|
if readErr != nil {
|
|
return nil, readErr
|
|
}
|
|
return detail, nil
|
|
}
|
|
|
|
// launch registers an active run's cancellation seam and workspace, adds it to
|
|
// the wait group and starts fn on a derived cancellable context that cleans up
|
|
// the registration when it returns. A fresh run and a resumed decision both go
|
|
// through it, so registration and teardown live once. Callers must hold r.mu.
|
|
func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
|
ctx, cancel := context.WithCancel(r.ctx)
|
|
generation := new(byte)
|
|
r.active[id] = activeRun{cancel: cancel, workspace: workspace, generation: generation}
|
|
r.wg.Add(1)
|
|
go func() {
|
|
defer r.wg.Done()
|
|
defer cancel()
|
|
defer func() {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if entry, ok := r.active[id]; !ok || entry.generation != generation {
|
|
return // A parked launch must not remove the resumed reservation.
|
|
}
|
|
check, done := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer done()
|
|
detail, err := r.store.GetRun(check, id)
|
|
if err != nil || detail.CleanupUnconfirmed {
|
|
return // Fail closed: execution/cleanup has not settled durably.
|
|
}
|
|
if detail.Status == protocol.RunStatusWaiting && detail.CancelRequested {
|
|
if err := r.store.CancelWaiting(check, id); err != nil {
|
|
return
|
|
}
|
|
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: detail.Workspace, Status: protocol.RunStatusCanceled, CancelRequested: true, CurrentStage: detail.CurrentStage, FinishedAt: time.Now().UTC().Format(time.RFC3339)}, true)
|
|
}
|
|
if detail.FinishedAt != "" || detail.Status == protocol.RunStatusWaiting {
|
|
delete(r.active, id)
|
|
}
|
|
}()
|
|
fn(ctx)
|
|
}()
|
|
}
|
|
|
|
// checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run
|
|
// reserves an overlapping resolved workspace. Callers must hold r.mu.
|
|
func (r *runner) checkWorkspaceFree(workspace string) error {
|
|
return r.checkWorkspaceFreeExcept(workspace, 0)
|
|
}
|
|
|
|
func (r *runner) checkWorkspaceFreeExcept(workspace string, ownGate int64) error {
|
|
for id, entry := range r.active {
|
|
if workspacesOverlap(entry.workspace, workspace) {
|
|
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
|
|
}
|
|
}
|
|
check, cancel := context.WithTimeout(context.WithoutCancel(r.ctx), 5*time.Second)
|
|
defer cancel()
|
|
reservations, err := r.store.SessionExportReservations(check)
|
|
if err != nil {
|
|
return fmt.Errorf("read pending export workspace reservations: %w", err)
|
|
}
|
|
for id, reserved := range reservations {
|
|
if id != ownGate && workspacesOverlap(reserved, workspace) {
|
|
return fmt.Errorf("run %d is awaiting its session export; workspace %s remains reserved", id, reserved)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func workspacesOverlap(a, b string) bool {
|
|
within := func(parent, child string) bool {
|
|
rel, err := filepath.Rel(parent, child)
|
|
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator))
|
|
}
|
|
return within(a, b) || within(b, a)
|
|
}
|
|
|
|
func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
d, err := r.store.GetRun(r.ctx, id)
|
|
if d != nil {
|
|
_, d.Executing = r.active[id]
|
|
if d.ExportPendingSequence > 0 {
|
|
d.Executing = false
|
|
}
|
|
for _, s := range d.Steps {
|
|
if s.AgentActive {
|
|
d.Executing = true
|
|
}
|
|
}
|
|
r.overlayEffectiveModels(d)
|
|
}
|
|
return d, err
|
|
}
|
|
|
|
// overlayEffectiveModels replaces each model stage's provider/model in a returned
|
|
// run's graph with the effective pair the run actually uses, so the app and TUI
|
|
// stage-definition surfaces show the override with no client change. It is
|
|
// presentation only: the persisted graph_json and the pinned source returned by
|
|
// `show` are never rewritten. A run with no overrides is left untouched.
|
|
func (r *runner) overlayEffectiveModels(d *protocol.RunDetail) {
|
|
if d == nil || (d.Model == "" && len(d.StageModels) == 0) {
|
|
return
|
|
}
|
|
effective, err := pipeline.EffectiveModels(d.Graph, d.Model, d.StageModels)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for i := range d.Graph.Stages {
|
|
s := &d.Graph.Stages[i]
|
|
if stage.EffectiveType(s.Type) != stage.ModelType {
|
|
continue
|
|
}
|
|
override, ok := effective[s.Key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if s.Configs == nil {
|
|
s.Configs = map[string]string{}
|
|
}
|
|
s.Configs["provider"] = override.Provider
|
|
s.Configs["model"] = override.Model
|
|
}
|
|
}
|
|
|
|
func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
items, err := r.store.ListRuns(r.ctx)
|
|
for i := range items {
|
|
_, items[i].Executing = r.active[items[i].ID]
|
|
if items[i].ExportPendingSequence > 0 {
|
|
items[i].Executing = false
|
|
}
|
|
if items[i].Status == "failed" && !items[i].Executing {
|
|
d, err := r.store.GetRun(r.ctx, items[i].ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, step := range d.Steps {
|
|
if step.AgentActive {
|
|
items[i].Executing = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return items, err
|
|
}
|
|
|
|
type retryCheckpoint struct {
|
|
detail *protocol.RunDetail
|
|
failed protocol.RunStep
|
|
handoff map[string]any
|
|
}
|
|
|
|
type retryPrefixRecord struct {
|
|
stage string
|
|
sequence int
|
|
public []string
|
|
publicKnown bool
|
|
}
|
|
|
|
const maxRetryInputBytes = 64 * 1024
|
|
|
|
func retryInputOverride(base map[string]any, override map[string]any) (map[string]any, error) {
|
|
if override == nil {
|
|
return base, nil
|
|
}
|
|
encoded, err := json.Marshal(override)
|
|
if err != nil || len(encoded) > maxRetryInputBytes {
|
|
return nil, fmt.Errorf("retry input must be a JSON object no larger than %d bytes", maxRetryInputBytes)
|
|
}
|
|
reserved := map[string]bool{"pwd": true, "workspace": true, "write_paths": true, "allow_tools": true, "pipeline_id": true, "pipeline_version": true, "request_key": true, "retry_mode": true, "retry_source": true, "retry_stage": true, "retry_visit": true, serverChangedFilesField: true}
|
|
merged := map[string]any{}
|
|
for key, value := range base {
|
|
merged[key] = value
|
|
}
|
|
for key, value := range override {
|
|
if reserved[key] {
|
|
return nil, fmt.Errorf("retry input field %q is reserved", key)
|
|
}
|
|
merged[key] = value
|
|
}
|
|
return merged, nil
|
|
}
|
|
|
|
// retryStageContractEqual compares the complete persisted stage contract. A
|
|
// retry may carry completed output across a published version only when the
|
|
// stages that produced that output are unchanged; comparing only effective
|
|
// types would allow prompt, config, edge, or limit changes through.
|
|
func retryStageContractEqual(left, right protocol.Stage) bool {
|
|
a, err := json.Marshal(left)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
b, err := json.Marshal(right)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return string(a) == string(b)
|
|
}
|
|
|
|
func (r *runner) retrySourceCheckpoint(sourceID int64, pinned *protocol.PipelineDetail) (retryCheckpoint, error) {
|
|
return r.retrySourceCheckpointSelected(sourceID, pinned, "", 0, nil)
|
|
}
|
|
|
|
func (r *runner) retrySourceCheckpointSelected(sourceID int64, pinned *protocol.PipelineDetail, selectedStage string, selectedVisit int, override map[string]any) (retryCheckpoint, error) {
|
|
source, err := r.store.GetRun(r.ctx, sourceID)
|
|
if err != nil {
|
|
return retryCheckpoint{}, err
|
|
}
|
|
if source.Status != "failed" || source.FinishedAt == "" {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d is not a terminal failed run", sourceID)
|
|
}
|
|
if source.PipelineID != pinned.ID {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d belongs to a different pipeline", sourceID)
|
|
}
|
|
if len(source.Steps) == 0 {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d has no failed stage checkpoint", sourceID)
|
|
}
|
|
failed := source.Steps[len(source.Steps)-1]
|
|
if selectedStage != "" {
|
|
failed = protocol.RunStep{}
|
|
visits := 0
|
|
for _, candidate := range source.Steps {
|
|
if candidate.Stage != selectedStage {
|
|
continue
|
|
}
|
|
visits++
|
|
if selectedVisit == 0 && visits == 1 {
|
|
failed = candidate
|
|
}
|
|
if selectedVisit > 0 && visits == selectedVisit {
|
|
failed = candidate
|
|
}
|
|
}
|
|
if failed.Stage == "" {
|
|
return retryCheckpoint{}, fmt.Errorf("retry stage %q was not recorded in run %d", selectedStage, sourceID)
|
|
}
|
|
if selectedVisit == 0 {
|
|
count := 0
|
|
for _, candidate := range source.Steps {
|
|
if candidate.Stage == selectedStage {
|
|
count++
|
|
}
|
|
}
|
|
if count > 1 {
|
|
return retryCheckpoint{}, fmt.Errorf("retry stage %q was visited %d times; --visit is required", selectedStage, count)
|
|
}
|
|
}
|
|
if selectedVisit > visits {
|
|
return retryCheckpoint{}, fmt.Errorf("retry stage %q has no visit %d", selectedStage, selectedVisit)
|
|
}
|
|
}
|
|
if (failed.Status != "completed" && failed.Status != "failed") || (selectedStage == "" && failed.Target != protocol.FailureEnd) {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d has no safe failed-stage checkpoint", sourceID)
|
|
}
|
|
var handoff map[string]any
|
|
if failed.Input == nil {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d is missing failed-stage handoff context", sourceID)
|
|
}
|
|
if _, err := retryInputOverride(failed.Input, override); err != nil {
|
|
return retryCheckpoint{}, err
|
|
}
|
|
// Keep the source-stage input immutable in the checkpoint. The selected
|
|
// stage's memory bindings must be resolved from the original prefix before
|
|
// this override is applied at invocation time.
|
|
handoff = map[string]any{"input": failed.Input, "override": override, "sequence": failed.Sequence, "stage": failed.Stage, "visit": failed.Sequence}
|
|
prefix, prefixRecords, prefixErr := r.retryPrefix(sourceID, source, failed.Sequence)
|
|
if prefixErr != nil {
|
|
return retryCheckpoint{}, prefixErr
|
|
}
|
|
handoff["prefix"] = prefix
|
|
serializedPrefix := make([]map[string]any, 0, len(prefixRecords))
|
|
for _, record := range prefixRecords {
|
|
entry := map[string]any{"stage": record.stage, "sequence": record.sequence}
|
|
if record.publicKnown {
|
|
// Persist the exact additions so a later retry can rebuild memory
|
|
// and carry public/ without reading this attempt's rows.
|
|
entry["public"] = record.public
|
|
}
|
|
serializedPrefix = append(serializedPrefix, entry)
|
|
}
|
|
handoff["prefix_provenance"] = serializedPrefix
|
|
// A newer explicitly selected version may continue only when every recorded
|
|
// prefix stage still exists with the same kind. Its outputs are evidence from
|
|
// the source attempt and cannot be safely handed to a changed stage contract.
|
|
sourceStages := map[string]protocol.Stage{}
|
|
for _, declared := range source.Graph.Stages {
|
|
sourceStages[declared.Key] = declared
|
|
}
|
|
for _, record := range prefixRecords {
|
|
_, found := retryPrefixStep(source, record)
|
|
if !found {
|
|
continue
|
|
}
|
|
var pinnedStage *protocol.Stage
|
|
for i := range pinned.Graph.Stages {
|
|
if pinned.Graph.Stages[i].Key == record.stage {
|
|
pinnedStage = &pinned.Graph.Stages[i]
|
|
break
|
|
}
|
|
}
|
|
previous, ok := sourceStages[record.stage]
|
|
if pinnedStage == nil || !ok || !retryStageContractEqual(previous, *pinnedStage) {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible changed prefix stage %q", sourceID, record.stage)
|
|
}
|
|
}
|
|
var sourceSelected protocol.Stage
|
|
var sourceSelectedOK bool
|
|
if sourceSelected, sourceSelectedOK = sourceStages[failed.Stage]; !sourceSelectedOK {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in its recorded graph", sourceID)
|
|
}
|
|
pinnedStages := map[string]protocol.Stage{}
|
|
for _, declared := range pinned.Graph.Stages {
|
|
pinnedStages[declared.Key] = declared
|
|
}
|
|
// The retry will continue through every stage that the source execution
|
|
// already recorded after the selected checkpoint. Validate those stages
|
|
// before admitting the new run as well: otherwise an explicitly selected
|
|
// version can pass prefix/selected-stage validation and fail later in the
|
|
// loop with the less useful "missing stage" error.
|
|
for _, step := range source.Steps {
|
|
if step.Sequence <= failed.Sequence {
|
|
continue
|
|
}
|
|
if _, ok := pinnedStages[step.Stage]; !ok {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d selected version is missing later stage %q", sourceID, step.Stage)
|
|
}
|
|
}
|
|
for _, declared := range pinned.Graph.Stages {
|
|
if declared.Key != failed.Stage {
|
|
continue
|
|
}
|
|
if stage.EffectiveType(sourceSelected.Type) != stage.EffectiveType(declared.Type) {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d has incompatible selected stage %q", sourceID, failed.Stage)
|
|
}
|
|
typ := stage.EffectiveType(declared.Type)
|
|
if typ == stage.HumanType || typ == stage.NtfyType {
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d cannot replay completed external stage %s", sourceID, failed.Stage)
|
|
}
|
|
if typ == stage.ModelType {
|
|
if _, _, toolErr := stage.ToolNames(pipeline.StageDefinition(declared)); toolErr != nil {
|
|
return retryCheckpoint{}, toolErr
|
|
}
|
|
}
|
|
return retryCheckpoint{detail: source, failed: failed, handoff: handoff}, nil
|
|
}
|
|
return retryCheckpoint{}, fmt.Errorf("retry source %d failed stage is not in the pinned graph", sourceID)
|
|
}
|
|
|
|
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("retry requires a positive run ID")
|
|
}
|
|
original, err := r.getRun(request.RunID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if original.Executing {
|
|
return nil, fmt.Errorf("run %d is still executing; wait for it to stop completely before retrying", original.ID)
|
|
}
|
|
if original.Status != "failed" || original.FinishedAt == "" {
|
|
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
|
}
|
|
if request.RequestKey == "" {
|
|
return nil, fmt.Errorf("retry requires an idempotency request key")
|
|
}
|
|
// Resolve an already admitted request before validating optional retry
|
|
// selectors. This makes replay after a restart genuinely idempotent even
|
|
// when the original request used an input override that the replay omits.
|
|
if existing, err := r.store.RunByKey(r.ctx, request.RequestKey); err != nil {
|
|
return nil, err
|
|
} else if existing > 0 {
|
|
return r.store.GetRun(r.ctx, existing)
|
|
}
|
|
if blocked, err := r.store.SessionExportBlocked(r.ctx, original.ID); err != nil {
|
|
return nil, err
|
|
} else if blocked {
|
|
return nil, fmt.Errorf("run %d has a retained computation awaiting export-only recovery; model retry refused", original.ID)
|
|
}
|
|
if request.RetryVisit < 0 || (request.RetryVisit > 0 && request.RetryStage == "") {
|
|
return nil, fmt.Errorf("retry visit requires a positive --stage visit selection")
|
|
}
|
|
mode := request.RetryMode
|
|
if mode == "" {
|
|
mode = "resume"
|
|
}
|
|
if mode != "resume" && mode != "restart" {
|
|
return nil, fmt.Errorf("retry mode must be resume or restart")
|
|
}
|
|
version := original.Version
|
|
if request.RetryVersion > 0 {
|
|
version = request.RetryVersion
|
|
}
|
|
d, err := r.store.Get(r.ctx, original.PipelineID, version)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if mode == "resume" {
|
|
if _, err := r.retrySourceCheckpointSelected(original.ID, d, request.RetryStage, request.RetryVisit, request.RetryInput); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
notifyMode := request.NotifyMode
|
|
if notifyMode == "" {
|
|
notifyMode, _ = r.store.RunNotifyMode(r.ctx, original.ID)
|
|
}
|
|
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: version, RequestKey: request.RequestKey, RetryMode: mode, RetrySource: original.ID, RetryStage: request.RetryStage, RetryVisit: request.RetryVisit, RetryInput: request.RetryInput, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths, Model: original.Model, StageModels: original.StageModels, NotifyMode: notifyMode, OutputFormat: original.OutputFormat}, original.ID)
|
|
}
|
|
|
|
func (r *runner) cancel(id int64) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
entry, ok := r.active[id]
|
|
if err := r.store.Cancel(r.ctx, id); err != nil {
|
|
return err
|
|
}
|
|
if ok {
|
|
entry.cancel()
|
|
} else {
|
|
// Only a parked human run is known to have no live executor.
|
|
if err := r.store.CancelWaiting(r.ctx, id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
r.forgetAttention(id)
|
|
if r.events != nil {
|
|
if detail, err := r.store.GetRun(r.ctx, id); err == nil && detail != nil {
|
|
r.rememberRunIdentity(id, identityOfRunSummary(detail.RunSummary))
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: detail.Workspace,
|
|
Status: detail.Status,
|
|
Outcome: detail.Outcome,
|
|
CurrentStage: detail.CurrentStage,
|
|
CompletedSteps: detail.CompletedSteps,
|
|
FinishedAt: detail.FinishedAt,
|
|
CancelRequested: detail.CancelRequested,
|
|
CleanupUnconfirmed: detail.CleanupUnconfirmed,
|
|
}, !ok)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// decide records an operator's choice for a waiting human stage and resumes the
|
|
// run along the chosen edge. ResolveDecision owns validation and idempotency, so
|
|
// decide does no option parsing of its own. A resumed run is registered and
|
|
// launched exactly like a fresh one but starts from the persisted state.
|
|
func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
|
if request.RunID <= 0 {
|
|
return nil, fmt.Errorf("decide requires a positive run ID")
|
|
}
|
|
r.mu.Lock()
|
|
detail, err := r.store.GetRun(r.ctx, request.RunID)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
// finishRunState persists the terminal status under r.mu, but the run stays
|
|
// in r.active until its goroutine finishes notifying and the launch teardown
|
|
// runs. A run whose terminal state is already durable (FinishedAt is only set
|
|
// once cleanup is confirmed) has no executor left to race with a decision, so
|
|
// ResolveDecision answers it ("already decided" or "not waiting") and never
|
|
// resumes a terminal run. A run that is still executing is refused.
|
|
if _, busy := r.active[detail.ID]; busy && detail.FinishedAt == "" {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
|
|
}
|
|
if detail.ExportPendingSequence > 0 {
|
|
r.mu.Unlock()
|
|
return nil, fmt.Errorf("run %d is awaiting session export, not a human decision", detail.ID)
|
|
}
|
|
if detail.Status == protocol.RunStatusWaiting {
|
|
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
}
|
|
step, applied, err := r.store.ResolveDecision(r.ctx, detail.ID, request.Stage, request.Option, request.Feedback)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return nil, err
|
|
}
|
|
if applied {
|
|
// Re-seed the identity so the resumed frames (this one and the resumed
|
|
// loop's) carry it even after a restart, when no creation was observed.
|
|
r.rememberRunIdentity(detail.ID, identityOfRunSummary(detail.RunSummary))
|
|
for i := range detail.Steps {
|
|
if detail.Steps[i].Sequence == step.Sequence && detail.Steps[i].Stage == step.Stage {
|
|
detail.Steps[i] = step
|
|
break
|
|
}
|
|
}
|
|
// The decision closes the waiting step and returns the run to running;
|
|
// report that before the resumed loop emits its own deltas.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: detail.ID,
|
|
Workspace: detail.Workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: step.Stage,
|
|
CompletedSteps: completedSteps(detail.Steps),
|
|
Step: &protocol.RunStepDelta{
|
|
Sequence: step.Sequence,
|
|
Stage: step.Stage,
|
|
Status: step.Status,
|
|
Outcome: step.Outcome,
|
|
Target: step.Target,
|
|
Diagnostic: step.Diagnostic,
|
|
},
|
|
}, false)
|
|
r.launch(detail.ID, detail.Workspace, func(ctx context.Context) {
|
|
r.resume(ctx, detail.ID, detail, detail.Workspace, step)
|
|
})
|
|
}
|
|
r.mu.Unlock()
|
|
return r.getRun(request.RunID)
|
|
}
|
|
|
|
func (r *runner) run(ctx context.Context, id int64, d *protocol.PipelineDetail, workspace string, request protocol.Message) {
|
|
if request.RetrySource > 0 && request.RetryMode != "restart" {
|
|
source, err := r.store.GetRun(ctx, request.RetrySource)
|
|
if err != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
|
return
|
|
}
|
|
checkpoint, err := r.retrySourceCheckpointSelected(request.RetrySource, d, request.RetryStage, request.RetryVisit, request.RetryInput)
|
|
if err != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
|
return
|
|
}
|
|
storedSource, storedMode, storedSequence, storedHandoff, checkpointErr := r.store.RetryCheckpoint(ctx, id)
|
|
if checkpointErr != nil || storedSource != request.RetrySource || storedMode != "resume" || storedSequence != checkpoint.failed.Sequence || storedHandoff == nil {
|
|
if checkpointErr == nil {
|
|
checkpointErr = fmt.Errorf("retry checkpoint is incomplete")
|
|
}
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", checkpointErr.Error(), 0, 0, "", nil)
|
|
return
|
|
}
|
|
if input, ok := storedHandoff["input"].(map[string]any); ok {
|
|
checkpoint.failed.Input = input
|
|
} else {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", "retry checkpoint is missing handoff context", 0, 0, "", nil)
|
|
return
|
|
}
|
|
state := rebuildRetryState(source, checkpoint.failed, storedHandoff)
|
|
if override, ok := storedHandoff["override"].(map[string]any); ok {
|
|
state.retryOverride = stage.Values(override)
|
|
}
|
|
// The retry is a new run with a new artifact root. Carry the completed
|
|
// prefix's recorded public files before the first visit, so memory does
|
|
// not name files this attempt cannot read.
|
|
if err := r.carryRetryPublic(id, request.RetrySource, checkpoint.failed.Sequence, storedHandoff, source); err != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", err.Error(), 0, 0, "", nil)
|
|
return
|
|
}
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
|
|
return
|
|
}
|
|
original, copyErr := copyValues(stage.Values(request.Input))
|
|
if copyErr != nil {
|
|
r.finishRunState(id, d.Name, workspace, "failed", "error", copyErr.Error(), 0, 0, "", stage.Values(request.Input))
|
|
return
|
|
}
|
|
input := stage.Values(request.Input)
|
|
state := newRunState(d.Graph.Stages[0].Key, input)
|
|
state.memory["input"] = memoryValue{output: original}
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, request, state)
|
|
}
|
|
|
|
// terminalFinish classifies a stage's resolved target. A terminal key yields the
|
|
// run's final status, outcome and diagnostic; an empty target means the stage
|
|
// named an undeclared outcome; any other value is a stage key and is not
|
|
// terminal. resume and the loop tail share it, so terminal routing lives once.
|
|
func terminalFinish(target, stageKey, outcome, diagnostic string) (status, finalOutcome, finalDiagnostic string, terminal bool) {
|
|
switch target {
|
|
case protocol.SuccessEnd:
|
|
return "succeeded", "success", "", true
|
|
case protocol.FailureEnd:
|
|
if diagnostic == "" {
|
|
diagnostic = fmt.Sprintf("stage %s returned outcome %q", stageKey, outcome)
|
|
}
|
|
return "failed", "error", diagnostic, true
|
|
case "":
|
|
return "failed", "error", "stage returned an undeclared outcome", true
|
|
}
|
|
return "", "", "", false
|
|
}
|
|
|
|
// resume continues a decided run from the state already persisted for its run
|
|
// detail: memory from the run input and the completed steps, visits counted per
|
|
// stage, and the next sequence above the highest recorded one. It then runs the
|
|
// shared loop from the decision's target; no full-table scan and no extra
|
|
// per-step query are introduced.
|
|
func (r *runner) resume(ctx context.Context, id int64, d *protocol.RunDetail, workspace string, step protocol.RunStep) {
|
|
// A resumed run's frames carry the same identity as its creation, so re-seed
|
|
// it here too: resume is also reachable from a store read after a restart.
|
|
r.rememberRunIdentity(id, identityOfRunSummary(d.RunSummary))
|
|
// The decision may route straight to a terminal, which has no stage to run.
|
|
if status, outcome, diagnostic, terminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic); terminal {
|
|
r.finishRunState(id, d.Name, workspace, status, outcome, diagnostic, completedSteps(d.Steps), step.Sequence, step.Stage, stage.Values(step.Output))
|
|
return
|
|
}
|
|
state := rebuildRunState(d, step)
|
|
r.runLoop(ctx, id, d.Graph, d.Name, workspace, protocol.Message{WritePaths: d.WritePaths, AllowTools: d.AllowTools, Model: d.Model, StageModels: d.StageModels}, state)
|
|
}
|
|
|
|
// runState is the loop's whole mutable state, extracted so a fresh run and a
|
|
// resumed run share one implementation. nextSequence is set before each visit so
|
|
// a wait that returns to the caller records where the run will continue.
|
|
type runState struct {
|
|
key string
|
|
nextSequence int
|
|
visits map[string]int
|
|
memory map[string]memoryValue
|
|
input stage.Values
|
|
// changedPaths retains a bounded sample of the run's recorded changed paths
|
|
// and changedCount the full total, so the skip guard reads only in-memory
|
|
// evidence. uiSurface is latched once any recorded change touched a UI
|
|
// surface. Both are folded in by recordChanges (ui_surface.go).
|
|
changedPaths []string
|
|
changedCount int
|
|
uiSurface bool
|
|
// capturedPaths is the full, deduplicated, first-seen-ordered set of the
|
|
// run's server-captured changed paths, distinct from the bounded
|
|
// changedPaths sample used only by the skip reason. capturedSeen is the
|
|
// single dedupe set. Both are folded in by recordChanges (ui_surface.go) and
|
|
// published to a selecting stage as input.server_changed_files.
|
|
capturedPaths []string
|
|
capturedSeen map[string]bool
|
|
// completed counts the run's finished steps, so a delta can report the
|
|
// running total without re-reading the store.
|
|
completed int
|
|
artifactDirs []string
|
|
// prompt is the original run prompt; lastOutput is the previous visit's
|
|
// markdown and turns is every completed turn in order. Together they render
|
|
// the next model visit's input.md and memory.md.
|
|
prompt string
|
|
lastOutput string
|
|
turns []turn.MemoryEntry
|
|
// retryOverride is applied once, after the selected stage's memory bindings
|
|
// have been resolved. It is intentionally not part of run input or memory.
|
|
retryOverride stage.Values
|
|
priorBudgets map[string]stage.BudgetSnapshot
|
|
}
|
|
|
|
func newRunState(key string, input stage.Values) *runState {
|
|
prompt, _ := input["prompt"].(string)
|
|
return &runState{key: key, nextSequence: 1, visits: map[string]int{}, memory: map[string]memoryValue{}, input: input, prompt: prompt}
|
|
}
|
|
|
|
func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runState {
|
|
state := newRunState(decided.Target, stage.Values(decided.Output))
|
|
if prompt, ok := detail.Input["prompt"].(string); ok {
|
|
state.prompt = prompt
|
|
}
|
|
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
|
state.memory["input"] = memoryValue{output: original}
|
|
}
|
|
publicSeen := map[string]bool{}
|
|
state.priorBudgets = map[string]stage.BudgetSnapshot{}
|
|
for _, step := range detail.Steps {
|
|
// The runtime stamps this cumulative snapshot on every executed model
|
|
// visit. Reusing it preserves per-stage limits across export recovery.
|
|
var audit struct {
|
|
Limits struct {
|
|
Budget stage.BudgetSnapshot `json:"budget"`
|
|
} `json:"stage_limits"`
|
|
}
|
|
if json.Unmarshal([]byte(step.Transcript), &audit) == nil {
|
|
state.priorBudgets[step.Stage] = audit.Limits.Budget
|
|
}
|
|
state.visits[step.Stage]++
|
|
if step.Sequence >= state.nextSequence {
|
|
state.nextSequence = step.Sequence + 1
|
|
}
|
|
added := publicArtifacts(step.Artifacts, publicSeen)
|
|
if step.Status == "completed" {
|
|
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
state.completed++
|
|
state.turns = append(state.turns, turn.MemoryEntry{Stage: step.Stage, Visit: state.visits[step.Stage], Output: turnOutput(step.Output), PublicFiles: added})
|
|
state.lastOutput = turnOutput(step.Output)
|
|
}
|
|
// Seed the UI-surface accumulator from the persisted evidence so a
|
|
// resumed run reaches the same skip decision with no extra query.
|
|
state.recordChanges(step.FileChanges, step.Artifacts)
|
|
// Only completed, recorded visits are eligible for a later stage's
|
|
// review capability. The path is reconstructed by the runner below.
|
|
if step.Status == "completed" {
|
|
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
|
}
|
|
}
|
|
return state
|
|
}
|
|
|
|
// rebuildRetryState restores only the completed prefix. The failed visit is
|
|
// deliberately not counted and its step/evidence is not copied into the new
|
|
// attempt, so retries cannot duplicate execution or usage aggregates.
|
|
func rebuildRetryState(detail *protocol.RunDetail, failed protocol.RunStep, handoff map[string]any) *runState {
|
|
state := newRunState(failed.Stage, stage.Values(failed.Input))
|
|
// Resume at the failed sequence. The retry is a new run with a new run ID
|
|
// and artifact root, so this sequence's turn directory does not overlap the
|
|
// source attempt's files.
|
|
state.nextSequence = failed.Sequence
|
|
if prompt, ok := detail.Input["prompt"].(string); ok {
|
|
state.prompt = prompt
|
|
}
|
|
if original, err := copyValues(stage.Values(detail.Input)); err == nil {
|
|
state.memory["input"] = memoryValue{output: original}
|
|
}
|
|
publicSeen := map[string]bool{}
|
|
_, hasCheckpointPrefix := handoff["prefix"]
|
|
if !hasCheckpointPrefix {
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence >= failed.Sequence {
|
|
break
|
|
}
|
|
state.visits[step.Stage]++
|
|
added := publicArtifacts(step.Artifacts, publicSeen)
|
|
if step.Status == "completed" {
|
|
if values, err := copyValues(stage.Values(step.Output)); err == nil {
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
state.completed++
|
|
state.turns = append(state.turns, turn.MemoryEntry{Stage: step.Stage, Visit: state.visits[step.Stage], Output: turnOutput(step.Output), PublicFiles: added})
|
|
state.lastOutput = turnOutput(step.Output)
|
|
}
|
|
state.recordChanges(step.FileChanges, step.Artifacts)
|
|
if step.Status == "completed" {
|
|
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(step.Sequence)))
|
|
}
|
|
}
|
|
}
|
|
// A selected-stage retry has no persisted rows for its inherited prefix.
|
|
// Restore it from the immutable checkpoint instead of guessing from the
|
|
// retry source's immediate steps (which is what made retry-of-retry fail
|
|
// closed with "memory source ... has not completed").
|
|
if prefix, ok := handoff["prefix"].(map[string]any); ok {
|
|
for _, record := range retryPrefixRecords(handoff["prefix_provenance"]) {
|
|
output, exists := prefix[record.stage].(map[string]any)
|
|
if !exists {
|
|
continue
|
|
}
|
|
values, err := copyValues(stage.Values(output))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
state.memory[record.stage] = memoryValue{output: values, sequence: record.sequence}
|
|
state.visits[record.stage]++
|
|
state.completed++
|
|
state.turns = append(state.turns, turn.MemoryEntry{Stage: record.stage, Visit: state.visits[record.stage], Output: turnOutput(values), PublicFiles: recordPublicFiles(record, detail, publicSeen)})
|
|
state.lastOutput = turnOutput(values)
|
|
state.artifactDirs = append(state.artifactDirs, filepath.Join("", strconv.Itoa(record.sequence)))
|
|
}
|
|
}
|
|
return state
|
|
}
|
|
|
|
// retryPrefix returns the completed prefix from the source attempt plus the
|
|
// immutable prefix carried by that attempt's checkpoint. A retry attempt only
|
|
// persists the selected stage onward, so immediate source steps are not a
|
|
// complete representation of its memory.
|
|
func (r *runner) retryPrefix(sourceID int64, source *protocol.RunDetail, failedSequence int) (map[string]any, []retryPrefixRecord, error) {
|
|
prefix := map[string]any{}
|
|
records := []retryPrefixRecord{}
|
|
inheritedPublic := map[string]bool{}
|
|
if source.RetryMode == "resume" && source.RetrySource > 0 {
|
|
checkpointSource, mode, _, inherited, err := r.store.RetryCheckpoint(r.ctx, sourceID)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("retry source %d has no immutable prefix checkpoint: %w", sourceID, err)
|
|
}
|
|
if checkpointSource != source.RetrySource || mode != "resume" {
|
|
return nil, nil, fmt.Errorf("retry source %d has an invalid prefix checkpoint", sourceID)
|
|
}
|
|
if values, ok := inherited["prefix"].(map[string]any); ok {
|
|
for key, value := range values {
|
|
prefix[key] = value
|
|
}
|
|
}
|
|
for _, record := range retryPrefixRecords(inherited["prefix_provenance"]) {
|
|
records = append(records, record)
|
|
for _, name := range record.public {
|
|
inheritedPublic[name] = true
|
|
}
|
|
}
|
|
}
|
|
publicBySequence := retryPublicBySequence(source, failedSequence)
|
|
for _, step := range source.Steps {
|
|
if step.Sequence >= failedSequence || step.Status != "completed" {
|
|
continue
|
|
}
|
|
prefix[step.Stage] = step.Output
|
|
added := make([]string, 0, len(publicBySequence[step.Sequence]))
|
|
for _, name := range publicBySequence[step.Sequence] {
|
|
// Carried prefix files reappear in this attempt's inventory but were
|
|
// added by an earlier turn, not this one.
|
|
if !inheritedPublic[name] {
|
|
added = append(added, name)
|
|
}
|
|
}
|
|
records = append(records, retryPrefixRecord{stage: step.Stage, sequence: step.Sequence, public: added, publicKnown: true})
|
|
}
|
|
return prefix, records, nil
|
|
}
|
|
|
|
func retryPrefixRecords(raw any) []retryPrefixRecord {
|
|
items, ok := raw.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
result := make([]retryPrefixRecord, 0, len(items))
|
|
for _, item := range items {
|
|
entry, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
stageName, ok := entry["stage"].(string)
|
|
if !ok || stageName == "" {
|
|
continue
|
|
}
|
|
sequence, ok := entry["sequence"].(float64)
|
|
if !ok || sequence < 1 {
|
|
continue
|
|
}
|
|
record := retryPrefixRecord{stage: stageName, sequence: int(sequence)}
|
|
if raw, ok := entry["public"]; ok {
|
|
record.public = stringValues(raw)
|
|
record.publicKnown = true
|
|
}
|
|
result = append(result, record)
|
|
}
|
|
return result
|
|
}
|
|
|
|
// stringValues accepts the JSON-decoded forms of a string list in a retry
|
|
// checkpoint and returns a copy; unknown element types are ignored.
|
|
func stringValues(raw any) []string {
|
|
switch values := raw.(type) {
|
|
case []string:
|
|
return append([]string(nil), values...)
|
|
case []any:
|
|
result := make([]string, 0, len(values))
|
|
for _, value := range values {
|
|
if text, ok := value.(string); ok {
|
|
result = append(result, text)
|
|
}
|
|
}
|
|
return result
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func retryPrefixStep(detail *protocol.RunDetail, record retryPrefixRecord) (protocol.RunStep, bool) {
|
|
for _, step := range detail.Steps {
|
|
if step.Stage == record.stage && step.Sequence == record.sequence {
|
|
return step, true
|
|
}
|
|
}
|
|
// Inherited rows intentionally are not present in a selected retry. The
|
|
// contract check still uses the stage key; this marker keeps that check
|
|
// independent from model-claimed output.
|
|
return protocol.RunStep{Stage: record.stage, Sequence: record.sequence}, true
|
|
}
|
|
|
|
// artifactAccess derives exact turn directories; it never consumes a path from
|
|
// a definition or model result. The current visit may read and write its own
|
|
// turn and the shared public directory. Earlier visits are not readable: their
|
|
// public files are already shared and their private/ files are never handed to
|
|
// a later stage. An unreadable or symlinked root fails closed.
|
|
func (r *runner) artifactAccess(runID int64, sequence int, prior []string, current string) stage.ArtifactAccess {
|
|
_ = prior // prior private directories remain deliberately inaccessible
|
|
root, err := filepath.EvalSymlinks(r.artifactsRoot)
|
|
if err != nil {
|
|
return stage.ArtifactAccess{}
|
|
}
|
|
runRoot := filepath.Join(root, strconv.FormatInt(runID, 10))
|
|
directory := func(path string) (string, bool) {
|
|
if path == "" {
|
|
return "", false
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(path)
|
|
if err != nil || !strings.HasPrefix(resolved, runRoot+string(os.PathSeparator)) {
|
|
return "", false
|
|
}
|
|
info, err := os.Stat(resolved)
|
|
if err != nil || !info.IsDir() {
|
|
return "", false
|
|
}
|
|
return resolved, true
|
|
}
|
|
access := stage.ArtifactAccess{}
|
|
currentDir, ok := directory(current)
|
|
if !ok {
|
|
return access
|
|
}
|
|
// The current directory must be this visit's turn, not another stage's turn
|
|
// in the same run.
|
|
expectedPrefix := fmt.Sprintf("%02d-", sequence)
|
|
if filepath.Dir(currentDir) != filepath.Join(runRoot, "stages") || !strings.HasPrefix(filepath.Base(currentDir), expectedPrefix) {
|
|
return access
|
|
}
|
|
if private, ok := directory(filepath.Join(currentDir, "private")); ok {
|
|
// The model may write its own private/ files, never the server-owned
|
|
// input.md, memory.md or output.md. A missing private/ grants no write
|
|
// scope; falling back to the turn directory would widen the permission.
|
|
access.Write = private
|
|
}
|
|
access.Read = append(access.Read, currentDir)
|
|
if public, ok := directory(filepath.Join(runRoot, "public")); ok {
|
|
access.WriteShared = public
|
|
access.Read = append(access.Read, public)
|
|
}
|
|
return access
|
|
}
|
|
|
|
func (r *runner) agentClient(runID int64, sequence int, prior []string, workspace, artifactsDir string, operations *stage.OperationLog, writePaths, tools []string) *stage.AgentClient {
|
|
return &stage.AgentClient{OpenCodeClient: stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}, WritePaths: writePaths, Tools: tools, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
|
|
}
|
|
|
|
func (r *runner) piClient(runID int64, sequence int, prior []string, executable, workspace, artifactsDir string, writePaths, tools []string) *stage.PiClient {
|
|
return &stage.PiClient{Executable: executable, Directory: workspace, Tools: tools, WritePaths: writePaths, ArtifactsDir: artifactsDir, ArtifactAccess: r.artifactAccess(runID, sequence, prior, artifactsDir)}
|
|
}
|
|
|
|
// serverChangedFilesField is the reserved run-input memory field under which the
|
|
// runner publishes the server-captured changed paths. A stage selects it with
|
|
// `"server_changed_files":"input.server_changed_files"`, alongside another
|
|
// stage's declared list, so it sees the verified evidence, not just a model's
|
|
// claim of authorship.
|
|
const serverChangedFilesField = "server_changed_files"
|
|
|
|
// capturedList returns the full, first-seen-ordered set of server-captured
|
|
// changed paths, or a non-nil empty list when nothing was captured so the field
|
|
// still resolves as a list. It copies the accumulator so the retained evidence
|
|
// cannot be aliased by a published value. O(captured paths).
|
|
func (s *runState) capturedList() []string {
|
|
if len(s.capturedPaths) == 0 {
|
|
return []string{}
|
|
}
|
|
return append([]string(nil), s.capturedPaths...)
|
|
}
|
|
|
|
// publishServerChangedFiles writes the run's captured changed-path set into the
|
|
// run input memory value under serverChangedFilesField, creating the entry and
|
|
// its output map when absent. Called before memoryInput so a stage selecting
|
|
// input.server_changed_files always resolves; only that field is added and only
|
|
// a selecting stage copies it out, so state.input and other stages' input are
|
|
// unchanged. O(captured paths).
|
|
func (s *runState) publishServerChangedFiles() {
|
|
entry, ok := s.memory["input"]
|
|
if !ok || entry.output == nil {
|
|
entry = memoryValue{output: stage.Values{}}
|
|
}
|
|
entry.output[serverChangedFilesField] = s.capturedList()
|
|
s.memory["input"] = entry
|
|
}
|
|
|
|
// runIdentity is the pipeline identity and stored title a run's pushed frames
|
|
// carry (RunEventFrame.PipelineID/Name/Title/Version). It is remembered where a
|
|
// run is created, decided or canceled and stamped onto every frame that leaves
|
|
// emitRun, so a client that has never seen the run can build its list row from
|
|
// the frame without re-reading the whole list.
|
|
type runIdentity struct {
|
|
scheduled bool
|
|
scheduleID int64
|
|
scheduleName string
|
|
pipelineID int64
|
|
name string
|
|
title string
|
|
version int
|
|
}
|
|
|
|
// identityOfRunSummary is the one projection of a run row onto the identity its
|
|
// pushed frames carry, so the frame and the list row cannot disagree.
|
|
func identityOfRunSummary(s protocol.RunSummary) runIdentity {
|
|
return runIdentity{scheduled: s.Scheduled, scheduleID: s.ScheduleID, scheduleName: s.ScheduleName, pipelineID: s.PipelineID, name: s.Name, title: s.Title, version: s.Version}
|
|
}
|
|
|
|
// rememberRunIdentity records the identity every later frame of run id carries.
|
|
// It is safe from any goroutine and takes its own lock (never r.mu), so it is
|
|
// callable from a path that already holds r.mu.
|
|
func (r *runner) rememberRunIdentity(id int64, ident runIdentity) {
|
|
r.identMu.Lock()
|
|
if r.idents == nil {
|
|
r.idents = map[int64]runIdentity{}
|
|
}
|
|
r.idents[id] = ident
|
|
r.identMu.Unlock()
|
|
}
|
|
|
|
// stampRunIdentity fills a frame's blank identity fields from the run's
|
|
// remembered identity, leaving any the emit site set untouched. A run with no
|
|
// remembered identity keeps them blank, so the client falls back to its resync.
|
|
func (r *runner) stampRunIdentity(frame *protocol.RunEventFrame) {
|
|
r.identMu.Lock()
|
|
ident, ok := r.idents[frame.RunID]
|
|
r.identMu.Unlock()
|
|
if !ok {
|
|
return
|
|
}
|
|
frame.Scheduled = ident.scheduled
|
|
frame.ScheduleID = ident.scheduleID
|
|
frame.ScheduleName = ident.scheduleName
|
|
if frame.PipelineID == 0 {
|
|
frame.PipelineID = ident.pipelineID
|
|
}
|
|
if frame.Name == "" {
|
|
frame.Name = ident.name
|
|
}
|
|
if frame.Title == "" {
|
|
frame.Title = ident.title
|
|
}
|
|
if frame.Version == 0 {
|
|
frame.Version = ident.version
|
|
}
|
|
}
|
|
|
|
// emitRun pushes one run delta to the subscribed-client hub when one is wired,
|
|
// and does nothing otherwise (a runner built directly in a test has no hub).
|
|
// terminal marks a final transition, which the hub flushes without coalescing.
|
|
func (r *runner) emitRun(frame protocol.RunEventFrame, terminal bool) {
|
|
// A run that is no longer waiting has no decision pending: its waiting
|
|
// notifications are cleared on every device.
|
|
if r.notes != nil && frame.Status != "" && frame.Status != protocol.RunStatusWaiting {
|
|
c, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
r.notes.clearFor(c, pipeline.AppNotificationFilter{RunID: frame.RunID, Kinds: []string{protocol.NotificationRunWaiting}})
|
|
cancel()
|
|
}
|
|
if r.events == nil {
|
|
return
|
|
}
|
|
r.stampRunIdentity(&frame)
|
|
frame.NotifyMode = r.notifyMode(frame.RunID)
|
|
r.events.emit(frame, terminal)
|
|
}
|
|
|
|
func (r *runner) finishRunState(id int64, name, workspace, status, outcome, diagnostic string, completed, sequence int, currentStage string, input stage.Values) {
|
|
r.mu.Lock()
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if status == "succeeded" {
|
|
if err := r.finalizeDeliverable(c, id, sequence, input); err != nil {
|
|
status, outcome, diagnostic = "failed", "error", "deliverable contract: "+err.Error()
|
|
}
|
|
}
|
|
// Measure the run's footprint while it is still recorded as active: its
|
|
// files are final, and measuring after the terminal status is written would
|
|
// widen the window in which a client sees the run finished but still owned.
|
|
// The status write re-marks the cache stale, so the background sizer then
|
|
// records the exact value including the final output row.
|
|
size := r.finishedSize(id)
|
|
if err := r.store.FinishRun(c, id, status, outcome, diagnostic, input); err != nil {
|
|
log.Printf("finish run %d: %v", id, err)
|
|
}
|
|
// The store atomically makes a stop request win over a late success.
|
|
detail, getErr := r.store.GetRun(c, id)
|
|
if getErr != nil || detail == nil {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
status, outcome, diagnostic = detail.Status, detail.Outcome, detail.Diagnostic
|
|
r.mu.Unlock()
|
|
if detail.FinishedAt == "" {
|
|
return // Cleanup is not confirmed: never emit a false terminal event.
|
|
}
|
|
// Push the terminal delta before the attention notification, so a slow ntfy
|
|
// never delays the live update the app folds.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: status,
|
|
Outcome: outcome,
|
|
CurrentStage: currentStage,
|
|
CompletedSteps: completed,
|
|
FinishedAt: detail.FinishedAt,
|
|
SizeBytes: size,
|
|
}, true)
|
|
r.kickSizes()
|
|
if status != protocol.RunStatusCanceled {
|
|
r.notifyTerminalStep(id, status)
|
|
}
|
|
r.deliverRunNotificationBatch(time.Now(), 1)
|
|
// The run is terminal: nothing more can notify for it.
|
|
r.forgetAttention(id)
|
|
r.forgetNotifyMode(id)
|
|
}
|
|
|
|
// completedSteps counts the completed steps of a run detail; resume uses it to
|
|
// seed the terminal delta without another store read.
|
|
func completedSteps(steps []protocol.RunStep) int {
|
|
n := 0
|
|
for _, step := range steps {
|
|
if step.Status == "completed" {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// stageUsage carries the usage a model stage already measured for budget
|
|
// accounting onto the run record. It reports a cost only when the provider
|
|
// reported one and an in/out split only when the harness reported it, so an
|
|
// unknown cost or split is never presented as zero. It returns nil when no usage
|
|
// was observed, so the step stays absent rather than zero.
|
|
func stageUsage(snapshot stage.BudgetSnapshot) *protocol.StageUsage {
|
|
if !snapshot.TokensKnown && !snapshot.USDKnown {
|
|
return nil
|
|
}
|
|
usage := &protocol.StageUsage{TokensKnown: snapshot.TokensKnown, CostKnown: snapshot.USDKnown}
|
|
if snapshot.TokensKnown {
|
|
usage.TokensTotal = snapshot.UsedTokens
|
|
if snapshot.SplitKnown {
|
|
usage.TokensIn, usage.TokensOut, usage.SplitKnown = snapshot.TokensInput, snapshot.TokensOutput, true
|
|
}
|
|
}
|
|
if snapshot.USDKnown {
|
|
usage.CostUSD = snapshot.UsedUSD
|
|
}
|
|
return usage
|
|
}
|
|
|
|
// finishStage persists one step, refreshes the shared memory for a completed
|
|
// step, notifies and classifies the step's target. Both the executed path and
|
|
// the skipped path use it, so persistence, notification and terminal routing
|
|
// cannot drift between them. A persist or memory-copy failure is returned for
|
|
// the caller to fail the run; the caller still owns the model-specific
|
|
// verdict, quiescence and cancellation handling around it.
|
|
// turnOutputOnly reports the turn protocol's single markdown body. The verdict
|
|
// summariser is written for the old field map (summary/option/reason); given a
|
|
// document it treats it as untrusted reference material and invents a reason.
|
|
func turnOutputOnly(output stage.Values) bool {
|
|
if len(output) != 1 {
|
|
return false
|
|
}
|
|
body, ok := output["markdown"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
_, isText := body.(string)
|
|
return isText
|
|
}
|
|
|
|
func (r *runner) finishStage(id int64, name, workspace string, selected protocol.Stage, state *runState, step protocol.RunStep) (status, outcome, diagnostic string, terminal bool, err error) {
|
|
r.mu.Lock()
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
detail, getErr := r.store.GetRun(save, id)
|
|
if getErr != nil {
|
|
cancel()
|
|
r.mu.Unlock()
|
|
return "", "", "", false, getErr
|
|
}
|
|
if detail.CancelRequested {
|
|
step.Status, step.Outcome, step.Target = "canceled", "", ""
|
|
step.Diagnostic = "Canceled by operator; execution returned"
|
|
}
|
|
persistErr := r.store.FinishStep(save, id, step)
|
|
cancel()
|
|
r.mu.Unlock()
|
|
if persistErr != nil {
|
|
return "", "", "", false, persistErr
|
|
}
|
|
if detail.CancelRequested {
|
|
return "canceled", "", step.Diagnostic, true, nil
|
|
}
|
|
if step.Status == "completed" {
|
|
values, copyErr := copyValues(stage.Values(step.Output))
|
|
if copyErr != nil {
|
|
return "", "", "", false, copyErr
|
|
}
|
|
state.memory[step.Stage] = memoryValue{output: values, sequence: step.Sequence}
|
|
}
|
|
state.input = stage.Values(step.Output)
|
|
status, outcome, diagnostic, terminal = terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
if !terminal {
|
|
r.notifyStep(id, name, workspace, selected.Notify, step)
|
|
}
|
|
return status, outcome, diagnostic, terminal, nil
|
|
}
|
|
|
|
func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, name, workspace string, request protocol.Message, state *runState) {
|
|
finish := func(status, outcome, diagnostic string) {
|
|
r.finishRunState(id, name, workspace, status, outcome, diagnostic, state.completed, state.nextSequence-1, state.key, state.input)
|
|
}
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
finish("failed", "error", "stage execution panicked")
|
|
}
|
|
}()
|
|
budgets := map[string]*stage.Budget{}
|
|
if state.key == "" {
|
|
finish("failed", "error", "stage returned an undeclared outcome")
|
|
return
|
|
}
|
|
// Resolve the effective provider/model for every model stage once, before the
|
|
// loop, so applying an override is an in-memory O(1) map lookup per visit and
|
|
// never re-parses the graph or re-resolves the whole definition. A run with no
|
|
// overrides skips the resolution entirely. Launch validation already ran this
|
|
// helper, so a failure here only guards a resumed run whose overrides drifted.
|
|
var effectiveModels map[string]pipeline.ModelOverride
|
|
if request.Model != "" || len(request.StageModels) > 0 {
|
|
resolved, err := pipeline.EffectiveModels(graph, request.Model, request.StageModels)
|
|
if err != nil {
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
effectiveModels = resolved
|
|
}
|
|
for sequence := state.nextSequence; sequence <= 30; sequence++ {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
key := state.key
|
|
var def stage.Definition
|
|
var selected protocol.Stage
|
|
found := false
|
|
for _, s := range graph.Stages {
|
|
if s.Key == key {
|
|
selected = s
|
|
def = pipeline.StageDefinition(s)
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
finish("failed", "error", "missing stage")
|
|
return
|
|
}
|
|
// StageDefinition clones Configs, so setting the effective pair on the
|
|
// clone is safe and the pinned graph is never mutated. Because
|
|
// ModelStage.Spec reads def.Configs and passes spec.Config into the repair
|
|
// path, the same effective model governs the stage and its repair.
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
if override, ok := effectiveModels[key]; ok {
|
|
def.Configs["provider"] = override.Provider
|
|
def.Configs["model"] = override.Model
|
|
}
|
|
}
|
|
isHuman := stage.EffectiveType(def.Type) == stage.HumanType
|
|
maxVisits := 5
|
|
if selected.Limits != nil {
|
|
maxVisits = selected.Limits.MaxVisits
|
|
}
|
|
// Validate the next visit without consuming it. An already-exhausted
|
|
// stage must not begin work it can never execute.
|
|
if !isHuman {
|
|
if state.visits[key]+1 > 5 {
|
|
finish("failed", "error", "stage visit budget exceeded")
|
|
return
|
|
}
|
|
if state.visits[key]+1 > maxVisits {
|
|
finish("failed", "error", fmt.Sprintf("stage %s visit limit reached (%d)", key, maxVisits))
|
|
return
|
|
}
|
|
}
|
|
_, declaresSkip := def.Edges["skipped"]
|
|
skip := !isHuman && declaresSkip && !state.uiSurface
|
|
state.nextSequence = sequence + 1
|
|
state.visits[key]++
|
|
// Publish the run's server-captured changed paths into the input memory
|
|
// value before resolving bindings, so a stage selecting
|
|
// input.server_changed_files always resolves (an empty list when nothing
|
|
// was captured). Only a selecting stage copies it into its input.
|
|
state.publishServerChangedFiles()
|
|
var reads map[string]protocol.MemoryRead
|
|
bindings := selected.Memory
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
// A model stage reads the turn's memory.md and input.md files; the
|
|
// retired per-field memory block no longer selects its input.
|
|
bindings = nil
|
|
}
|
|
input, reads, copyErr := memoryInput(state.input, bindings, state.memory)
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
if changed := state.capturedList(); stage.EffectiveType(def.Type) == stage.ModelType && len(changed) > 0 {
|
|
// The turn files, not a memory binding, carry the stage context. The
|
|
// prompt reads this reserved value so a committer still sees the
|
|
// server-captured paths that distinguish this run from pre-existing
|
|
// dirty work. An empty list adds nothing and leaves the entry input
|
|
// unchanged.
|
|
input[serverChangedFilesField] = changed
|
|
}
|
|
var retryInput stage.Values
|
|
if state.retryOverride != nil {
|
|
input, copyErr = retryInputOverride(input, state.retryOverride)
|
|
if copyErr != nil {
|
|
finish("failed", "error", copyErr.Error())
|
|
return
|
|
}
|
|
// Keep the merged values for this visit's input.md: the model reads
|
|
// files, not the persisted step row, so the override must be visible
|
|
// there. A selected-stage override belongs only to this invocation,
|
|
// not to a later visit reached through the stage graph.
|
|
retryInput = input
|
|
state.retryOverride = nil
|
|
}
|
|
if err := r.store.BeginStep(ctx, id, sequence, key, input); err != nil {
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
// queued -> running: report the new current stage and the step start.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusRunning},
|
|
}, false)
|
|
if isHuman {
|
|
// A human step is O(1): one BeginStep plus one MarkWaiting, no
|
|
// filediff capture, no artifacts directory and no workspace mutation.
|
|
evidence := protocol.NewWaitingEvidence(def.Blocks["prompt"], stage.HumanOptions(def.Edges))
|
|
r.mu.Lock()
|
|
if err := r.store.MarkWaiting(ctx, id, sequence, key, evidence); err != nil {
|
|
r.mu.Unlock()
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
finish("failed", "error", err.Error())
|
|
return
|
|
}
|
|
waitStep := protocol.RunStep{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting, Output: evidence}
|
|
// A human step parks the run without a live goroutine; the delta is
|
|
// coalesced by the hub's timer, which keeps running after this returns.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusWaiting,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: protocol.StepStatusWaiting},
|
|
}, false)
|
|
// Publishing can be slow. Once waiting is visible, release execution
|
|
// atomically so a decision is not spuriously rejected as "active".
|
|
delete(r.active, id)
|
|
r.mu.Unlock()
|
|
r.notifyStep(id, name, workspace, selected.Notify, waitStep)
|
|
r.deliverRunNotificationBatch(time.Now(), 1)
|
|
return
|
|
}
|
|
// A stage that declares a `skipped` route is short-circuited when the run
|
|
// has recorded no UI surface: the stage would answer exactly that, so skip
|
|
// the model call and its capture work, record the step as completed and
|
|
// route along the declared edge. Only a declared route is honoured, and no
|
|
// other stage or outcome routing changes.
|
|
if skip {
|
|
reason := skipReasonLine(state)
|
|
step := protocol.RunStep{
|
|
Sequence: sequence,
|
|
Stage: key,
|
|
Status: "completed",
|
|
Outcome: "skipped",
|
|
Output: stage.Values{"summary": reason},
|
|
Target: def.Edges["skipped"],
|
|
Control: &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads},
|
|
}
|
|
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
|
|
if finishErr != nil {
|
|
finish("failed", "error", finishErr.Error())
|
|
return
|
|
}
|
|
if terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
continue
|
|
}
|
|
var tools []string
|
|
explicitTools := false
|
|
var toolErr error
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
tools, explicitTools, toolErr = stage.ToolNames(def)
|
|
}
|
|
if toolErr != nil {
|
|
finish("failed", "error", toolErr.Error())
|
|
return
|
|
}
|
|
useAgent := stage.EffectiveType(def.Type) == stage.ModelType && (len(tools) > 0 || !explicitTools)
|
|
// Every model visit gets a file-based turn: input.md and memory.md are
|
|
// written before execution, output.md after it. Creation is part of the
|
|
// protocol, so an unusable turn directory fails the run instead of
|
|
// silently degrading to the retired JSON envelope.
|
|
artifactsDir := ""
|
|
var visitTurn turn.Paths
|
|
var publicBefore map[string]bool
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
visitTurn = r.turnPaths(id, sequence, key)
|
|
if turnErr := r.prepareTurn(visitTurn, state, retryInput); turnErr != nil {
|
|
finish("failed", "error", turnErr.Error())
|
|
return
|
|
}
|
|
artifactsDir = visitTurn.TurnDir
|
|
def.ArtifactsDir = artifactsDir
|
|
def.Turn = stageTurnPaths(visitTurn)
|
|
publicBefore = publicFileSet(visitTurn.RunDir)
|
|
}
|
|
operations := &stage.OperationLog{}
|
|
var landing stage.LandingAccess
|
|
delivery, deliveryErr := stage.DeliveryEnabled(def)
|
|
if deliveryErr != nil {
|
|
finish("failed", "error", deliveryErr.Error())
|
|
return
|
|
}
|
|
if delivery {
|
|
access, accessErr := landingAccess(workspace)
|
|
if accessErr != nil {
|
|
finish("failed", "error", "delivery access denied: "+accessErr.Error())
|
|
return
|
|
}
|
|
landing = access
|
|
}
|
|
client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools)
|
|
client.LandingAccess = landing
|
|
client.OnSession = func(session string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, client.Backend, client.Reference)
|
|
}
|
|
var result stage.Result
|
|
var piClient *stage.PiClient
|
|
var err error
|
|
stageCtx := ctx
|
|
var modelConfig stage.ModelConfig
|
|
var budgetBefore stage.BudgetSnapshot
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
spec, specErr := (stage.ModelStage{}).Spec(def)
|
|
if specErr != nil {
|
|
finish("failed", "error", specErr.Error())
|
|
return
|
|
}
|
|
modelConfig = spec.Config
|
|
if budgets[key] == nil {
|
|
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
|
if used, ok := state.priorBudgets[key]; ok {
|
|
budgets[key].RestoreUsage(used)
|
|
}
|
|
}
|
|
// The budget is shared across every visit to this stage key, so capture
|
|
// its cumulative usage before the visit and record only this visit's
|
|
// increment on the step, never the cumulative total again.
|
|
budgetBefore = budgets[key].Snapshot()
|
|
stageCtx = stage.WithBudget(stageCtx, budgets[key])
|
|
}
|
|
snapshotCtx, snapshotCancel := context.WithTimeout(ctx, 5*time.Second)
|
|
before := filediff.Capture(snapshotCtx, workspace)
|
|
snapshotCancel()
|
|
var stageSession string
|
|
panicked := false
|
|
func() {
|
|
defer func() {
|
|
if recover() != nil {
|
|
panicked = true
|
|
err = fmt.Errorf("stage execution panicked")
|
|
}
|
|
}()
|
|
if ctx.Err() != nil {
|
|
err = ctx.Err()
|
|
return
|
|
}
|
|
if r.execute != nil {
|
|
timeout, _ := stage.StageTimeout(def)
|
|
executeCtx, cancel := context.WithTimeout(stageCtx, timeout)
|
|
defer cancel()
|
|
result, err = r.execute(executeCtx, def, input)
|
|
} else if modelConfig.Harness == "pi" {
|
|
configured, configErr := harness.Load()
|
|
if configErr != nil {
|
|
err = configErr
|
|
return
|
|
}
|
|
profile, ok := configured.Profiles["pi"]
|
|
if !ok {
|
|
err = fmt.Errorf("Pi is not configured on execution server")
|
|
return
|
|
}
|
|
piClient = r.piClient(id, sequence, state.artifactDirs, profile.Executable, workspace, artifactsDir, request.WritePaths, tools)
|
|
piClient.OnOwnership = func(session, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return r.store.RunSession(c, id, sequence, session, active, "pi", reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: piClient}).Run(stageCtx, def, input)
|
|
stageSession = piClient.SessionID
|
|
client.Transcript = piClient.Transcript
|
|
} else if useAgent {
|
|
result, err = (stage.ModelStage{Client: client}).Run(stageCtx, def, input)
|
|
stageSession = client.SessionID
|
|
} else if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
plain := &stage.OpenCodeClient{Directory: workspace, Operations: operations, ArtifactsDir: artifactsDir}
|
|
plain.OnGenerationSession = func(session, backend, reference string, active bool) error {
|
|
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if active {
|
|
stageSession = session
|
|
}
|
|
return r.store.RunSession(c, id, sequence, session, active, backend, reference)
|
|
}
|
|
result, err = (stage.ModelStage{Client: plain}).Run(stageCtx, def, input)
|
|
client.Transcript = plain.FormatAudit
|
|
} else {
|
|
result, err = stage.DefaultRegistry().Run(stageCtx, def, input)
|
|
}
|
|
}()
|
|
ownershipCtx, ownershipCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
activeSessions, ownershipErr := r.store.ActiveAgentSessionsForRun(ownershipCtx, id)
|
|
ownershipCancel()
|
|
if ownershipErr != nil || len(activeSessions) > 0 {
|
|
err = errors.Join(stage.ErrStageCleanup, err, ownershipErr)
|
|
}
|
|
if ctx.Err() != nil && !errors.Is(err, stage.ErrStageCleanup) {
|
|
err = ctx.Err() // Ignore a context-unaware executor's late success.
|
|
}
|
|
if stage.EffectiveType(def.Type) == stage.ModelType && err == nil {
|
|
ensureTurnOutput(visitTurn, result)
|
|
}
|
|
// Capture after cleanup, even when the stage's context was canceled.
|
|
snapshotCtx, snapshotCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
after := filediff.Capture(snapshotCtx, workspace, before)
|
|
snapshotCancel()
|
|
changes := filediff.Compare(before, after)
|
|
if errors.Is(err, stage.ErrStageCleanup) {
|
|
changes.Status = "partial"
|
|
changes.Diagnostic = "Agent cleanup was not confirmed; files may still be changing."
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
if markErr := r.store.MarkCleanupUnconfirmed(save, id); markErr != nil {
|
|
log.Printf("mark cleanup unconfirmed for run %d: %v", id, markErr)
|
|
}
|
|
cancel()
|
|
}
|
|
// Only settled, exact recorded ownership can satisfy the handoff gate.
|
|
// A canceled execution must not cancel this ownership read silently.
|
|
var sessionExport *protocol.SessionExport
|
|
exportRequired := artifactsDir != "" && r.execute == nil
|
|
if artifactsDir != "" {
|
|
ownerCtx, ownerCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
owner, ownerErr := r.store.RepairSessionOwner(ownerCtx, id, sequence)
|
|
if ownerErr != nil && r.execute != nil {
|
|
// Only genuinely unowned fake executors are exempt. A recorded
|
|
// session with absent ownership metadata must still fail closed.
|
|
if recorded, readErr := r.store.RunSessionOwner(ownerCtx, id, sequence); readErr != nil || recorded.ID != "" {
|
|
exportRequired = true
|
|
if readErr == nil {
|
|
stageSession = recorded.ID
|
|
}
|
|
}
|
|
}
|
|
ownerCancel()
|
|
if ownerErr == nil && owner.ID != "" {
|
|
exportRequired = true // Owned test executors exercise the real gate too.
|
|
stageSession = owner.ID
|
|
if !errors.Is(err, stage.ErrStageCleanup) {
|
|
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
|
|
}
|
|
} else if exportRequired {
|
|
sessionExport = &protocol.SessionExport{Status: "failed", Source: "native_export", Diagnostic: "exact settled session ownership unavailable; handoff refused"}
|
|
}
|
|
}
|
|
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir, visitTurn.PublicDir)
|
|
step := protocol.RunStep{Sequence: sequence, Stage: key, Status: "completed", Outcome: result.Outcome, Output: result.Output, Transcript: operations.Transcript(client.Transcript), SessionID: stageSession, FileChanges: changes, SessionExport: sessionExport, Artifacts: artifacts, ArtifactsSkipped: artifactsSkipped, ArtifactsTruncated: artifactsTruncated}
|
|
// Stamp the effective provider/model this visit actually ran with for every
|
|
// model stage, including failures, so the app/TUI evidence reflects any
|
|
// override. Non-model stages keep empty values. The usage the stage already
|
|
// measured for budget accounting is carried onto the same step, never
|
|
// re-read from the harness or parsed from the transcript.
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
step.Provider = modelConfig.Provider
|
|
step.Model = modelConfig.Model
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Usage = stageUsage(budget.Snapshot().Since(budgetBefore))
|
|
}
|
|
}
|
|
if budget := budgets[key]; budget != nil {
|
|
step.Transcript = stage.BudgetTranscript(step.Transcript, budget, modelConfig.Timeout, modelConfig.Retries)
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
if len(result.Output) > 0 {
|
|
step.Output["salvage"] = result.Output
|
|
}
|
|
}
|
|
if err == nil && result.Outcome == "success" {
|
|
target, targetErr := stage.VerificationTargetFor(def)
|
|
if targetErr != nil {
|
|
err = targetErr
|
|
} else if target != nil {
|
|
if r.verifyTarget == nil {
|
|
err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
|
|
} else {
|
|
evidence, verificationErr := r.verifyTarget(stageCtx, *target)
|
|
if verificationErr != nil {
|
|
err = verificationErr
|
|
} else {
|
|
err = stage.VerifyTargetEvidence(*target, evidence)
|
|
}
|
|
}
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
}
|
|
}
|
|
if err == nil && delivery && result.Outcome == "success" {
|
|
checks, checksErr := stage.DeliveryChecks(def)
|
|
if checksErr != nil {
|
|
err = checksErr
|
|
} else {
|
|
gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout)
|
|
var catchUp string
|
|
catchUp, err = verifyDelivery(gateCtx, workspace, landing, checks)
|
|
gateCancel()
|
|
// A failed gate already carries the note in its diagnostic. A step
|
|
// diagnostic on success renders as a stop, so success is logged.
|
|
if catchUp != "" {
|
|
log.Printf("run %d stage %s: %s", id, key, catchUp)
|
|
}
|
|
}
|
|
if err != nil {
|
|
step.Status = "failed"
|
|
step.Diagnostic = err.Error()
|
|
step.Outcome = "error"
|
|
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
|
}
|
|
}
|
|
// A failed or timed-out step carries the session pointer even when the
|
|
// bounded transcript would otherwise be only its stage_limits envelope.
|
|
if err != nil && stageSession != "" {
|
|
step.Transcript = stage.RetainSession(step.Transcript, stageSession)
|
|
}
|
|
step.Target = def.Edges[step.Outcome]
|
|
if errors.Is(err, stage.ErrStageCleanup) {
|
|
// Retain the partial evidence without pretending that the agent stopped.
|
|
step.Status, step.Outcome, step.Target = "running", "", ""
|
|
save, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
if saveErr := r.store.FinishStep(save, id, step); saveErr != nil {
|
|
log.Printf("save cleanup evidence for run %d: %v", id, saveErr)
|
|
}
|
|
cancel()
|
|
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusRunning, CleanupUnconfirmed: true}, false)
|
|
return
|
|
}
|
|
step.Control = &protocol.ControlEvidence{Visit: state.visits[key], MaxVisits: maxVisits, Memory: reads}
|
|
if err == nil && selected.Limits != nil && state.visits[key] >= maxVisits {
|
|
for _, outcome := range selected.Limits.On {
|
|
if outcome == step.Outcome {
|
|
step.Control.LimitReached = true
|
|
step.Control.ModelTarget = step.Target
|
|
step.Target = protocol.FailureEnd
|
|
step.Diagnostic = fmt.Sprintf("Pipelines stopped %s after %d visits on outcome %s", key, maxVisits, step.Outcome)
|
|
}
|
|
}
|
|
}
|
|
if panicked {
|
|
step.Target = protocol.FailureEnd
|
|
}
|
|
_, _, _, failedTerminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
|
if exportRequired && !completeSessionExport(sessionExport) && !(err != nil && failedTerminal && step.Target != protocol.SuccessEnd) {
|
|
// Preserve the real computation, files, outcome and target. In
|
|
// particular an export failure is not an agent failure/retry request.
|
|
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
saveErr := r.store.FinishStepExportBlocked(save, id, step)
|
|
cancel()
|
|
if saveErr != nil {
|
|
finish("failed", "error", saveErr.Error())
|
|
return
|
|
}
|
|
// No finishStage, memory publication, recordTurn or next executor.
|
|
check, done := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
detail, readErr := r.store.GetRun(check, id)
|
|
done()
|
|
if readErr != nil {
|
|
log.Printf("read parked export gate for run %d: %v", id, readErr)
|
|
return // The checkpoint is durable; never fabricate a terminal.
|
|
}
|
|
if detail.CancelRequested && detail.ExportPendingSequence == 0 {
|
|
finish("canceled", "", "Canceled by operator; computed visit retained")
|
|
return
|
|
}
|
|
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, ExportPendingSequence: sequence, CurrentStage: key, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts)}}, true)
|
|
return
|
|
}
|
|
// Generate the one-line verdict reason once, at completion. It is
|
|
// best-effort: a summariser failure or timeout returns NoSummary and must
|
|
// never fail the stage or the run, so every error is swallowed. A failed
|
|
// step still gets a verdict from its error envelope. A turn output is one
|
|
// markdown body, which the field-map summariser reads as untrusted data and
|
|
// answers with an invented rationale; those steps get the honest read-time
|
|
// fallback instead.
|
|
if r.verdict != nil && !turnOutputOnly(step.Output) {
|
|
reasonCtx, reasonCancel := context.WithTimeout(context.Background(), summarise.DefaultTimeout)
|
|
if line := r.verdict(reasonCtx, step.Output); line != "" && line != summarise.NoSummary {
|
|
step.Verdict = &protocol.StageVerdict{Reason: line, ReasonSource: protocol.VerdictReasonModel}
|
|
}
|
|
reasonCancel()
|
|
}
|
|
status, outcome, diagnostic, terminal, finishErr := r.finishStage(id, name, workspace, selected, state, step)
|
|
if finishErr != nil {
|
|
finish("failed", "error", finishErr.Error())
|
|
return
|
|
}
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
// Advance the in-memory UI-surface evidence only after the step is
|
|
// persisted, so the skip guard agrees with what the store holds.
|
|
state.recordChanges(step.FileChanges, step.Artifacts)
|
|
if step.Status == "completed" {
|
|
added := []string(nil)
|
|
if stage.EffectiveType(def.Type) == stage.ModelType {
|
|
added = addedPublicFiles(publicBefore, publicFileSet(visitTurn.RunDir))
|
|
}
|
|
state.recordTurn(step, added)
|
|
state.completed++
|
|
if artifactsDir != "" {
|
|
state.artifactDirs = append(state.artifactDirs, strconv.Itoa(sequence))
|
|
}
|
|
}
|
|
// Step finished: report its outcome, routing target, diagnostic and the
|
|
// artifacts recorded for the visit, plus the run's running step count.
|
|
r.emitRun(protocol.RunEventFrame{
|
|
RunID: id,
|
|
Workspace: workspace,
|
|
Status: protocol.RunStatusRunning,
|
|
CurrentStage: key,
|
|
CompletedSteps: state.completed,
|
|
Artifacts: len(step.Artifacts),
|
|
Step: &protocol.RunStepDelta{
|
|
Sequence: step.Sequence,
|
|
Stage: step.Stage,
|
|
Status: step.Status,
|
|
Outcome: step.Outcome,
|
|
Target: step.Target,
|
|
Diagnostic: step.Diagnostic,
|
|
Artifacts: len(step.Artifacts),
|
|
},
|
|
}, false)
|
|
if ctx.Err() != nil {
|
|
finish("canceled", "", ctx.Err().Error())
|
|
return
|
|
}
|
|
if terminal {
|
|
finish(status, outcome, diagnostic)
|
|
return
|
|
}
|
|
state.key = step.Target
|
|
}
|
|
finish("failed", "error", "total stage budget exceeded")
|
|
}
|
|
|
|
// exportStageSession writes the finished stage's session into its own artifacts
|
|
// directory and returns verified evidence. The caller enforces the handoff gate;
|
|
// an export failure never replaces the computation's result.
|
|
func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport {
|
|
evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend, Directory: owner.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339)}
|
|
defer classifySessionExport(evidence)
|
|
if owner.Backend != "v1" && owner.Backend != "v2" && !(owner.Backend == "pi" && owner.Reference != "") {
|
|
evidence.Source = "unavailable"
|
|
evidence.Diagnostic = "session export unavailable: harness has no native export path"
|
|
return evidence
|
|
}
|
|
export := r.exportSession
|
|
if export == nil {
|
|
export = stage.ExportSession
|
|
}
|
|
// A canceled stage still gets one bounded, non-blocking attempt; the export
|
|
// never consumes the stage timeout and never waits for the session.
|
|
exportCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stage.SessionExportTimeout)
|
|
defer cancel()
|
|
result, err := export(exportCtx, stage.SessionExportRequest{
|
|
SessionID: owner.ID,
|
|
Backend: owner.Backend,
|
|
Reference: owner.Reference,
|
|
Directory: owner.Directory,
|
|
ArtifactsDir: artifactsDir,
|
|
RunID: runID,
|
|
Sequence: sequence,
|
|
Stage: key,
|
|
Provider: modelConfig.Provider,
|
|
Model: modelConfig.Model,
|
|
})
|
|
evidence.Attempts = result.Attempts
|
|
if err != nil {
|
|
evidence.Source = "native_export"
|
|
evidence.Diagnostic = stage.RedactEvidence(err.Error())
|
|
return evidence
|
|
}
|
|
evidence.Source = result.Source
|
|
evidence.Status = result.Status
|
|
evidence.HTML = result.HTML
|
|
evidence.HTMLBytes = result.HTMLBytes
|
|
evidence.Messages = result.Messages
|
|
evidence.Truncated = result.Truncated
|
|
evidence.Diagnostic = result.Diagnostic
|
|
if result.Status != "" && result.Status != "complete" && evidence.Diagnostic == "" {
|
|
evidence.Diagnostic = "native session export reported " + result.Status
|
|
}
|
|
verifySessionExportFile(evidence, artifactsDir)
|
|
return evidence
|
|
}
|
|
|
|
// setTitle renames a run: it replaces the stored operator-facing title without
|
|
// touching the run's input, steps or execution, then returns the updated detail.
|
|
func (r *runner) setTitle(id int64, title string) (*protocol.RunDetail, error) {
|
|
explicit, ok := summarise.ExplicitTitle(title)
|
|
if !ok {
|
|
return nil, fmt.Errorf("a title with usable text is required")
|
|
}
|
|
if err := r.store.SetRunTitle(r.ctx, id, explicit); err != nil {
|
|
return nil, err
|
|
}
|
|
return r.getRun(id)
|
|
}
|