runner: gate stage handoff on settled verified session export

This commit is contained in:
2026-10-20 20:54:51 +01:00
parent 3d35e8a863
commit f87e1ed488
18 changed files with 791 additions and 26 deletions
+30 -3
View File
@@ -44,12 +44,18 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0", key, id)); err != nil {
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'')<>'complete')`, key, id)); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
return err
}
// Consuming a recovered handoff and starting its successor are atomic. A
// crash before this transaction leaves the gate available after restart.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND sequence<?`, id, sequence); err != nil {
return err
}
return tx.Commit()
}
@@ -73,6 +79,16 @@ type stageVerdictEvidence struct {
}
func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep) error {
return s.finishStep(ctx, id, step, false)
}
// FinishStepExportBlocked retains the computed step and its routing target, but
// checkpoints the handoff separately: the output is not yet eligible for reuse.
func (s *Store) FinishStepExportBlocked(ctx context.Context, id int64, step protocol.RunStep) error {
return s.finishStep(ctx, id, step, true)
}
func (s *Store) finishStep(ctx context.Context, id int64, step protocol.RunStep, exportBlocked bool) error {
output, err := json.Marshal(step.Output)
if err != nil {
return err
@@ -148,6 +164,11 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome,
if err := enqueueStepExportJob(ctx, tx, id, step.Sequence); err != nil {
return err
}
if exportBlocked && !requested {
if _, err := tx.ExecContext(ctx, `INSERT INTO session_export_gates(run_id,sequence) VALUES(?,?)`, id, step.Sequence); err != nil {
return err
}
}
// The reported usage the stage already measured for budget accounting is
// persisted as step-scoped evidence in the same transaction, mirroring the
// other step evidence. It is written only when the stage actually reported
@@ -302,7 +323,9 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
return err
}
defer tx.Rollback()
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting)
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,
diagnostic=CASE WHEN status='waiting' THEN 'Canceled while waiting for a decision' ELSE 'Canceled while awaiting session export; computed visit retained' END,
finished_at=CURRENT_TIMESTAMP WHERE id=? AND (status=? OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id, protocol.RunStatusWaiting)
if err != nil {
return err
}
@@ -323,7 +346,7 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
func (s *Store) Cancel(ctx context.Context, id int64) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cancel_requested=1,
diagnostic=CASE WHEN cleanup_unconfirmed=1 THEN diagnostic ELSE 'Stop requested; waiting for execution and cleanup to stop' END
WHERE id=? AND status IN ('queued','running','waiting')`, id))
WHERE id=? AND (status IN ('queued','running','waiting') OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id))
}
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
@@ -347,6 +370,10 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
return err
}
// Terminal targets have no successor BeginStep to consume their gate.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status='complete')`, id); err != nil {
return err
}
return tx.Commit()
}
+74
View File
@@ -0,0 +1,74 @@
package pipeline
import "context"
func (s *Store) SessionExportBlocked(ctx context.Context, id int64) (bool, error) {
var blocked bool
err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&blocked)
return blocked, err
}
// PendingSessionExportGate authorizes only export repair of the exact completed
// visit, never a model call or a human decision. No live executor is implied by
// the workspace reservation maintained for this checkpoint.
func (s *Store) PendingSessionExportGate(ctx context.Context, id int64, sequence int) (bool, error) {
var pending bool
err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates g
JOIN pipeline_runs r ON r.id=g.run_id JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
WHERE g.run_id=? AND g.sequence=? AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND s.status NOT IN ('running','waiting')
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=g.run_id OR a.session_id=s.session_id) AND a.agent_active<>0))`, id, sequence).Scan(&pending)
return pending, err
}
// SessionExportReservations survive process restarts and cannot be confused
// with agent-active/executing flags in the UI.
func (s *Store) SessionExportReservations(ctx context.Context) (map[int64]string, error) {
rows, err := s.db.QueryContext(ctx, `SELECT r.id,r.workspace FROM session_export_gates g JOIN pipeline_runs r ON r.id=g.run_id WHERE r.cancel_requested=0 AND r.status<>'canceled'`)
if err != nil {
return nil, err
}
defer rows.Close()
reservations := map[int64]string{}
for rows.Next() {
var id int64
var workspace string
if err := rows.Scan(&id, &workspace); err != nil {
return nil, err
}
reservations[id] = workspace
}
return reservations, rows.Err()
}
// ReadySessionExportGates includes interrupted continuations. The gate remains
// durable until a successor is begun (or a terminal target is committed).
func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, error) {
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
JOIN pipeline_runs r ON r.id=g.run_id
WHERE j.status='complete' AND r.status IN ('failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`)
if err != nil {
return nil, err
}
defer rows.Close()
ready := map[int64]int{}
for rows.Next() {
var id int64
var sequence int
if err := rows.Scan(&id, &sequence); err != nil {
return nil, err
}
ready[id] = sequence
}
return ready, rows.Err()
}
// ResumeSessionExportGate is a CAS, not a retry of the model invocation.
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='queued',outcome='',diagnostic='',finished_at=NULL
WHERE id=? AND status IN ('failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`)
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence))
}
@@ -0,0 +1,62 @@
package pipeline
import (
"context"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "failed", SessionID: "ses_original", Backend: "v2", Diagnostic: "offline"}}
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
t.Fatal("successor started while export was incomplete")
}
if err := s.FinishRun(ctx, id, "failed", "error", "export gate", step.Output); err != nil {
t.Fatal(err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
if err != nil || claimed == nil {
t.Fatalf("missing export-only retry: %v", err)
}
if err := s.RecordSessionExportRepair(ctx, id, 1, &protocol.SessionExport{Status: "complete", SessionID: claimed.SessionID, Backend: claimed.Backend, Directory: claimed.Directory, Messages: 2, HTML: "session-repair-1-test.html", HTMLBytes: 100}); err != nil {
t.Fatal(err)
}
if err := s.FinishSessionExportJob(ctx, claimed, time.Now(), ""); err != nil {
t.Fatal(err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err == nil {
t.Fatal("duplicate continuation admitted")
}
// A crash after reserving the continuation but before starting its next
// executor must retain the checkpoint for startup's interrupted run.
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status='interrupted' WHERE id=?`, id); err != nil {
t.Fatal(err)
}
ready, err := s.ReadySessionExportGates(ctx)
if err != nil || ready[id] != 1 {
t.Fatalf("lost restart checkpoint: %v %v", ready, err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
t.Fatal(err)
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("successor did not consume checkpoint: %v %v", blocked, err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
t.Fatal("successor started twice")
}
}
+6 -1
View File
@@ -35,6 +35,11 @@ status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','com
next_at INTEGER NOT NULL, lease_until INTEGER NOT NULL DEFAULT 0, token TEXT NOT NULL DEFAULT '', diagnostic TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE);
CREATE INDEX IF NOT EXISTS session_export_jobs_due ON session_export_jobs(status,next_at,lease_until);`)
if err == nil {
_, err = s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_gates (
run_id INTEGER PRIMARY KEY, sequence INTEGER NOT NULL,
FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE)`)
}
return err
}
@@ -45,7 +50,7 @@ SELECT s.run_id,s.sequence,s.session_id,o.backend,r.workspace,o.reference,?
FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id
JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence
LEFT JOIN step_session_exports x ON x.run_id=s.run_id AND x.sequence=s.sequence
WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND o.backend IN ('v1','v2')
WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND (o.backend IN ('v1','v2') OR (o.backend='pi' AND o.reference<>''))
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=s.run_id AND j.sequence=s.sequence)
AND NOT (COALESCE(json_extract(x.export_json,'$.status'),'complete')='complete'
AND COALESCE(json_extract(x.export_json,'$.session_id'),'')=s.session_id
@@ -259,7 +259,7 @@ func TestSessionExportJobMigrationReconcilesMissingJobsWithoutReset(t *testing.T
}
func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) {
for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "wrong-session"} {
for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "pi-native", "wrong-session"} {
t.Run(status, func(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
@@ -272,10 +272,16 @@ func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) {
evidence.SessionID = "ses_wrong"
}
if status == "unsupported" {
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi'"); err != nil {
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference=''"); err != nil {
t.Fatal(err)
}
}
if status == "pi-native" {
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference='/original/session.jsonl'"); err != nil {
t.Fatal(err)
}
evidence.Backend, evidence.Status = "pi", "failed"
}
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", SessionExport: evidence, Output: map[string]any{}}); err != nil {
t.Fatal(err)
}
+2 -1
View File
@@ -11,7 +11,8 @@ import (
// legacy display lookup it never guesses a backend for a historical session.
func (s *Store) RepairSessionOwner(ctx context.Context, id int64, sequence int) (SessionOwner, error) {
var owner SessionOwner
err := s.db.QueryRowContext(ctx, `SELECT s.session_id,o.backend,r.workspace,o.reference FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence WHERE s.run_id=? AND s.sequence=? AND s.session_id<>'' AND s.agent_active=0`, id, sequence).Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference)
err := s.db.QueryRowContext(ctx, `SELECT s.session_id,o.backend,r.workspace,o.reference FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence WHERE s.run_id=? AND s.sequence=? AND s.session_id<>'' AND s.agent_active=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.session_id=s.session_id AND a.agent_active<>0)`, id, sequence).Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference)
return owner, err
}
+1
View File
@@ -417,6 +417,7 @@ func waitTerminal(t *testing.T, r *runner, id int64) *protocol.RunDetail {
// deterministically against B's active session.
func TestOverlappingRunsDoNotFailQuiescence(t *testing.T) {
_, r, pipelineID := agentStore(t)
r.exportSession = validGateExport // Quiescence, not export transport, is under test.
a, b := t.TempDir(), t.TempDir()
type agentRun struct {
+3
View File
@@ -95,6 +95,7 @@ esac
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait() }()
r.exportSession = validGateExport // Export is a separate mandatory handoff gate.
workspace, err := filepath.EvalSymlinks(dir)
if err != nil {
t.Fatal(err)
@@ -455,6 +456,7 @@ esac
t.Fatal(err)
}
handler := Handler(ctx, store)
handler.(*runHandler).runner.exportSession = validGateExport
defer func() { cancel(); WaitRuns(handler) }()
items, err := store.List(ctx)
if err != nil {
@@ -566,6 +568,7 @@ esac
toolsBlock := func(tools string) string {
return "# stage worker\n\n```output\nResult.\n```\n\n```prompt\nDo the work.\n```\n\n```tools\n" + tools + "\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=1\ntimeout=30s\n```\n"
}
r.exportSession = validGateExport
runStage := func(name, tools string) (status, sessions, diagnostic string) {
t.Helper()
workspace := t.TempDir()
+88 -12
View File
@@ -384,11 +384,26 @@ func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
// checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run
// reserves an overlapping resolved workspace. Callers must hold r.mu.
func (r *runner) checkWorkspaceFree(workspace string) error {
return r.checkWorkspaceFreeExcept(workspace, 0)
}
func (r *runner) checkWorkspaceFreeExcept(workspace string, ownGate int64) error {
for id, entry := range r.active {
if workspacesOverlap(entry.workspace, workspace) {
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
}
}
check, cancel := context.WithTimeout(context.WithoutCancel(r.ctx), 5*time.Second)
defer cancel()
reservations, err := r.store.SessionExportReservations(check)
if err != nil {
return fmt.Errorf("read pending export workspace reservations: %w", err)
}
for id, reserved := range reservations {
if id != ownGate && workspacesOverlap(reserved, workspace) {
return fmt.Errorf("run %d is awaiting its session export; workspace %s remains reserved", id, reserved)
}
}
return nil
}
@@ -694,6 +709,11 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
} else if existing > 0 {
return r.store.GetRun(r.ctx, existing)
}
if blocked, err := r.store.SessionExportBlocked(r.ctx, original.ID); err != nil {
return nil, err
} else if blocked {
return nil, fmt.Errorf("run %d has a retained computation awaiting export-only recovery; model retry refused", original.ID)
}
if request.RetryVisit < 0 || (request.RetryVisit > 0 && request.RetryStage == "") {
return nil, fmt.Errorf("retry visit requires a positive --stage visit selection")
}
@@ -947,6 +967,7 @@ type runState struct {
// retryOverride is applied once, after the selected stage's memory bindings
// have been resolved. It is intentionally not part of run input or memory.
retryOverride stage.Values
priorBudgets map[string]stage.BudgetSnapshot
}
func newRunState(key string, input stage.Values) *runState {
@@ -963,7 +984,18 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
state.memory["input"] = memoryValue{output: original}
}
publicSeen := map[string]bool{}
state.priorBudgets = map[string]stage.BudgetSnapshot{}
for _, step := range detail.Steps {
// The runtime stamps this cumulative snapshot on every executed model
// visit. Reusing it preserves per-stage limits across export recovery.
var audit struct {
Limits struct {
Budget stage.BudgetSnapshot `json:"budget"`
} `json:"stage_limits"`
}
if json.Unmarshal([]byte(step.Transcript), &audit) == nil {
state.priorBudgets[step.Stage] = audit.Limits.Budget
}
state.visits[step.Stage]++
if step.Sequence >= state.nextSequence {
state.nextSequence = step.Sequence + 1
@@ -1715,6 +1747,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
modelConfig = spec.Config
if budgets[key] == nil {
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
if used, ok := state.priorBudgets[key]; ok {
budgets[key].RestoreUsage(used)
}
}
// The budget is shared across every visit to this stage key, so capture
// its cumulative usage before the visit and record only this visit's
@@ -1808,15 +1843,32 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
cancel()
}
// Export the stage's session into its own artifacts directory before the
// bounded scan, so the existing listing and serving paths find the files
// with no special casing. This is bounded, best-effort evidence: it never
// waits for session idleness and a failure never changes the stage's
// status, outcome, target or result validation.
// Only settled, exact recorded ownership can satisfy the handoff gate.
// A canceled execution must not cancel this ownership read silently.
var sessionExport *protocol.SessionExport
exportRequired := artifactsDir != "" && r.execute == nil
if artifactsDir != "" {
if owner, ownerErr := r.store.RunSessionOwner(ctx, id, sequence); ownerErr == nil && owner.ID != "" {
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
ownerCtx, ownerCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
owner, ownerErr := r.store.RepairSessionOwner(ownerCtx, id, sequence)
if ownerErr != nil && r.execute != nil {
// Only genuinely unowned fake executors are exempt. A recorded
// session with absent ownership metadata must still fail closed.
if recorded, readErr := r.store.RunSessionOwner(ownerCtx, id, sequence); readErr != nil || recorded.ID != "" {
exportRequired = true
if readErr == nil {
stageSession = recorded.ID
}
}
}
ownerCancel()
if ownerErr == nil && owner.ID != "" {
exportRequired = true // Owned test executors exercise the real gate too.
stageSession = owner.ID
if !errors.Is(err, stage.ErrStageCleanup) {
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
}
} else if exportRequired {
sessionExport = &protocol.SessionExport{Status: "failed", Source: "native_export", Diagnostic: "exact settled session ownership unavailable; handoff refused"}
}
}
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir, visitTurn.PublicDir)
@@ -1915,6 +1967,28 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
if panicked {
step.Target = protocol.FailureEnd
}
_, _, _, failedTerminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
if exportRequired && !completeSessionExport(sessionExport) && !(err != nil && failedTerminal && step.Target != protocol.SuccessEnd) {
// Preserve the real computation, files, outcome and target. In
// particular an export failure is not an agent failure/retry request.
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
saveErr := r.store.FinishStepExportBlocked(save, id, step)
cancel()
if saveErr != nil {
finish("failed", "error", saveErr.Error())
return
}
diagnostic := "Session export gate blocked; computed visit retained; export-only recovery pending"
if sessionExport != nil && sessionExport.Diagnostic != "" {
diagnostic += ": " + sessionExport.Diagnostic
}
if err != nil {
diagnostic = err.Error() + "; " + diagnostic
}
// No finishStage, memory publication, recordTurn or next executor.
r.finishRunState(id, name, workspace, "failed", "error", diagnostic, state.completed, sequence, key, stage.Values(step.Output))
return
}
// Generate the one-line verdict reason once, at completion. It is
// best-effort: a summariser failure or timeout returns NoSummary and must
// never fail the stage or the run, so every error is swallowed. A failed
@@ -1985,14 +2059,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
// exportStageSession writes the finished stage's session into its own artifacts
// directory and returns the evidence for the step. A non-OpenCode harness (pi)
// has no native export path, so it records that limitation and writes no files.
// Any export error is recorded as a sanitized diagnostic and never returned to
// the stage: the export is read-only, bounded and best-effort.
// directory and returns verified evidence. The caller enforces the handoff gate;
// an export failure never replaces the computation's result.
func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport {
evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend, Directory: owner.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339)}
defer classifySessionExport(evidence)
if owner.Backend != "v1" && owner.Backend != "v2" {
if owner.Backend != "v1" && owner.Backend != "v2" && !(owner.Backend == "pi" && owner.Reference != "") {
evidence.Source = "unavailable"
evidence.Diagnostic = "session export unavailable: harness has no native export path"
return evidence
@@ -2008,6 +2080,7 @@ func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionO
result, err := export(exportCtx, stage.SessionExportRequest{
SessionID: owner.ID,
Backend: owner.Backend,
Reference: owner.Reference,
Directory: owner.Directory,
ArtifactsDir: artifactsDir,
RunID: runID,
@@ -2029,6 +2102,9 @@ func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionO
evidence.Messages = result.Messages
evidence.Truncated = result.Truncated
evidence.Diagnostic = result.Diagnostic
if result.Status != "" && result.Status != "complete" && evidence.Diagnostic == "" {
evidence.Diagnostic = "native session export reported " + result.Status
}
verifySessionExportFile(evidence, artifactsDir)
return evidence
}
+99
View File
@@ -0,0 +1,99 @@
package server
import (
"context"
"os"
"path/filepath"
"strconv"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func completeSessionExport(e *protocol.SessionExport) bool {
return e != nil && e.Status == "complete" && e.SessionID != "" && e.Messages > 0 && e.HTML != "" && e.HTMLBytes > 0 && !e.Truncated && e.Diagnostic == ""
}
// resumeExportGates is also run when no job is due: a manual export repair or
// a crash after persisting repair evidence may have made a handoff eligible.
func (r *runner) resumeExportGates(ctx context.Context) error {
ready, err := r.store.ReadySessionExportGates(ctx)
if err != nil {
return err
}
for id, sequence := range ready {
r.mu.Lock()
if _, active := r.active[id]; active {
r.mu.Unlock()
continue
}
d, err := r.store.GetRun(ctx, id)
if err != nil {
r.mu.Unlock()
return err
}
if err := r.checkWorkspaceFreeExcept(d.Workspace, id); err != nil {
r.mu.Unlock()
continue
}
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
if err != nil {
r.mu.Unlock()
continue
}
job, err := r.store.SessionExportJob(ctx, id, sequence)
if err != nil || job == nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
r.mu.Unlock()
continue
}
var saved *protocol.RunStep
for i := range d.Steps {
if d.Steps[i].Sequence == sequence {
saved = &d.Steps[i]
break
}
}
verified := false
if saved != nil {
for _, e := range saved.SessionExportRepairs {
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
continue
}
// New repairs stay private in the original visit. Historical
// public repairs remain valid, but are never rewritten there.
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
if strings.HasPrefix(e.HTML, "public/") {
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
continue
}
if filepath.Clean(e.HTML) != e.HTML {
continue
}
root, openErr := os.OpenRoot(directory)
if openErr != nil {
continue
}
info, statErr := root.Lstat(e.HTML)
root.Close()
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
verified = true
break
}
}
}
if !verified {
r.mu.Unlock()
continue
}
if err := r.store.ResumeSessionExportGate(ctx, id, sequence); err != nil {
r.mu.Unlock()
continue
}
step := *saved
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
r.mu.Unlock()
}
return nil
}
+339
View File
@@ -0,0 +1,339 @@
package server
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
type gateTestOptions struct {
missingOwner bool
waitForCancel bool
}
func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error), active bool, options ...gateTestOptions) (*runHandler, *pipeline.Store, int64, *atomic.Int32, *atomic.Int32) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
s, err := pipeline.Open(filepath.Join(t.TempDir(), "gate.db"))
if err != nil {
t.Fatal(err)
}
h := Handler(ctx, s).(*runHandler)
t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() })
h.runner.artifactsRoot = t.TempDir()
h.runner.exportSession = export
if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
ready := make(chan struct{})
var id int64
first, second := &atomic.Int32{}, &atomic.Int32{}
h.runner.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
<-ready
if def.Key == "one" {
first.Add(1)
metadata := []string{"v2", "exact-ref"}
if len(options) > 0 && options[0].missingOwner {
metadata = nil
}
if err := s.RunSession(ctx, id, 1, "ses_exact", active, metadata...); err != nil {
return stage.Result{}, err
}
if len(options) > 0 && options[0].waitForCancel {
<-ctx.Done()
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "computed output"}}, nil
}
second.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
d, err := h.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "gate", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
id = d.ID
close(ready)
return h, s, id, first, second
}
func validGateExport(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil {
return stage.SessionExportResult{}, err
}
name := req.FileName
if name == "" {
name = stage.SessionHTMLFile
}
body := []byte("<!doctype html><html>exact session</html>")
if err := os.WriteFile(filepath.Join(req.ArtifactsDir, name), body, 0600); err != nil {
return stage.SessionExportResult{}, err
}
return stage.SessionExportResult{Source: "native_export", Status: "complete", HTML: name, HTMLBytes: int64(len(body)), Messages: 2}, nil
}
func TestSessionExportGateOrdersExecutors(t *testing.T) {
entered, release := make(chan struct{}), make(chan struct{})
h, s, id, first, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if req.SessionID != "ses_exact" || req.Backend != "v2" || req.Reference != "exact-ref" {
t.Errorf("wrong owner: %+v", req)
}
close(entered)
<-release
return validGateExport(ctx, req)
}, false)
<-entered
if second.Load() != 0 {
t.Fatal("next executor ran before verified export")
}
close(release)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
t.Fatalf("handoff failed: %+v", d)
}
}
func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("export offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "failed" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
t.Fatalf("lost computation/advanced: %+v", d)
}
h.runner.exportSession = validGateExport
job, _ := s.SessionExportJob(ctx, id, 1)
if job == nil {
t.Fatal("missing durable recovery job")
}
if _, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
t.Fatal(err)
}
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ = s.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || len(d.Steps) != 2 || d.Status != "succeeded" {
t.Fatalf("export retry replayed execution: %+v (%d/%d)", d, first.Load(), second.Load())
}
}
func TestSessionExportGateCleanupUnconfirmedDoesNotExportOrAdvance(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, true)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
t.Fatalf("unconfirmed cleanup advanced/exported: %+v", d)
}
}
func TestSessionExportGateCanceledCleanupUnconfirmedDoesNotAdvance(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, true, gateTestOptions{waitForCancel: true})
deadline := time.Now().Add(3 * time.Second)
for {
d, _ := s.GetRun(context.Background(), id)
if len(d.Steps) > 0 && d.Steps[0].AgentActive {
break
}
if time.Now().After(deadline) {
t.Fatal("executor did not acquire session")
}
time.Sleep(time.Millisecond)
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if !d.CancelRequested || !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
t.Fatalf("canceled unconfirmed owner advanced: %+v", d)
}
}
func TestSessionExportGateCanceledExecutionStillReadsExactOwnerAndExports(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if ctx.Err() != nil || req.SessionID != "ses_exact" || req.Reference != "exact-ref" {
t.Errorf("lost bounded uncanceled exact owner: %+v %v", req, ctx.Err())
}
exports.Add(1)
return validGateExport(ctx, req)
}, false, gateTestOptions{waitForCancel: true})
deadline := time.Now().Add(3 * time.Second)
for {
d, _ := s.GetRun(context.Background(), id)
if len(d.Steps) > 0 && d.Steps[0].SessionID != "" {
break
}
if time.Now().After(deadline) {
t.Fatal("executor did not record session")
}
time.Sleep(time.Millisecond)
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "canceled" || second.Load() != 0 || exports.Load() != 1 || d.Steps[0].SessionExport == nil {
t.Fatalf("canceled ownership/export was silently lost: %+v", d)
}
}
func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, false, gateTestOptions{missingOwner: true})
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
t.Fatalf("missing owner did not fail closed: %+v", d)
}
}
func TestSessionExportGateReservesWorkspaceWithoutExecutingAndCanCancel(t *testing.T) {
h, s, id, _, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
d, err := h.runner.getRun(id)
if err != nil || d.Executing {
t.Fatalf("parked export mislabeled executing: %+v %v", d, err)
}
pending, err := s.PendingSessionExportGate(ctx, id, 1)
if err != nil || !pending {
t.Fatalf("quiescent pending visit not exposed: %v %v", pending, err)
}
request := protocol.Message{PipelineID: d.PipelineID, Version: 1, RequestKey: "overlap", Workspace: d.Workspace, Input: map[string]any{"prompt": "another task"}}
if _, err := h.runner.start(request); err == nil {
t.Fatal("parked export released workspace")
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
pending, err = s.PendingSessionExportGate(ctx, id, 1)
if err != nil || pending {
t.Fatalf("canceled gate remains resumable: %v %v", pending, err)
}
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
request.RequestKey = "after-cancel"
if _, err := h.runner.start(request); err != nil {
t.Fatalf("cancel did not release workspace: %v", err)
}
h.runner.wg.Wait()
if second.Load() != 0 {
t.Fatal("canceled gate advanced")
}
}
func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
for _, mode := range []string{"missing", "empty", "partial", "wrong-size", "symlink"} {
t.Run(mode, func(t *testing.T) {
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
result, err := validGateExport(ctx, req)
path := filepath.Join(req.ArtifactsDir, result.HTML)
switch mode {
case "missing":
err = os.Remove(path)
case "empty":
result.Status = "empty"
case "partial":
result.Status = "partial"
case "wrong-size":
result.HTMLBytes++
case "symlink":
if err = os.Remove(path); err == nil {
err = os.Symlink("input.md", path)
}
}
return result, err
}, false)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || d.Output["markdown"] != "computed output" {
t.Fatalf("invalid export advanced: %+v", d)
}
})
}
}
func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
if err := s.Close(); err != nil {
t.Fatal(err)
}
reopened, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunner(ctx, reopened)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.artifactsRoot = root
r.exportSession = validGateExport
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
t.Errorf("replayed owner: %s", def.Key)
}
second.Add(1)
body, err := os.ReadFile(def.Turn.Input)
if err != nil || !strings.Contains(string(body), "computed output") {
t.Errorf("lost computed handoff: %q %v", body, err)
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
job, _ := reopened.SessionExportJob(ctx, id, 1)
if _, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
t.Fatal(err)
}
if err := r.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
r.wg.Wait()
if err := r.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
r.wg.Wait()
d, _ := reopened.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
t.Fatalf("restart replayed or lost successor: %+v", d)
}
}
+3
View File
@@ -64,6 +64,9 @@ func (r *runner) startSessionExportJobs() error {
break
}
}
if err := r.resumeExportGates(ctx); err != nil && ctx.Err() == nil {
log.Printf("session export handoff continuation failed: %v", err)
}
}
}()
return nil
+3 -2
View File
@@ -190,7 +190,8 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
if err != nil {
t.Fatal(err)
}
if job != nil && job.Status == "complete" {
settled, _ := s.GetRun(ctx, id)
if job != nil && job.Status == "complete" && settled.Status == "succeeded" {
break
}
if time.Now().After(deadline) {
@@ -203,7 +204,7 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
t.Fatal(err)
}
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != original.Status {
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "failed" {
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
}
}
+28 -5
View File
@@ -12,6 +12,7 @@ import (
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
@@ -25,8 +26,8 @@ const sessionExportBody = `{"info":{"id":"ses_run"},"messages":[` +
`{"type":"text","text":"done"},` +
`{"type":"tool","name":"shell","state":{"status":"completed","input":{"cmd":"ls"},"content":[{"type":"text","text":"output here"}]}}]}]}`
// sessionExportCLI points PIPELINES_OPENCODE at a fake `opencode export` that
// returns sessionExportBody, so the runner's real export path runs.
// sessionExportCLI isolates both the authenticated V2 API and legacy CLI.
// Neither export transport may fall through to the operator's real service.
func sessionExportCLI(t *testing.T) {
t.Helper()
path := filepath.Join(t.TempDir(), "opencode")
@@ -38,6 +39,28 @@ func sessionExportCLI(t *testing.T) {
t.Fatal(err)
}
t.Setenv("PIPELINES_OPENCODE", path)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, password, ok := r.BasicAuth()
if !ok || user != "opencode" || password != "fixture-password" {
w.WriteHeader(http.StatusUnauthorized)
return
}
if r.Method != "GET" || r.URL.Path != "/api/experimental/session/ses_run/export" || r.URL.Query().Get("sanitize") != "false" {
t.Errorf("unexpected export operation: %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":`+sessionExportBody+`}`)
}))
t.Cleanup(srv.Close)
passwordPath := filepath.Join(t.TempDir(), "password")
if err := os.WriteFile(passwordPath, []byte("fixture-password"), 0600); err != nil {
t.Fatal(err)
}
t.Setenv(opencode.EndpointEnvKey, srv.URL)
t.Setenv(opencode.EndpointPasswordFileEnvKey, passwordPath)
t.Setenv(opencode.EndpointUsernameEnvKey, "")
}
// sessionExportRunner runs one stage that records a session for the run, so the
@@ -145,7 +168,7 @@ func TestRunnerExportsTheHTMLSessionIntoArtifactsAndServesIt(t *testing.T) {
}
// TestRunnerSessionExportFailureTruncationAndUnavailableHarness proves the three
// non-success export outcomes are recorded as evidence and never fail the run.
// non-success exports block success without replacing the completed computation.
func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T) {
cases := []struct {
name string
@@ -188,8 +211,8 @@ func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" {
t.Fatalf("export outcome failed the run: %+v", detail.Status)
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "Session export gate blocked") || detail.Output["summary"] != "done" {
t.Fatalf("export gate did not preserve and block computation: %+v", detail)
}
step := detail.Steps[0]
if step.Status != "completed" || step.Outcome != "success" {
+1
View File
@@ -61,6 +61,7 @@ esac
t.Fatal(err)
}
handler := Handler(ctx, store)
handler.(*runHandler).runner.exportSession = validGateExport
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
+1
View File
@@ -116,6 +116,7 @@ func TestRunnerRecordsPerStageUsageAcrossMixedModels(t *testing.T) {
if err != nil {
t.Fatal(err)
}
r.exportSession = validGateExport
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mixed", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
if err != nil {
t.Fatal(err)
+9
View File
@@ -99,6 +99,15 @@ type Budget struct {
func NewBudget(usd float64, tokens int64) *Budget {
return &Budget{state: BudgetSnapshot{LimitUSD: usd, LimitTokens: tokens}, records: map[string]BudgetSnapshot{}, pollInterval: time.Second}
}
// RestoreUsage seeds persisted earlier visits when a durable handoff resumes.
// New sessions have new accounting keys; prior records must not be polled again.
func (b *Budget) RestoreUsage(used BudgetSnapshot) {
b.mu.Lock()
defer b.mu.Unlock()
used.LimitUSD, used.LimitTokens = b.state.LimitUSD, b.state.LimitTokens
b.state = used
}
func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) }
func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state }