runner: keep export handoffs nonterminal through recovery

This commit is contained in:
2026-10-20 21:19:26 +01:00
parent ca78b24f47
commit ec3eda9e05
28 changed files with 370 additions and 82 deletions
+13 -2
View File
@@ -7,8 +7,10 @@ reads and final export attempts have bounded contexts independent of execution
cancellation. Unconfirmed cleanup never exports or advances.
An export failure retains the computed step, outcome, target, output and file
evidence. The run fails closed with an export-gate diagnostic, not an invented
model failure. A separate `session_export_gates` checkpoint is committed with the
evidence. The run parks nonterminally in `waiting`, with
`export_pending_sequence` identifying the computed visit and no `finished_at`.
This is not a human decision or a model failure. A separate
`session_export_gates` checkpoint is committed with the
step and its existing durable export job. Model retry is refused for that
checkpoint: recovery reads only the original session. Missing exact ownership
cannot be reconstructed or guessed and remains blocked.
@@ -18,6 +20,15 @@ The checkpoint reserves the workspace durably even though no agent is executing.
`PendingSessionExportGate(runID, sequence)` identifies the exact quiescent visit
eligible for export-only repair. Cancellation releases the reservation without
discarding or replaying the computation, and prevents automatic continuation.
The repair handler must permit `waiting` only when that predicate is true for
the requested sequence; unrelated human/resource waits remain ineligible.
Both CLI wait paths keep waiting for export recovery; `run --wait` still pauses
for genuine human decisions. Lifecycle events carry the same pending sequence,
and a lifecycle frame with zero/absent sequence clears it. Status-less repair
freshness frames leave it unchanged. Continuation is `waiting -> running`, never
terminal -> running. Startup normalizes historical failed/interrupted export
checkpoints back to waiting before clients attach.
After a successful automatic or manual repair, the worker re-verifies ownership
(including backend/reference) and the actual file, then rebuilds memory, turn
+5 -5
View File
@@ -1461,16 +1461,16 @@ func (a App) wait(ctx context.Context, pos []string, o *options, call callFunc)
return nil
}
// awaitRuns polls every run that active reports as unfinished until each is
// terminal, using the same GetRun reattachment the run --wait path uses, and
// calls onTerminal for each as it finishes (including runs already terminal at
// awaitRuns polls unfinished runs using GetRun reattachment. Export waits remain
// unfinished even for run --wait, whose status-only predicate intentionally stops
// at human decisions. It calls onTerminal for each as it finishes (including runs already terminal at
// entry, in the order supplied). A deadline that passes first returns a
// *waitLimitError naming the runs still active; ctx ending first returns
// ctx.Err(). It never cancels, retries or otherwise changes a run.
func (a App) awaitRuns(ctx context.Context, runs []protocol.Message, deadline <-chan time.Time, call callFunc, active func(string) bool, onTerminal func(protocol.Message) error) error {
remaining := make([]protocol.Message, 0, len(runs))
for _, r := range runs {
if active(r.RunDetail.Status) {
if active(r.RunDetail.Status) || (r.RunDetail.Status == protocol.RunStatusWaiting && r.RunDetail.ExportPendingSequence > 0) {
remaining = append(remaining, r)
continue
}
@@ -1495,7 +1495,7 @@ func (a App) awaitRuns(ctx context.Context, runs []protocol.Message, deadline <-
if refreshed.RunDetail == nil {
return fmt.Errorf("server did not return run detail for %d", r.RunDetail.ID)
}
if active(refreshed.RunDetail.Status) {
if active(refreshed.RunDetail.Status) || (refreshed.RunDetail.Status == protocol.RunStatusWaiting && refreshed.RunDetail.ExportPendingSequence > 0) {
next = append(next, refreshed)
continue
}
+40
View File
@@ -0,0 +1,40 @@
package cli
import (
"context"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestRunAndExplicitWaitDoNotReturnDuringExportRecovery(t *testing.T) {
for _, mode := range []string{"run --wait", "wait"} {
t.Run(mode, func(t *testing.T) {
active := runInProgress
if mode == "wait" {
active = runUnfinished
}
initial := protocol.Message{RunDetail: &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: protocol.RunStatusWaiting, ExportPendingSequence: 2}}}
polls, finished := 0, false
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err := (App{}).awaitRuns(ctx, []protocol.Message{initial}, nil, func(request protocol.Message) (protocol.Message, error) {
polls++
if request.Type != protocol.GetRun || request.RunID != 7 {
t.Errorf("unexpected request: %+v", request)
}
return protocol.Message{RunDetail: &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: "succeeded"}}}, nil
}, active, func(message protocol.Message) error {
if message.RunDetail.Status != "succeeded" {
t.Errorf("wait returned before actual terminal: %+v", message.RunDetail)
}
finished = true
return nil
})
if err != nil || polls != 1 || !finished {
t.Fatalf("export wait ended early: polls=%d finished=%v error=%v", polls, finished, err)
}
})
}
}
+3 -2
View File
@@ -22,12 +22,13 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
` + runStamps + `,r.diagnostic,
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format,
CASE WHEN r.status='waiting' THEN COALESCE((SELECT sequence FROM session_export_gates WHERE run_id=r.id),0) ELSE 0 END
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output, stageModels string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat); err != nil {
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence); err != nil {
return err
}
if stageModels != "" && stageModels != "{}" {
+14 -1
View File
@@ -168,6 +168,9 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome,
if _, err := tx.ExecContext(ctx, `INSERT INTO session_export_gates(run_id,sequence) VALUES(?,?)`, id, step.Sequence); err != nil {
return err
}
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',diagnostic='Session export pending; computed visit retained',output_json=?,finished_at=NULL WHERE id=? AND status='running'`, string(output), id)); err != nil {
return err
}
}
// The reported usage the stage already measured for budget accounting is
// persisted as step-scoped evidence in the same transaction, mirroring the
@@ -226,6 +229,13 @@ func (s *Store) ResolveDecision(ctx context.Context, id int64, stageKey, option,
return protocol.RunStep{}, false, err
}
defer tx.Rollback()
var exportPending bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&exportPending); err != nil {
return protocol.RunStep{}, false, err
}
if exportPending {
return protocol.RunStep{}, false, fmt.Errorf("run %d is awaiting session export, not a human decision", id)
}
var status, currentStage, graphJSON string
if err := tx.QueryRowContext(ctx, `SELECT r.status,r.current_stage,v.graph_json FROM pipeline_runs r
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&status, &currentStage, &graphJSON); err != nil {
@@ -253,6 +263,9 @@ JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version
found := false
for _, declared := range graph.Stages {
if declared.Key == stageKey {
if stage.EffectiveType(declared.Type) != stage.HumanType {
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not a human decision stage", stageKey)
}
edges = declared.Edges
found = true
break
@@ -324,7 +337,7 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
}
defer tx.Rollback()
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,
diagnostic=CASE WHEN status='waiting' THEN 'Canceled while waiting for a decision' ELSE 'Canceled while awaiting session export; computed visit retained' END,
diagnostic=CASE WHEN EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id) THEN 'Canceled while awaiting session export; computed visit retained' ELSE 'Canceled while waiting for a decision' END,
finished_at=CURRENT_TIMESTAMP WHERE id=? AND (status=? OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id, protocol.RunStatusWaiting)
if err != nil {
return err
+9
View File
@@ -204,5 +204,14 @@ func (s *Store) RecoverRuns(ctx context.Context) error {
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
return err
}
// An export checkpoint is a paused handoff, not an interrupted model call.
// Also migrate the earlier failed-gate representation before clients attach.
if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',finished_at=NULL,diagnostic='Session export pending; computed visit retained'
WHERE status IN ('queued','running','failed','interrupted','waiting') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
WHERE g.run_id=pipeline_runs.id AND s.status IN ('completed','failed') AND s.agent_active=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.sequence>g.sequence OR a.agent_active<>0)))`); err != nil {
return err
}
return tx.Commit()
}
+3 -3
View File
@@ -47,7 +47,7 @@ func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, err
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
JOIN pipeline_runs r ON r.id=g.run_id
WHERE j.status='complete' AND r.status IN ('failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
WHERE j.status='complete' AND r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`)
if err != nil {
return nil, err
@@ -67,8 +67,8 @@ AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_ac
// ResumeSessionExportGate is a CAS, not a retry of the model invocation.
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='queued',outcome='',diagnostic='',finished_at=NULL
WHERE id=? AND status IN ('failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`)
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence))
}
+17 -4
View File
@@ -18,8 +18,12 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
t.Fatal("successor started while export was incomplete")
}
if err := s.FinishRun(ctx, id, "failed", "error", "export gate", step.Output); err != nil {
t.Fatal(err)
paused, err := s.GetRun(ctx, id)
if err != nil || paused.Status != "waiting" || paused.ExportPendingSequence != 1 || paused.FinishedAt != "" {
t.Fatalf("gate became terminal: %+v %v", paused, err)
}
if _, _, err := s.ResolveDecision(ctx, id, "one", "success", ""); err == nil {
t.Fatal("model export wait accepted human decision")
}
job, _ := s.SessionExportJob(ctx, id, 1)
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
@@ -40,8 +44,17 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
}
// A crash after reserving the continuation but before starting its next
// executor must retain the checkpoint for startup's interrupted run.
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status='interrupted' WHERE id=?`, id); err != nil {
t.Fatal(err)
for _, prior := range []string{"failed", "interrupted", "running", "queued"} {
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status=?,finished_at=CURRENT_TIMESTAMP WHERE id=?`, prior, id); err != nil {
t.Fatal(err)
}
if err := s.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
recovered, err := s.GetRun(ctx, id)
if err != nil || recovered.Status != "waiting" || recovered.ExportPendingSequence != 1 || recovered.FinishedAt != "" {
t.Fatalf("restart exposed a terminal export wait from %s: %+v %v", prior, recovered, err)
}
}
ready, err := s.ReadySessionExportGates(ctx)
if err != nil || ready[id] != 1 {
+1 -1
View File
@@ -139,7 +139,7 @@ func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease
err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic
FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence
WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts<?
AND r.status IN ('succeeded','failed','canceled','interrupted') AND s.status NOT IN ('running','waiting')`+guard+`
AND (r.status IN ('succeeded','failed','canceled','interrupted') OR (r.status='waiting' AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=j.run_id AND g.sequence=j.sequence))) AND s.status NOT IN ('running','waiting')`+guard+`
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=j.run_id OR a.session_id=j.session_id) AND a.agent_active<>0)
ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
if err == sql.ErrNoRows {
+14 -12
View File
@@ -37,18 +37,20 @@ type RunEventFrame struct {
Workspace string `json:"workspace,omitempty"`
// PipelineID, Name, Title and Version identify the run's pipeline and its
// operator-facing title, so a new run's row can be built without a fetch.
PipelineID int64 `json:"pipeline_id,omitempty"`
Name string `json:"name,omitempty"`
Title string `json:"title,omitempty"`
Version int `json:"version,omitempty"`
Status string `json:"status,omitempty"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Outcome string `json:"outcome,omitempty"`
CurrentStage string `json:"current_stage,omitempty"`
CompletedSteps int `json:"completed_steps,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
FinishedAt string `json:"finished_at,omitempty"`
PipelineID int64 `json:"pipeline_id,omitempty"`
Name string `json:"name,omitempty"`
Title string `json:"title,omitempty"`
Version int `json:"version,omitempty"`
Status string `json:"status,omitempty"`
// On a lifecycle frame, zero/absence clears a prior export wait.
ExportPendingSequence int `json:"export_pending_sequence,omitempty"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Outcome string `json:"outcome,omitempty"`
CurrentStage string `json:"current_stage,omitempty"`
CompletedSteps int `json:"completed_steps,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
FinishedAt string `json:"finished_at,omitempty"`
// Artifacts is the number of files recorded for the referenced step, so a
// client can surface artifact capture without reading the step.
Artifacts int `json:"artifacts,omitempty"`
+18 -15
View File
@@ -142,21 +142,24 @@ type RunSummary struct {
// Title is the operator-facing, bounded single-line description of what the
// run is for, generated from its prompt. It is never empty for a new run;
// historical rows predating the title column may omit it.
Title string `json:"title,omitempty"`
Version int `json:"version"`
ContentHash string `json:"content_hash"`
Status string `json:"status"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Executing bool `json:"executing"`
Outcome string `json:"outcome"`
CurrentStage string `json:"current_stage"`
Workspace string `json:"workspace,omitempty"`
CreatedAt string `json:"created_at"`
FinishedAt string `json:"finished_at"`
Diagnostic string `json:"diagnostic"`
ResultPreview string `json:"result_preview,omitempty"`
CompletedSteps int `json:"completed_steps"`
Title string `json:"title,omitempty"`
Version int `json:"version"`
ContentHash string `json:"content_hash"`
Status string `json:"status"`
// ExportPendingSequence distinguishes a quiescent export handoff from a
// human decision while Status is waiting. It never implies agent execution.
ExportPendingSequence int `json:"export_pending_sequence,omitempty"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Executing bool `json:"executing"`
Outcome string `json:"outcome"`
CurrentStage string `json:"current_stage"`
Workspace string `json:"workspace,omitempty"`
CreatedAt string `json:"created_at"`
FinishedAt string `json:"finished_at"`
Diagnostic string `json:"diagnostic"`
ResultPreview string `json:"result_preview,omitempty"`
CompletedSteps int `json:"completed_steps"`
// Model and StageModels are the launch-time overrides this run records so it
// stays reproducible: Model covers every model stage and StageModels overrides
// individual stages by key. They are empty for a run that used the pinned
+18 -16
View File
@@ -451,6 +451,7 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
}
if in.Status != "" {
into.Status = in.Status
into.ExportPendingSequence = in.ExportPendingSequence
}
if in.Outcome != "" {
into.Outcome = in.Outcome
@@ -509,22 +510,23 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
GraphEvidence: summary.GraphEvidence,
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
GraphEvidence: summary.GraphEvidence,
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
ExportPendingSequence: summary.ExportPendingSequence,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
+25 -8
View File
@@ -335,6 +335,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
}
if detail != nil {
_, detail.Executing = r.active[id]
if detail.ExportPendingSequence > 0 {
detail.Executing = false
}
r.overlayEffectiveModels(detail)
}
if readErr != nil {
@@ -421,6 +424,9 @@ func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
d, err := r.store.GetRun(r.ctx, id)
if d != nil {
_, d.Executing = r.active[id]
if d.ExportPendingSequence > 0 {
d.Executing = false
}
for _, s := range d.Steps {
if s.AgentActive {
d.Executing = true
@@ -467,6 +473,9 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
items, err := r.store.ListRuns(r.ctx)
for i := range items {
_, items[i].Executing = r.active[items[i].ID]
if items[i].ExportPendingSequence > 0 {
items[i].Executing = false
}
if items[i].Status == "failed" && !items[i].Executing {
d, err := r.store.GetRun(r.ctx, items[i].ID)
if err != nil {
@@ -797,6 +806,10 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
}
if detail.ExportPendingSequence > 0 {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is awaiting session export, not a human decision", detail.ID)
}
if detail.Status == protocol.RunStatusWaiting {
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
r.mu.Unlock()
@@ -1978,15 +1991,19 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
finish("failed", "error", saveErr.Error())
return
}
diagnostic := "Session export gate blocked; computed visit retained; export-only recovery pending"
if sessionExport != nil && sessionExport.Diagnostic != "" {
diagnostic += ": " + sessionExport.Diagnostic
}
if err != nil {
diagnostic = err.Error() + "; " + diagnostic
}
// No finishStage, memory publication, recordTurn or next executor.
r.finishRunState(id, name, workspace, "failed", "error", diagnostic, state.completed, sequence, key, stage.Values(step.Output))
check, done := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
detail, readErr := r.store.GetRun(check, id)
done()
if readErr != nil {
log.Printf("read parked export gate for run %d: %v", id, readErr)
return // The checkpoint is durable; never fabricate a terminal.
}
if detail.CancelRequested && detail.ExportPendingSequence == 0 {
finish("canceled", "", "Canceled by operator; computed visit retained")
return
}
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, ExportPendingSequence: sequence, CurrentStage: key, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts)}}, true)
return
}
// Generate the one-line verdict reason once, at completion. It is
+1
View File
@@ -92,6 +92,7 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
continue
}
step := *saved
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
r.mu.Unlock()
}
+3 -3
View File
@@ -111,7 +111,7 @@ func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "failed" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
t.Fatalf("lost computation/advanced: %+v", d)
}
h.runner.exportSession = validGateExport
@@ -214,7 +214,7 @@ func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
}, false, gateTestOptions{missingOwner: true})
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
t.Fatalf("missing owner did not fail closed: %+v", d)
}
}
@@ -281,7 +281,7 @@ func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
}, false)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || d.Output["markdown"] != "computed output" {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" {
t.Fatalf("invalid export advanced: %+v", d)
}
})
+1 -1
View File
@@ -204,7 +204,7 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
t.Fatal(err)
}
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "failed" {
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
}
}
+1 -1
View File
@@ -211,7 +211,7 @@ func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T)
if err != nil {
t.Fatal(err)
}
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "Session export gate blocked") || detail.Output["summary"] != "done" {
if detail.Status != "waiting" || detail.ExportPendingSequence != 1 || detail.FinishedAt != "" || detail.Output["summary"] != "done" {
t.Fatalf("export gate did not preserve and block computation: %+v", detail)
}
step := detail.Steps[0]
@@ -0,0 +1,89 @@
package server
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestExportWaitEventCoalescingClearsOnlyOnLifecycleTransition(t *testing.T) {
frame := protocol.RunEventFrame{RunID: 7, Status: "waiting", ExportPendingSequence: 2}
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Artifacts: 1})
if frame.ExportPendingSequence != 2 {
t.Fatal("repair-only frame cleared export pending")
}
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Status: "running"})
if frame.ExportPendingSequence != 0 || frame.Status != "running" {
t.Fatalf("continuation did not clear export wait: %+v", frame)
}
}
func TestExportWaitPrivateRepairResumesWithoutTerminalRegression(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil {
t.Fatalf("missing recovery checkpoint: %v", err)
}
name := "session-repair-1-private-test.html"
result, err := validGateExport(ctx, stage.SessionExportRequest{ArtifactsDir: h.runner.turnPaths(id, 1, "one").TurnDir, FileName: name})
if err != nil {
t.Fatal(err)
}
evidence := &protocol.SessionExport{Status: "complete", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Source: "native_export", HTML: name, HTMLBytes: result.HTMLBytes, Messages: result.Messages}
if err := s.RecordDeliverable(ctx, id, 1, protocol.StageArtifact{Name: name, SizeBytes: result.HTMLBytes, ContentType: "text/html"}); err != nil {
t.Fatal(err)
}
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
t.Fatal(err)
}
// Reconciliation recognizes a durable manual repair without another export.
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
t.Fatalf("repair triggered a redundant export: %+v %v", claimed, err)
}
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
d, err := h.runner.getRun(id)
if err != nil || d.Status != "succeeded" || d.ExportPendingSequence != 0 || first.Load() != 1 || second.Load() != 1 {
t.Fatalf("private repair replayed or lost continuation: %+v %v", d, err)
}
input, err := os.ReadFile(h.runner.turnPaths(id, 2, "two").InputPath)
if err != nil || !strings.Contains(string(input), "computed output") || strings.Contains(string(input), name) {
t.Fatalf("private export leaked or computed output lost: %q %v", input, err)
}
}
func TestExportWaitSubscriptionSnapshotCarriesPendingSequence(t *testing.T) {
h, _, id, _, _ := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
hub := newRunEventHub()
defer hub.unsubscribeClient("export-wait-test")
var snapshot *protocol.RunEventFrame
if !handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{id}}, hub, h.runner, "export-wait-test", func(message protocol.Message) error {
if message.RunEvent != nil {
snapshot = message.RunEvent
}
return nil
}) {
t.Fatal("subscribe not handled")
}
if snapshot == nil || snapshot.Status != "waiting" || snapshot.ExportPendingSequence != 1 || snapshot.FinishedAt != "" {
t.Fatalf("snapshot lost nonterminal export wait: %+v", snapshot)
}
}
+4 -1
View File
@@ -27,6 +27,9 @@ func lastWaitingStep(r *protocol.RunDetail) (protocol.RunStep, bool) {
// waiting step and falls back to the pinned graph's edges when a step recorded
// no options.
func (m Model) waitingStage() (string, []string, string) {
if m.runDetail != nil && m.runDetail.ExportPendingSequence > 0 {
return m.runDetail.CurrentStage, nil, ""
}
if m.runDetail == nil {
return "", nil, ""
}
@@ -43,7 +46,7 @@ func (m Model) waitingStage() (string, []string, string) {
}
func (m Model) openDecision() (tea.Model, tea.Cmd) {
if m.runDetail == nil || m.runDetail.Status != protocol.RunStatusWaiting {
if m.runDetail == nil || m.runDetail.Status != protocol.RunStatusWaiting || m.runDetail.ExportPendingSequence > 0 {
m.runError = "This run is not waiting for a decision."
return m, nil
}
+25
View File
@@ -0,0 +1,25 @@
package tui
import (
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestExportWaitRendersRecoveryNotHumanDecision(t *testing.T) {
run := &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: "waiting", ExportPendingSequence: 2, CurrentStage: "worker"}, Graph: protocol.Graph{Stages: []protocol.Stage{{Key: "worker", Type: "model", Edges: map[string]string{"success": "next", "error": "#failure"}}}}}
frame := strings.Join(waitingLines(run), "\n")
if !strings.Contains(frame, "Waiting for session export") || strings.Contains(frame, "person") || strings.Contains(frame, "Decide:") {
t.Fatalf("export wait offered a human choice:\n%s", frame)
}
m := Model{runDetail: run}
_, options, _ := m.waitingStage()
if len(options) != 0 {
t.Fatal("model-stage edges exposed as human options")
}
updated, _ := m.openDecision()
if updated.(Model).decisionOpen {
t.Fatal("opened decision for export wait")
}
}
+6
View File
@@ -18,6 +18,9 @@ func resultLabel(r protocol.RunSummary) string {
}
return "Finished · no result recorded"
}
if r.Status == protocol.RunStatusWaiting && r.ExportPendingSequence > 0 {
return "Waiting for session export · stage " + r.CurrentStage
}
if r.Diagnostic != "" {
return "Stopped: " + ansi.Strip(r.Diagnostic)
}
@@ -111,6 +114,9 @@ func waitingLines(r *protocol.RunDetail) []string {
if r.Status != protocol.RunStatusWaiting {
return nil
}
if r.ExportPendingSequence > 0 {
return []string{fmt.Sprintf("⏸ Waiting for session export · stage %s · visit %d", r.CurrentStage, r.ExportPendingSequence), "Computed output retained; automatic export-only recovery pending."}
}
lines := []string{"⏸ Waiting for a person · stage " + r.CurrentStage}
if step, ok := lastWaitingStep(r); ok {
if prompt := strings.TrimSpace(protocol.WaitingPrompt(step.Output)); prompt != "" {
+2 -2
View File
@@ -657,7 +657,7 @@ func (m Model) runActions() string {
if m.runDetail != nil {
text += " · a evidence · A artifacts"
}
if m.runDetail != nil && m.runDetail.Status == protocol.RunStatusWaiting {
if m.runDetail != nil && m.runDetail.Status == protocol.RunStatusWaiting && m.runDetail.ExportPendingSequence == 0 {
text += " · c decide"
}
if m.selectedFailedRun() > 0 {
@@ -740,7 +740,7 @@ func (m Model) runsDetailsView(height int) string {
page := max(0, innerHeight-5)
top := max(0, min(m.runTop, max(0, len(lines)-page)))
footerHint := "o result · d diff · s progress · i input · t evidence · m details · A artifacts"
if r.Status == protocol.RunStatusWaiting {
if r.Status == protocol.RunStatusWaiting && r.ExportPendingSequence == 0 {
footerHint += " · c decide"
}
footer := bar(footerHint, fmt.Sprintf("%d–%d / %d", min(top+1, len(lines)), min(top+page, len(lines)), len(lines)), max(1, m.width-4), false)
+3 -1
View File
@@ -96,6 +96,7 @@ export function decisionTargetLabel(option, target) {
*/
export function pendingDecision(run) {
if (run == null || run.status !== RUN_STATUS_WAITING) return null;
if (Number(run.export_pending_sequence) > 0) return null;
const step = lastWaitingStep(run.steps ?? []);
const stepStage = str(step?.stage).trim() !== "" ? String(step.stage) : "";
const runStage = str(run.current_stage).trim() !== "" ? String(run.current_stage) : "";
@@ -103,6 +104,7 @@ export function pendingDecision(run) {
if (stageKey === "") return null;
const declared = (run.graph?.stages ?? []).find((candidate) => candidate?.key === stageKey) ?? null;
if (declared?.type && declared.type !== "human") return null;
const recorded = waitingOptions(step?.output);
const keys = recorded.length > 0 ? recorded : humanOptions(declared?.edges);
if (keys.length === 0) return null;
@@ -154,7 +156,7 @@ export class DecisionDraft {
/** Cancel is offered only while the run is executing. */
export function canCancel(run) {
return Boolean(run?.executing) && !run.cancel_requested && !run.cleanup_unconfirmed;
return (Boolean(run?.executing) || (run?.status === RUN_STATUS_WAITING && Number(run.export_pending_sequence) > 0)) && !run.cancel_requested && !run.cleanup_unconfirmed;
}
/** Retry is offered only for a loaded, non-executing, `failed` run. */
+3 -2
View File
@@ -33,7 +33,8 @@ export const AttentionKind = Object.freeze({
* so queued, running, succeeded, canceled, interrupted and any unknown status
* can never invent an entry.
*/
export function attentionKindOf(status) {
export function attentionKindOf(status, exportPendingSequence = 0) {
if (Number(exportPendingSequence) > 0 && String(status ?? "").trim().toLowerCase() === "waiting") return null;
switch (String(status ?? "").trim().toLowerCase()) {
case "waiting":
return AttentionKind.Decision;
@@ -99,7 +100,7 @@ export function attentionEntries(runs, dismissed, { since = 0 } = {}) {
dismissed instanceof Set ? dismissed : new Set(Array.isArray(dismissed) ? dismissed : []);
return (runs ?? [])
.map((run) => {
const kind = attentionKindOf(run?.status);
const kind = attentionKindOf(run?.status, run?.export_pending_sequence);
if (!kind) return null;
const key = attentionKey(run.id, kind);
if (dismissedKeys.has(key)) return null;
+1 -1
View File
@@ -122,7 +122,7 @@ export function createAlertLedger() {
if (frame?.notify_mode != null) modes.set(runId, normalizeStatus(frame.notify_mode));
const mode = modes.get(runId) || "auto";
const kind = status === "succeeded" && mode === "on" ? RunAlertKind.Succeeded : attentionKindOf(status);
const kind = status === "succeeded" && mode === "on" ? RunAlertKind.Succeeded : attentionKindOf(status, frame?.export_pending_sequence);
if (!kind) {
forget(runId, AttentionKind.Decision);
forget(runId, AttentionKind.Failed);
+7 -1
View File
@@ -183,6 +183,7 @@ function runStateFields(current) {
executing: Boolean(current.executing),
cancel_requested: Boolean(current.cancel_requested),
cleanup_unconfirmed: Boolean(current.cleanup_unconfirmed),
export_pending_sequence: Number(current.export_pending_sequence) || 0,
};
}
@@ -206,6 +207,9 @@ function applyRunStateFields(current, frame) {
? (frame.resource_wait ?? null)
: before.resource_wait,
status,
...(before.export_pending_sequence || Number(frame.export_pending_sequence) > 0 || Object.hasOwn(current, "export_pending_sequence")
? { export_pending_sequence: blank(frame.status) ? before.export_pending_sequence : status === "waiting" ? Number(frame.export_pending_sequence) || 0 : 0 }
: {}),
outcome: blank(frame.outcome) ? before.outcome : frame.outcome,
current_stage: blank(frame.current_stage) ? before.current_stage : frame.current_stage,
completed_steps: Math.max(before.completed_steps, Number(frame.completed_steps) || 0),
@@ -230,7 +234,8 @@ function applyRunStateFields(current, frame) {
merged.executing !== before.executing ||
merged.cancel_requested !== before.cancel_requested ||
merged.cleanup_unconfirmed !== before.cleanup_unconfirmed;
return changed ? { ...current, ...merged, ...(graphEvidence ? { graph_evidence: graphEvidence } : {}) } : current;
const exportWaitChanged = (Number(merged.export_pending_sequence) || 0) !== before.export_pending_sequence;
return changed || exportWaitChanged ? { ...current, ...merged, ...(graphEvidence ? { graph_evidence: graphEvidence } : {}) } : current;
}
// Reuse the detail's monotonic visit merge, then compact by last stage visit and
@@ -280,6 +285,7 @@ export function newRunSummaryFromFrame(frame) {
title: blank(frame.title) ? null : frame.title,
version: Number(frame.version) || 0,
status: frame.status ?? "",
...(Number(frame.export_pending_sequence) > 0 ? { export_pending_sequence: Number(frame.export_pending_sequence) } : {}),
executing: executingFor(frame.status, true),
cancel_requested: frame.cancel_requested === true,
cleanup_unconfirmed: frame.cleanup_unconfirmed === true,
+1
View File
@@ -41,6 +41,7 @@ export function runStatusLabel(run) {
if (["queued", "running", "waiting"].includes(run?.status)) {
if (run.cleanup_unconfirmed) return "Cleanup unconfirmed — workspace reserved";
if (run.cancel_requested) return "Stopping — waiting for execution to stop";
if (run.status === "waiting" && Number(run.export_pending_sequence) > 0) return "Waiting for session export";
}
return run?.status || "unknown";
}
+43
View File
@@ -0,0 +1,43 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { foldRunEventDetail, foldRunEvent, isTerminalRunStatus } from "../src/runEvents.js";
import { canCancel, canRetry, pendingDecision } from "../src/actions.js";
import { runStatusLabel } from "../src/runs.js";
import { attentionEntries, attentionKindOf } from "../src/attention.js";
import { createAlertLedger } from "../src/notify.js";
test("export waiting remains nonterminal and folds forward without reopening its computed visit", () => {
let run = { id: 7, status: "running", executing: true, steps: [{ sequence: 1, stage: "one", status: "running" }], graph: { stages: [{ key: "one", type: "model", edges: { success: "two", error: "#failure" } }] } };
const waiting = { run_id: 7, status: "waiting", export_pending_sequence: 1, current_stage: "one", step: { sequence: 1, stage: "one", status: "completed", outcome: "success", target: "two" } };
run = foldRunEventDetail(run, waiting);
assert.equal(run.status, "waiting");
assert.equal(run.export_pending_sequence, 1);
assert.equal(run.executing, false);
assert.equal(isTerminalRunStatus(run.status), false);
assert.equal(pendingDecision(run), null);
assert.equal(canCancel(run), true);
assert.equal(canRetry(run), false);
assert.equal(runStatusLabel(run), "Waiting for session export");
assert.equal(attentionKindOf(run.status, run.export_pending_sequence), null);
assert.deepEqual(attentionEntries([run], new Set()), []);
const ledger = createAlertLedger();
ledger.seed([{ id: 7, status: "running" }]);
assert.equal(ledger.observe(waiting), null);
const snapshot = foldRunEvent([], { ...waiting, pipeline_id: 3, name: "gate" }).runs[0];
assert.equal(snapshot.export_pending_sequence, 1);
// Repair-only frames must not clear the pending sequence.
run = foldRunEventDetail(run, { run_id: 7, session_export_updated: true });
assert.equal(run.export_pending_sequence, 1);
run = foldRunEventDetail(run, { run_id: 7, status: "running" });
assert.equal(run.export_pending_sequence, 0);
assert.equal(run.steps[0].status, "completed");
run = foldRunEventDetail(run, { run_id: 7, status: "succeeded", finished_at: "2026-10-20T20:00:00Z" });
assert.equal(run.status, "succeeded");
assert.equal(run.export_pending_sequence, 0);
assert.equal(run.executing, false);
assert.equal(foldRunEventDetail(run, waiting), run);
});
test("model stages never become human decisions via waiting fallback", () => {
assert.equal(pendingDecision({ id: 8, status: "waiting", current_stage: "one", steps: [], graph: { stages: [{ key: "one", type: "model", edges: { success: "two" } }] } }), null);
});