runner: keep export handoffs nonterminal through recovery

This commit is contained in:
2026-10-20 21:19:26 +01:00
parent ca78b24f47
commit ec3eda9e05
28 changed files with 370 additions and 82 deletions
+5 -5
View File
@@ -1461,16 +1461,16 @@ func (a App) wait(ctx context.Context, pos []string, o *options, call callFunc)
return nil
}
// awaitRuns polls every run that active reports as unfinished until each is
// terminal, using the same GetRun reattachment the run --wait path uses, and
// calls onTerminal for each as it finishes (including runs already terminal at
// awaitRuns polls unfinished runs using GetRun reattachment. Export waits remain
// unfinished even for run --wait, whose status-only predicate intentionally stops
// at human decisions. It calls onTerminal for each as it finishes (including runs already terminal at
// entry, in the order supplied). A deadline that passes first returns a
// *waitLimitError naming the runs still active; ctx ending first returns
// ctx.Err(). It never cancels, retries or otherwise changes a run.
func (a App) awaitRuns(ctx context.Context, runs []protocol.Message, deadline <-chan time.Time, call callFunc, active func(string) bool, onTerminal func(protocol.Message) error) error {
remaining := make([]protocol.Message, 0, len(runs))
for _, r := range runs {
if active(r.RunDetail.Status) {
if active(r.RunDetail.Status) || (r.RunDetail.Status == protocol.RunStatusWaiting && r.RunDetail.ExportPendingSequence > 0) {
remaining = append(remaining, r)
continue
}
@@ -1495,7 +1495,7 @@ func (a App) awaitRuns(ctx context.Context, runs []protocol.Message, deadline <-
if refreshed.RunDetail == nil {
return fmt.Errorf("server did not return run detail for %d", r.RunDetail.ID)
}
if active(refreshed.RunDetail.Status) {
if active(refreshed.RunDetail.Status) || (refreshed.RunDetail.Status == protocol.RunStatusWaiting && refreshed.RunDetail.ExportPendingSequence > 0) {
next = append(next, refreshed)
continue
}
+40
View File
@@ -0,0 +1,40 @@
package cli
import (
"context"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestRunAndExplicitWaitDoNotReturnDuringExportRecovery(t *testing.T) {
for _, mode := range []string{"run --wait", "wait"} {
t.Run(mode, func(t *testing.T) {
active := runInProgress
if mode == "wait" {
active = runUnfinished
}
initial := protocol.Message{RunDetail: &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: protocol.RunStatusWaiting, ExportPendingSequence: 2}}}
polls, finished := 0, false
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err := (App{}).awaitRuns(ctx, []protocol.Message{initial}, nil, func(request protocol.Message) (protocol.Message, error) {
polls++
if request.Type != protocol.GetRun || request.RunID != 7 {
t.Errorf("unexpected request: %+v", request)
}
return protocol.Message{RunDetail: &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: "succeeded"}}}, nil
}, active, func(message protocol.Message) error {
if message.RunDetail.Status != "succeeded" {
t.Errorf("wait returned before actual terminal: %+v", message.RunDetail)
}
finished = true
return nil
})
if err != nil || polls != 1 || !finished {
t.Fatalf("export wait ended early: polls=%d finished=%v error=%v", polls, finished, err)
}
})
}
}
+3 -2
View File
@@ -22,12 +22,13 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
` + runStamps + `,r.diagnostic,
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format,
CASE WHEN r.status='waiting' THEN COALESCE((SELECT sequence FROM session_export_gates WHERE run_id=r.id),0) ELSE 0 END
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
var output, stageModels string
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat); err != nil {
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence); err != nil {
return err
}
if stageModels != "" && stageModels != "{}" {
+14 -1
View File
@@ -168,6 +168,9 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome,
if _, err := tx.ExecContext(ctx, `INSERT INTO session_export_gates(run_id,sequence) VALUES(?,?)`, id, step.Sequence); err != nil {
return err
}
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',diagnostic='Session export pending; computed visit retained',output_json=?,finished_at=NULL WHERE id=? AND status='running'`, string(output), id)); err != nil {
return err
}
}
// The reported usage the stage already measured for budget accounting is
// persisted as step-scoped evidence in the same transaction, mirroring the
@@ -226,6 +229,13 @@ func (s *Store) ResolveDecision(ctx context.Context, id int64, stageKey, option,
return protocol.RunStep{}, false, err
}
defer tx.Rollback()
var exportPending bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&exportPending); err != nil {
return protocol.RunStep{}, false, err
}
if exportPending {
return protocol.RunStep{}, false, fmt.Errorf("run %d is awaiting session export, not a human decision", id)
}
var status, currentStage, graphJSON string
if err := tx.QueryRowContext(ctx, `SELECT r.status,r.current_stage,v.graph_json FROM pipeline_runs r
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&status, &currentStage, &graphJSON); err != nil {
@@ -253,6 +263,9 @@ JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version
found := false
for _, declared := range graph.Stages {
if declared.Key == stageKey {
if stage.EffectiveType(declared.Type) != stage.HumanType {
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not a human decision stage", stageKey)
}
edges = declared.Edges
found = true
break
@@ -324,7 +337,7 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
}
defer tx.Rollback()
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,
diagnostic=CASE WHEN status='waiting' THEN 'Canceled while waiting for a decision' ELSE 'Canceled while awaiting session export; computed visit retained' END,
diagnostic=CASE WHEN EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id) THEN 'Canceled while awaiting session export; computed visit retained' ELSE 'Canceled while waiting for a decision' END,
finished_at=CURRENT_TIMESTAMP WHERE id=? AND (status=? OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id, protocol.RunStatusWaiting)
if err != nil {
return err
+9
View File
@@ -204,5 +204,14 @@ func (s *Store) RecoverRuns(ctx context.Context) error {
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
return err
}
// An export checkpoint is a paused handoff, not an interrupted model call.
// Also migrate the earlier failed-gate representation before clients attach.
if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',finished_at=NULL,diagnostic='Session export pending; computed visit retained'
WHERE status IN ('queued','running','failed','interrupted','waiting') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
WHERE g.run_id=pipeline_runs.id AND s.status IN ('completed','failed') AND s.agent_active=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.sequence>g.sequence OR a.agent_active<>0)))`); err != nil {
return err
}
return tx.Commit()
}
+3 -3
View File
@@ -47,7 +47,7 @@ func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, err
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
JOIN pipeline_runs r ON r.id=g.run_id
WHERE j.status='complete' AND r.status IN ('failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
WHERE j.status='complete' AND r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`)
if err != nil {
return nil, err
@@ -67,8 +67,8 @@ AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_ac
// ResumeSessionExportGate is a CAS, not a retry of the model invocation.
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='queued',outcome='',diagnostic='',finished_at=NULL
WHERE id=? AND status IN ('failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`)
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence))
}
+17 -4
View File
@@ -18,8 +18,12 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
t.Fatal("successor started while export was incomplete")
}
if err := s.FinishRun(ctx, id, "failed", "error", "export gate", step.Output); err != nil {
t.Fatal(err)
paused, err := s.GetRun(ctx, id)
if err != nil || paused.Status != "waiting" || paused.ExportPendingSequence != 1 || paused.FinishedAt != "" {
t.Fatalf("gate became terminal: %+v %v", paused, err)
}
if _, _, err := s.ResolveDecision(ctx, id, "one", "success", ""); err == nil {
t.Fatal("model export wait accepted human decision")
}
job, _ := s.SessionExportJob(ctx, id, 1)
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
@@ -40,8 +44,17 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
}
// A crash after reserving the continuation but before starting its next
// executor must retain the checkpoint for startup's interrupted run.
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status='interrupted' WHERE id=?`, id); err != nil {
t.Fatal(err)
for _, prior := range []string{"failed", "interrupted", "running", "queued"} {
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status=?,finished_at=CURRENT_TIMESTAMP WHERE id=?`, prior, id); err != nil {
t.Fatal(err)
}
if err := s.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
recovered, err := s.GetRun(ctx, id)
if err != nil || recovered.Status != "waiting" || recovered.ExportPendingSequence != 1 || recovered.FinishedAt != "" {
t.Fatalf("restart exposed a terminal export wait from %s: %+v %v", prior, recovered, err)
}
}
ready, err := s.ReadySessionExportGates(ctx)
if err != nil || ready[id] != 1 {
+1 -1
View File
@@ -139,7 +139,7 @@ func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease
err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic
FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence
WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts<?
AND r.status IN ('succeeded','failed','canceled','interrupted') AND s.status NOT IN ('running','waiting')`+guard+`
AND (r.status IN ('succeeded','failed','canceled','interrupted') OR (r.status='waiting' AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=j.run_id AND g.sequence=j.sequence))) AND s.status NOT IN ('running','waiting')`+guard+`
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=j.run_id OR a.session_id=j.session_id) AND a.agent_active<>0)
ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
if err == sql.ErrNoRows {
+14 -12
View File
@@ -37,18 +37,20 @@ type RunEventFrame struct {
Workspace string `json:"workspace,omitempty"`
// PipelineID, Name, Title and Version identify the run's pipeline and its
// operator-facing title, so a new run's row can be built without a fetch.
PipelineID int64 `json:"pipeline_id,omitempty"`
Name string `json:"name,omitempty"`
Title string `json:"title,omitempty"`
Version int `json:"version,omitempty"`
Status string `json:"status,omitempty"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Outcome string `json:"outcome,omitempty"`
CurrentStage string `json:"current_stage,omitempty"`
CompletedSteps int `json:"completed_steps,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
FinishedAt string `json:"finished_at,omitempty"`
PipelineID int64 `json:"pipeline_id,omitempty"`
Name string `json:"name,omitempty"`
Title string `json:"title,omitempty"`
Version int `json:"version,omitempty"`
Status string `json:"status,omitempty"`
// On a lifecycle frame, zero/absence clears a prior export wait.
ExportPendingSequence int `json:"export_pending_sequence,omitempty"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Outcome string `json:"outcome,omitempty"`
CurrentStage string `json:"current_stage,omitempty"`
CompletedSteps int `json:"completed_steps,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
FinishedAt string `json:"finished_at,omitempty"`
// Artifacts is the number of files recorded for the referenced step, so a
// client can surface artifact capture without reading the step.
Artifacts int `json:"artifacts,omitempty"`
+18 -15
View File
@@ -142,21 +142,24 @@ type RunSummary struct {
// Title is the operator-facing, bounded single-line description of what the
// run is for, generated from its prompt. It is never empty for a new run;
// historical rows predating the title column may omit it.
Title string `json:"title,omitempty"`
Version int `json:"version"`
ContentHash string `json:"content_hash"`
Status string `json:"status"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Executing bool `json:"executing"`
Outcome string `json:"outcome"`
CurrentStage string `json:"current_stage"`
Workspace string `json:"workspace,omitempty"`
CreatedAt string `json:"created_at"`
FinishedAt string `json:"finished_at"`
Diagnostic string `json:"diagnostic"`
ResultPreview string `json:"result_preview,omitempty"`
CompletedSteps int `json:"completed_steps"`
Title string `json:"title,omitempty"`
Version int `json:"version"`
ContentHash string `json:"content_hash"`
Status string `json:"status"`
// ExportPendingSequence distinguishes a quiescent export handoff from a
// human decision while Status is waiting. It never implies agent execution.
ExportPendingSequence int `json:"export_pending_sequence,omitempty"`
CancelRequested bool `json:"cancel_requested,omitempty"`
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
Executing bool `json:"executing"`
Outcome string `json:"outcome"`
CurrentStage string `json:"current_stage"`
Workspace string `json:"workspace,omitempty"`
CreatedAt string `json:"created_at"`
FinishedAt string `json:"finished_at"`
Diagnostic string `json:"diagnostic"`
ResultPreview string `json:"result_preview,omitempty"`
CompletedSteps int `json:"completed_steps"`
// Model and StageModels are the launch-time overrides this run records so it
// stays reproducible: Model covers every model stage and StageModels overrides
// individual stages by key. They are empty for a run that used the pinned
+18 -16
View File
@@ -451,6 +451,7 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
}
if in.Status != "" {
into.Status = in.Status
into.ExportPendingSequence = in.ExportPendingSequence
}
if in.Outcome != "" {
into.Outcome = in.Outcome
@@ -509,22 +510,23 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
GraphEvidence: summary.GraphEvidence,
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
GraphEvidence: summary.GraphEvidence,
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
ExportPendingSequence: summary.ExportPendingSequence,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
+25 -8
View File
@@ -335,6 +335,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
}
if detail != nil {
_, detail.Executing = r.active[id]
if detail.ExportPendingSequence > 0 {
detail.Executing = false
}
r.overlayEffectiveModels(detail)
}
if readErr != nil {
@@ -421,6 +424,9 @@ func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
d, err := r.store.GetRun(r.ctx, id)
if d != nil {
_, d.Executing = r.active[id]
if d.ExportPendingSequence > 0 {
d.Executing = false
}
for _, s := range d.Steps {
if s.AgentActive {
d.Executing = true
@@ -467,6 +473,9 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
items, err := r.store.ListRuns(r.ctx)
for i := range items {
_, items[i].Executing = r.active[items[i].ID]
if items[i].ExportPendingSequence > 0 {
items[i].Executing = false
}
if items[i].Status == "failed" && !items[i].Executing {
d, err := r.store.GetRun(r.ctx, items[i].ID)
if err != nil {
@@ -797,6 +806,10 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
}
if detail.ExportPendingSequence > 0 {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is awaiting session export, not a human decision", detail.ID)
}
if detail.Status == protocol.RunStatusWaiting {
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
r.mu.Unlock()
@@ -1978,15 +1991,19 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
finish("failed", "error", saveErr.Error())
return
}
diagnostic := "Session export gate blocked; computed visit retained; export-only recovery pending"
if sessionExport != nil && sessionExport.Diagnostic != "" {
diagnostic += ": " + sessionExport.Diagnostic
}
if err != nil {
diagnostic = err.Error() + "; " + diagnostic
}
// No finishStage, memory publication, recordTurn or next executor.
r.finishRunState(id, name, workspace, "failed", "error", diagnostic, state.completed, sequence, key, stage.Values(step.Output))
check, done := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
detail, readErr := r.store.GetRun(check, id)
done()
if readErr != nil {
log.Printf("read parked export gate for run %d: %v", id, readErr)
return // The checkpoint is durable; never fabricate a terminal.
}
if detail.CancelRequested && detail.ExportPendingSequence == 0 {
finish("canceled", "", "Canceled by operator; computed visit retained")
return
}
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, ExportPendingSequence: sequence, CurrentStage: key, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts)}}, true)
return
}
// Generate the one-line verdict reason once, at completion. It is
+1
View File
@@ -92,6 +92,7 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
continue
}
step := *saved
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
r.mu.Unlock()
}
+3 -3
View File
@@ -111,7 +111,7 @@ func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "failed" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
t.Fatalf("lost computation/advanced: %+v", d)
}
h.runner.exportSession = validGateExport
@@ -214,7 +214,7 @@ func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
}, false, gateTestOptions{missingOwner: true})
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
t.Fatalf("missing owner did not fail closed: %+v", d)
}
}
@@ -281,7 +281,7 @@ func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
}, false)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || d.Output["markdown"] != "computed output" {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" {
t.Fatalf("invalid export advanced: %+v", d)
}
})
+1 -1
View File
@@ -204,7 +204,7 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
t.Fatal(err)
}
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "failed" {
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
}
}
+1 -1
View File
@@ -211,7 +211,7 @@ func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T)
if err != nil {
t.Fatal(err)
}
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "Session export gate blocked") || detail.Output["summary"] != "done" {
if detail.Status != "waiting" || detail.ExportPendingSequence != 1 || detail.FinishedAt != "" || detail.Output["summary"] != "done" {
t.Fatalf("export gate did not preserve and block computation: %+v", detail)
}
step := detail.Steps[0]
@@ -0,0 +1,89 @@
package server
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestExportWaitEventCoalescingClearsOnlyOnLifecycleTransition(t *testing.T) {
frame := protocol.RunEventFrame{RunID: 7, Status: "waiting", ExportPendingSequence: 2}
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Artifacts: 1})
if frame.ExportPendingSequence != 2 {
t.Fatal("repair-only frame cleared export pending")
}
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Status: "running"})
if frame.ExportPendingSequence != 0 || frame.Status != "running" {
t.Fatalf("continuation did not clear export wait: %+v", frame)
}
}
func TestExportWaitPrivateRepairResumesWithoutTerminalRegression(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil {
t.Fatalf("missing recovery checkpoint: %v", err)
}
name := "session-repair-1-private-test.html"
result, err := validGateExport(ctx, stage.SessionExportRequest{ArtifactsDir: h.runner.turnPaths(id, 1, "one").TurnDir, FileName: name})
if err != nil {
t.Fatal(err)
}
evidence := &protocol.SessionExport{Status: "complete", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Source: "native_export", HTML: name, HTMLBytes: result.HTMLBytes, Messages: result.Messages}
if err := s.RecordDeliverable(ctx, id, 1, protocol.StageArtifact{Name: name, SizeBytes: result.HTMLBytes, ContentType: "text/html"}); err != nil {
t.Fatal(err)
}
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
t.Fatal(err)
}
// Reconciliation recognizes a durable manual repair without another export.
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
t.Fatalf("repair triggered a redundant export: %+v %v", claimed, err)
}
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
d, err := h.runner.getRun(id)
if err != nil || d.Status != "succeeded" || d.ExportPendingSequence != 0 || first.Load() != 1 || second.Load() != 1 {
t.Fatalf("private repair replayed or lost continuation: %+v %v", d, err)
}
input, err := os.ReadFile(h.runner.turnPaths(id, 2, "two").InputPath)
if err != nil || !strings.Contains(string(input), "computed output") || strings.Contains(string(input), name) {
t.Fatalf("private export leaked or computed output lost: %q %v", input, err)
}
}
func TestExportWaitSubscriptionSnapshotCarriesPendingSequence(t *testing.T) {
h, _, id, _, _ := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
hub := newRunEventHub()
defer hub.unsubscribeClient("export-wait-test")
var snapshot *protocol.RunEventFrame
if !handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{id}}, hub, h.runner, "export-wait-test", func(message protocol.Message) error {
if message.RunEvent != nil {
snapshot = message.RunEvent
}
return nil
}) {
t.Fatal("subscribe not handled")
}
if snapshot == nil || snapshot.Status != "waiting" || snapshot.ExportPendingSequence != 1 || snapshot.FinishedAt != "" {
t.Fatalf("snapshot lost nonterminal export wait: %+v", snapshot)
}
}
+4 -1
View File
@@ -27,6 +27,9 @@ func lastWaitingStep(r *protocol.RunDetail) (protocol.RunStep, bool) {
// waiting step and falls back to the pinned graph's edges when a step recorded
// no options.
func (m Model) waitingStage() (string, []string, string) {
if m.runDetail != nil && m.runDetail.ExportPendingSequence > 0 {
return m.runDetail.CurrentStage, nil, ""
}
if m.runDetail == nil {
return "", nil, ""
}
@@ -43,7 +46,7 @@ func (m Model) waitingStage() (string, []string, string) {
}
func (m Model) openDecision() (tea.Model, tea.Cmd) {
if m.runDetail == nil || m.runDetail.Status != protocol.RunStatusWaiting {
if m.runDetail == nil || m.runDetail.Status != protocol.RunStatusWaiting || m.runDetail.ExportPendingSequence > 0 {
m.runError = "This run is not waiting for a decision."
return m, nil
}
+25
View File
@@ -0,0 +1,25 @@
package tui
import (
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestExportWaitRendersRecoveryNotHumanDecision(t *testing.T) {
run := &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: "waiting", ExportPendingSequence: 2, CurrentStage: "worker"}, Graph: protocol.Graph{Stages: []protocol.Stage{{Key: "worker", Type: "model", Edges: map[string]string{"success": "next", "error": "#failure"}}}}}
frame := strings.Join(waitingLines(run), "\n")
if !strings.Contains(frame, "Waiting for session export") || strings.Contains(frame, "person") || strings.Contains(frame, "Decide:") {
t.Fatalf("export wait offered a human choice:\n%s", frame)
}
m := Model{runDetail: run}
_, options, _ := m.waitingStage()
if len(options) != 0 {
t.Fatal("model-stage edges exposed as human options")
}
updated, _ := m.openDecision()
if updated.(Model).decisionOpen {
t.Fatal("opened decision for export wait")
}
}
+6
View File
@@ -18,6 +18,9 @@ func resultLabel(r protocol.RunSummary) string {
}
return "Finished · no result recorded"
}
if r.Status == protocol.RunStatusWaiting && r.ExportPendingSequence > 0 {
return "Waiting for session export · stage " + r.CurrentStage
}
if r.Diagnostic != "" {
return "Stopped: " + ansi.Strip(r.Diagnostic)
}
@@ -111,6 +114,9 @@ func waitingLines(r *protocol.RunDetail) []string {
if r.Status != protocol.RunStatusWaiting {
return nil
}
if r.ExportPendingSequence > 0 {
return []string{fmt.Sprintf("⏸ Waiting for session export · stage %s · visit %d", r.CurrentStage, r.ExportPendingSequence), "Computed output retained; automatic export-only recovery pending."}
}
lines := []string{"⏸ Waiting for a person · stage " + r.CurrentStage}
if step, ok := lastWaitingStep(r); ok {
if prompt := strings.TrimSpace(protocol.WaitingPrompt(step.Output)); prompt != "" {
+2 -2
View File
@@ -657,7 +657,7 @@ func (m Model) runActions() string {
if m.runDetail != nil {
text += " · a evidence · A artifacts"
}
if m.runDetail != nil && m.runDetail.Status == protocol.RunStatusWaiting {
if m.runDetail != nil && m.runDetail.Status == protocol.RunStatusWaiting && m.runDetail.ExportPendingSequence == 0 {
text += " · c decide"
}
if m.selectedFailedRun() > 0 {
@@ -740,7 +740,7 @@ func (m Model) runsDetailsView(height int) string {
page := max(0, innerHeight-5)
top := max(0, min(m.runTop, max(0, len(lines)-page)))
footerHint := "o result · d diff · s progress · i input · t evidence · m details · A artifacts"
if r.Status == protocol.RunStatusWaiting {
if r.Status == protocol.RunStatusWaiting && r.ExportPendingSequence == 0 {
footerHint += " · c decide"
}
footer := bar(footerHint, fmt.Sprintf("%d–%d / %d", min(top+1, len(lines)), min(top+page, len(lines)), len(lines)), max(1, m.width-4), false)