runner: keep export handoffs nonterminal through recovery

This commit is contained in:
2026-10-20 21:19:26 +01:00
parent ca78b24f47
commit ec3eda9e05
28 changed files with 370 additions and 82 deletions
+18 -16
View File
@@ -451,6 +451,7 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
}
if in.Status != "" {
into.Status = in.Status
into.ExportPendingSequence = in.ExportPendingSequence
}
if in.Outcome != "" {
into.Outcome = in.Outcome
@@ -509,22 +510,23 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
continue
}
frame := protocol.RunEventFrame{
GraphEvidence: summary.GraphEvidence,
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
GraphEvidence: summary.GraphEvidence,
NotifyMode: summary.NotifyMode,
RunID: summary.ID,
Workspace: summary.Workspace,
PipelineID: summary.PipelineID,
Name: summary.Name,
Title: summary.Title,
Version: summary.Version,
Status: summary.Status,
Outcome: summary.Outcome,
CurrentStage: summary.CurrentStage,
CompletedSteps: summary.CompletedSteps,
CreatedAt: summary.CreatedAt,
FinishedAt: summary.FinishedAt,
CancelRequested: summary.CancelRequested,
CleanupUnconfirmed: summary.CleanupUnconfirmed,
ExportPendingSequence: summary.ExportPendingSequence,
}
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
// The connection write failed; the subscription is closed and the
+25 -8
View File
@@ -335,6 +335,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
}
if detail != nil {
_, detail.Executing = r.active[id]
if detail.ExportPendingSequence > 0 {
detail.Executing = false
}
r.overlayEffectiveModels(detail)
}
if readErr != nil {
@@ -421,6 +424,9 @@ func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
d, err := r.store.GetRun(r.ctx, id)
if d != nil {
_, d.Executing = r.active[id]
if d.ExportPendingSequence > 0 {
d.Executing = false
}
for _, s := range d.Steps {
if s.AgentActive {
d.Executing = true
@@ -467,6 +473,9 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
items, err := r.store.ListRuns(r.ctx)
for i := range items {
_, items[i].Executing = r.active[items[i].ID]
if items[i].ExportPendingSequence > 0 {
items[i].Executing = false
}
if items[i].Status == "failed" && !items[i].Executing {
d, err := r.store.GetRun(r.ctx, items[i].ID)
if err != nil {
@@ -797,6 +806,10 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
}
if detail.ExportPendingSequence > 0 {
r.mu.Unlock()
return nil, fmt.Errorf("run %d is awaiting session export, not a human decision", detail.ID)
}
if detail.Status == protocol.RunStatusWaiting {
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
r.mu.Unlock()
@@ -1978,15 +1991,19 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
finish("failed", "error", saveErr.Error())
return
}
diagnostic := "Session export gate blocked; computed visit retained; export-only recovery pending"
if sessionExport != nil && sessionExport.Diagnostic != "" {
diagnostic += ": " + sessionExport.Diagnostic
}
if err != nil {
diagnostic = err.Error() + "; " + diagnostic
}
// No finishStage, memory publication, recordTurn or next executor.
r.finishRunState(id, name, workspace, "failed", "error", diagnostic, state.completed, sequence, key, stage.Values(step.Output))
check, done := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
detail, readErr := r.store.GetRun(check, id)
done()
if readErr != nil {
log.Printf("read parked export gate for run %d: %v", id, readErr)
return // The checkpoint is durable; never fabricate a terminal.
}
if detail.CancelRequested && detail.ExportPendingSequence == 0 {
finish("canceled", "", "Canceled by operator; computed visit retained")
return
}
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, ExportPendingSequence: sequence, CurrentStage: key, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts)}}, true)
return
}
// Generate the one-line verdict reason once, at completion. It is
+1
View File
@@ -92,6 +92,7 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
continue
}
step := *saved
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
r.mu.Unlock()
}
+3 -3
View File
@@ -111,7 +111,7 @@ func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "failed" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
t.Fatalf("lost computation/advanced: %+v", d)
}
h.runner.exportSession = validGateExport
@@ -214,7 +214,7 @@ func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
}, false, gateTestOptions{missingOwner: true})
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
t.Fatalf("missing owner did not fail closed: %+v", d)
}
}
@@ -281,7 +281,7 @@ func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
}, false)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || d.Output["markdown"] != "computed output" {
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" {
t.Fatalf("invalid export advanced: %+v", d)
}
})
+1 -1
View File
@@ -204,7 +204,7 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
t.Fatal(err)
}
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "failed" {
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
}
}
+1 -1
View File
@@ -211,7 +211,7 @@ func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T)
if err != nil {
t.Fatal(err)
}
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "Session export gate blocked") || detail.Output["summary"] != "done" {
if detail.Status != "waiting" || detail.ExportPendingSequence != 1 || detail.FinishedAt != "" || detail.Output["summary"] != "done" {
t.Fatalf("export gate did not preserve and block computation: %+v", detail)
}
step := detail.Steps[0]
@@ -0,0 +1,89 @@
package server
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestExportWaitEventCoalescingClearsOnlyOnLifecycleTransition(t *testing.T) {
frame := protocol.RunEventFrame{RunID: 7, Status: "waiting", ExportPendingSequence: 2}
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Artifacts: 1})
if frame.ExportPendingSequence != 2 {
t.Fatal("repair-only frame cleared export pending")
}
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Status: "running"})
if frame.ExportPendingSequence != 0 || frame.Status != "running" {
t.Fatalf("continuation did not clear export wait: %+v", frame)
}
}
func TestExportWaitPrivateRepairResumesWithoutTerminalRegression(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil {
t.Fatalf("missing recovery checkpoint: %v", err)
}
name := "session-repair-1-private-test.html"
result, err := validGateExport(ctx, stage.SessionExportRequest{ArtifactsDir: h.runner.turnPaths(id, 1, "one").TurnDir, FileName: name})
if err != nil {
t.Fatal(err)
}
evidence := &protocol.SessionExport{Status: "complete", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Source: "native_export", HTML: name, HTMLBytes: result.HTMLBytes, Messages: result.Messages}
if err := s.RecordDeliverable(ctx, id, 1, protocol.StageArtifact{Name: name, SizeBytes: result.HTMLBytes, ContentType: "text/html"}); err != nil {
t.Fatal(err)
}
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
t.Fatal(err)
}
// Reconciliation recognizes a durable manual repair without another export.
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
t.Fatalf("repair triggered a redundant export: %+v %v", claimed, err)
}
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
d, err := h.runner.getRun(id)
if err != nil || d.Status != "succeeded" || d.ExportPendingSequence != 0 || first.Load() != 1 || second.Load() != 1 {
t.Fatalf("private repair replayed or lost continuation: %+v %v", d, err)
}
input, err := os.ReadFile(h.runner.turnPaths(id, 2, "two").InputPath)
if err != nil || !strings.Contains(string(input), "computed output") || strings.Contains(string(input), name) {
t.Fatalf("private export leaked or computed output lost: %q %v", input, err)
}
}
func TestExportWaitSubscriptionSnapshotCarriesPendingSequence(t *testing.T) {
h, _, id, _, _ := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
hub := newRunEventHub()
defer hub.unsubscribeClient("export-wait-test")
var snapshot *protocol.RunEventFrame
if !handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{id}}, hub, h.runner, "export-wait-test", func(message protocol.Message) error {
if message.RunEvent != nil {
snapshot = message.RunEvent
}
return nil
}) {
t.Fatal("subscribe not handled")
}
if snapshot == nil || snapshot.Status != "waiting" || snapshot.ExportPendingSequence != 1 || snapshot.FinishedAt != "" {
t.Fatalf("snapshot lost nonterminal export wait: %+v", snapshot)
}
}