runner: keep export handoffs nonterminal through recovery
This commit is contained in:
@@ -451,6 +451,7 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
||||
}
|
||||
if in.Status != "" {
|
||||
into.Status = in.Status
|
||||
into.ExportPendingSequence = in.ExportPendingSequence
|
||||
}
|
||||
if in.Outcome != "" {
|
||||
into.Outcome = in.Outcome
|
||||
@@ -509,22 +510,23 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
continue
|
||||
}
|
||||
frame := protocol.RunEventFrame{
|
||||
GraphEvidence: summary.GraphEvidence,
|
||||
NotifyMode: summary.NotifyMode,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
CancelRequested: summary.CancelRequested,
|
||||
CleanupUnconfirmed: summary.CleanupUnconfirmed,
|
||||
GraphEvidence: summary.GraphEvidence,
|
||||
NotifyMode: summary.NotifyMode,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
CancelRequested: summary.CancelRequested,
|
||||
CleanupUnconfirmed: summary.CleanupUnconfirmed,
|
||||
ExportPendingSequence: summary.ExportPendingSequence,
|
||||
}
|
||||
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
|
||||
// The connection write failed; the subscription is closed and the
|
||||
|
||||
@@ -335,6 +335,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
|
||||
}
|
||||
if detail != nil {
|
||||
_, detail.Executing = r.active[id]
|
||||
if detail.ExportPendingSequence > 0 {
|
||||
detail.Executing = false
|
||||
}
|
||||
r.overlayEffectiveModels(detail)
|
||||
}
|
||||
if readErr != nil {
|
||||
@@ -421,6 +424,9 @@ func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
||||
d, err := r.store.GetRun(r.ctx, id)
|
||||
if d != nil {
|
||||
_, d.Executing = r.active[id]
|
||||
if d.ExportPendingSequence > 0 {
|
||||
d.Executing = false
|
||||
}
|
||||
for _, s := range d.Steps {
|
||||
if s.AgentActive {
|
||||
d.Executing = true
|
||||
@@ -467,6 +473,9 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
||||
items, err := r.store.ListRuns(r.ctx)
|
||||
for i := range items {
|
||||
_, items[i].Executing = r.active[items[i].ID]
|
||||
if items[i].ExportPendingSequence > 0 {
|
||||
items[i].Executing = false
|
||||
}
|
||||
if items[i].Status == "failed" && !items[i].Executing {
|
||||
d, err := r.store.GetRun(r.ctx, items[i].ID)
|
||||
if err != nil {
|
||||
@@ -797,6 +806,10 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
r.mu.Unlock()
|
||||
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
|
||||
}
|
||||
if detail.ExportPendingSequence > 0 {
|
||||
r.mu.Unlock()
|
||||
return nil, fmt.Errorf("run %d is awaiting session export, not a human decision", detail.ID)
|
||||
}
|
||||
if detail.Status == protocol.RunStatusWaiting {
|
||||
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
|
||||
r.mu.Unlock()
|
||||
@@ -1978,15 +1991,19 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
finish("failed", "error", saveErr.Error())
|
||||
return
|
||||
}
|
||||
diagnostic := "Session export gate blocked; computed visit retained; export-only recovery pending"
|
||||
if sessionExport != nil && sessionExport.Diagnostic != "" {
|
||||
diagnostic += ": " + sessionExport.Diagnostic
|
||||
}
|
||||
if err != nil {
|
||||
diagnostic = err.Error() + "; " + diagnostic
|
||||
}
|
||||
// No finishStage, memory publication, recordTurn or next executor.
|
||||
r.finishRunState(id, name, workspace, "failed", "error", diagnostic, state.completed, sequence, key, stage.Values(step.Output))
|
||||
check, done := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
detail, readErr := r.store.GetRun(check, id)
|
||||
done()
|
||||
if readErr != nil {
|
||||
log.Printf("read parked export gate for run %d: %v", id, readErr)
|
||||
return // The checkpoint is durable; never fabricate a terminal.
|
||||
}
|
||||
if detail.CancelRequested && detail.ExportPendingSequence == 0 {
|
||||
finish("canceled", "", "Canceled by operator; computed visit retained")
|
||||
return
|
||||
}
|
||||
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, ExportPendingSequence: sequence, CurrentStage: key, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts)}}, true)
|
||||
return
|
||||
}
|
||||
// Generate the one-line verdict reason once, at completion. It is
|
||||
|
||||
@@ -92,6 +92,7 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
step := *saved
|
||||
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
|
||||
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -111,7 +111,7 @@ func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
|
||||
h.runner.wg.Wait()
|
||||
ctx := context.Background()
|
||||
d, _ := s.GetRun(ctx, id)
|
||||
if d.Status != "failed" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
|
||||
if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
|
||||
t.Fatalf("lost computation/advanced: %+v", d)
|
||||
}
|
||||
h.runner.exportSession = validGateExport
|
||||
@@ -214,7 +214,7 @@ func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
|
||||
}, false, gateTestOptions{missingOwner: true})
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if d.Status != "failed" || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
|
||||
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
|
||||
t.Fatalf("missing owner did not fail closed: %+v", d)
|
||||
}
|
||||
}
|
||||
@@ -281,7 +281,7 @@ func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if d.Status != "failed" || second.Load() != 0 || d.Output["markdown"] != "computed output" {
|
||||
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" {
|
||||
t.Fatalf("invalid export advanced: %+v", d)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -204,7 +204,7 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
||||
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "failed" {
|
||||
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
|
||||
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,7 +211,7 @@ func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "Session export gate blocked") || detail.Output["summary"] != "done" {
|
||||
if detail.Status != "waiting" || detail.ExportPendingSequence != 1 || detail.FinishedAt != "" || detail.Output["summary"] != "done" {
|
||||
t.Fatalf("export gate did not preserve and block computation: %+v", detail)
|
||||
}
|
||||
step := detail.Steps[0]
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestExportWaitEventCoalescingClearsOnlyOnLifecycleTransition(t *testing.T) {
|
||||
frame := protocol.RunEventFrame{RunID: 7, Status: "waiting", ExportPendingSequence: 2}
|
||||
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Artifacts: 1})
|
||||
if frame.ExportPendingSequence != 2 {
|
||||
t.Fatal("repair-only frame cleared export pending")
|
||||
}
|
||||
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Status: "running"})
|
||||
if frame.ExportPendingSequence != 0 || frame.Status != "running" {
|
||||
t.Fatalf("continuation did not clear export wait: %+v", frame)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportWaitPrivateRepairResumesWithoutTerminalRegression(t *testing.T) {
|
||||
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return stage.SessionExportResult{}, errors.New("offline")
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
ctx := context.Background()
|
||||
job, err := s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil || job == nil {
|
||||
t.Fatalf("missing recovery checkpoint: %v", err)
|
||||
}
|
||||
name := "session-repair-1-private-test.html"
|
||||
result, err := validGateExport(ctx, stage.SessionExportRequest{ArtifactsDir: h.runner.turnPaths(id, 1, "one").TurnDir, FileName: name})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evidence := &protocol.SessionExport{Status: "complete", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Source: "native_export", HTML: name, HTMLBytes: result.HTMLBytes, Messages: result.Messages}
|
||||
if err := s.RecordDeliverable(ctx, id, 1, protocol.StageArtifact{Name: name, SizeBytes: result.HTMLBytes, ContentType: "text/html"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Reconciliation recognizes a durable manual repair without another export.
|
||||
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
|
||||
t.Fatalf("repair triggered a redundant export: %+v %v", claimed, err)
|
||||
}
|
||||
if err := h.runner.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
if err := h.runner.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := h.runner.getRun(id)
|
||||
if err != nil || d.Status != "succeeded" || d.ExportPendingSequence != 0 || first.Load() != 1 || second.Load() != 1 {
|
||||
t.Fatalf("private repair replayed or lost continuation: %+v %v", d, err)
|
||||
}
|
||||
input, err := os.ReadFile(h.runner.turnPaths(id, 2, "two").InputPath)
|
||||
if err != nil || !strings.Contains(string(input), "computed output") || strings.Contains(string(input), name) {
|
||||
t.Fatalf("private export leaked or computed output lost: %q %v", input, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportWaitSubscriptionSnapshotCarriesPendingSequence(t *testing.T) {
|
||||
h, _, id, _, _ := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return stage.SessionExportResult{}, errors.New("offline")
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
hub := newRunEventHub()
|
||||
defer hub.unsubscribeClient("export-wait-test")
|
||||
var snapshot *protocol.RunEventFrame
|
||||
if !handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{id}}, hub, h.runner, "export-wait-test", func(message protocol.Message) error {
|
||||
if message.RunEvent != nil {
|
||||
snapshot = message.RunEvent
|
||||
}
|
||||
return nil
|
||||
}) {
|
||||
t.Fatal("subscribe not handled")
|
||||
}
|
||||
if snapshot == nil || snapshot.Status != "waiting" || snapshot.ExportPendingSequence != 1 || snapshot.FinishedAt != "" {
|
||||
t.Fatalf("snapshot lost nonterminal export wait: %+v", snapshot)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user