d35739cb49
An exhausted export recovery job left its run parked forever in the export wait, reserving the workspace and refusing later scheduled runs (runs 221/226: OpenCode sessions above the 8 MiB export bound are always truncated, so every retry stays partial). A gate is now releasable when its job is exhausted: the visit keeps its partial/unavailable evidence and diagnostic (still repairable), the run continues to its next stage or terminal, and the workspace is freed. Gates older than 15 minutes with a pending, lease-expired or missing job are exhausted by a bounded wait. The worker reconciles on its first tick after startup and every 30 seconds, so existing wedged runs recover after a restart. Truncation diagnostics now name the size bound.
157 lines
6.8 KiB
Go
157 lines
6.8 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
|
|
s, id, _ := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "failed", SessionID: "ses_original", Backend: "v2", Diagnostic: "offline"}}
|
|
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
|
|
t.Fatal("successor started while export was incomplete")
|
|
}
|
|
paused, err := s.GetRun(ctx, id)
|
|
if err != nil || paused.Status != "waiting" || paused.ExportPendingSequence != 1 || paused.FinishedAt != "" {
|
|
t.Fatalf("gate became terminal: %+v %v", paused, err)
|
|
}
|
|
if _, _, err := s.ResolveDecision(ctx, id, "one", "success", ""); err == nil {
|
|
t.Fatal("model export wait accepted human decision")
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
|
|
if err != nil || claimed == nil {
|
|
t.Fatalf("missing export-only retry: %v", err)
|
|
}
|
|
if err := s.RecordSessionExportRepair(ctx, id, 1, &protocol.SessionExport{Status: "complete", SessionID: claimed.SessionID, Backend: claimed.Backend, Directory: claimed.Directory, Messages: 2, HTML: "session-repair-1-test.html", HTMLBytes: 100}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.FinishSessionExportJob(ctx, claimed, time.Now(), ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.ResumeSessionExportGate(ctx, id, 1); err == nil {
|
|
t.Fatal("duplicate continuation admitted")
|
|
}
|
|
// A crash after reserving the continuation but before starting its next
|
|
// executor must retain the checkpoint for startup's interrupted run.
|
|
for _, prior := range []string{"failed", "interrupted", "running", "queued"} {
|
|
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status=?,finished_at=CURRENT_TIMESTAMP WHERE id=?`, prior, id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.RecoverRuns(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recovered, err := s.GetRun(ctx, id)
|
|
if err != nil || recovered.Status != "waiting" || recovered.ExportPendingSequence != 1 || recovered.FinishedAt != "" {
|
|
t.Fatalf("restart exposed a terminal export wait from %s: %+v %v", prior, recovered, err)
|
|
}
|
|
}
|
|
ready, err := s.ReadySessionExportGates(ctx)
|
|
if err != nil || ready[id] != 1 {
|
|
t.Fatalf("lost restart checkpoint: %v %v", ready, err)
|
|
}
|
|
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
|
|
t.Fatalf("successor did not consume checkpoint: %v %v", blocked, err)
|
|
}
|
|
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
|
|
t.Fatal("successor started twice")
|
|
}
|
|
}
|
|
|
|
// A recovery job wedged in running past its lease (or pending and never
|
|
// claimable) is bounded by the gate's overall wait; a live lease is not stolen.
|
|
func TestSessionExportGateWaitIsBoundedForStuckJob(t *testing.T) {
|
|
s, id, _ := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "partial", SessionID: "ses_original", Backend: "v2", Truncated: true}}
|
|
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
start := time.UnixMilli(job.NextAt)
|
|
if claimed, err := s.ClaimSessionExportJob(ctx, start, time.Hour); err != nil || claimed == nil {
|
|
t.Fatalf("claim: %+v %v", claimed, err)
|
|
}
|
|
if n, err := s.ExpireSessionExportGates(ctx, start.Add(5*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
|
|
t.Fatalf("young gate expired: %d %v", n, err)
|
|
}
|
|
if n, err := s.ExpireSessionExportGates(ctx, start.Add(20*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
|
|
t.Fatalf("live lease stolen: %d %v", n, err)
|
|
}
|
|
if ready, _ := s.ReadySessionExportGates(ctx); ready[id] != 0 {
|
|
t.Fatal("running job released its gate")
|
|
}
|
|
if n, err := s.ExpireSessionExportGates(ctx, start.Add(2*time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
|
|
t.Fatalf("stuck job not bounded: %d %v", n, err)
|
|
}
|
|
job, _ = s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "wait exceeded") {
|
|
t.Fatalf("job not exhausted: %+v", job)
|
|
}
|
|
ready, err := s.ReadySessionExportGates(ctx)
|
|
if err != nil || ready[id] != 1 {
|
|
t.Fatalf("exhausted gate not ready: %v %v", ready, err)
|
|
}
|
|
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
|
|
t.Fatalf("terminal finish kept exhausted gate: %v %v", blocked, err)
|
|
}
|
|
d, _ := s.GetRun(ctx, id)
|
|
if d.Status != "succeeded" || d.Steps[0].SessionExport == nil || d.Steps[0].SessionExport.Status != "partial" || len(d.Steps[0].SessionExportRepairs) != 0 {
|
|
t.Fatalf("partial evidence not retained as-is: %+v", d)
|
|
}
|
|
}
|
|
|
|
// A gate whose visit never got a recovery job (no exact owner) is given an
|
|
// exhausted job after the bounded wait, and the release records why.
|
|
func TestSessionExportGateWithoutJobIsReleasedAfterWait(t *testing.T) {
|
|
s, id, _ := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
if _, err := s.db.Exec(`DELETE FROM run_session_owners WHERE run_id=?`, id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}}
|
|
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if job, _ := s.SessionExportJob(ctx, id, 1); job != nil {
|
|
t.Fatalf("fixture unexpectedly has a job: %+v", job)
|
|
}
|
|
if n, err := s.ExpireSessionExportGates(ctx, time.Now().Add(time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
|
|
t.Fatalf("jobless gate not bounded: %d %v", n, err)
|
|
}
|
|
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
|
|
t.Fatalf("successor refused after release: %v", err)
|
|
}
|
|
d, _ := s.GetRun(ctx, id)
|
|
repairs := d.Steps[0].SessionExportRepairs
|
|
if len(repairs) != 1 || repairs[0].Status != "unavailable" || !strings.Contains(repairs[0].Diagnostic, "No automatic session export recovery") {
|
|
t.Fatalf("release did not record unavailable evidence: %+v", repairs)
|
|
}
|
|
}
|