Files
pipelines/internal/pipeline/session_export_gates_test.go
alan d35739cb49 Release session export gates whose recovery is exhausted or overdue
An exhausted export recovery job left its run parked forever in the
export wait, reserving the workspace and refusing later scheduled runs
(runs 221/226: OpenCode sessions above the 8 MiB export bound are
always truncated, so every retry stays partial).

A gate is now releasable when its job is exhausted: the visit keeps its
partial/unavailable evidence and diagnostic (still repairable), the run
continues to its next stage or terminal, and the workspace is freed.
Gates older than 15 minutes with a pending, lease-expired or missing job
are exhausted by a bounded wait. The worker reconciles on its first
tick after startup and every 30 seconds, so existing wedged runs recover
after a restart. Truncation diagnostics now name the size bound.
2026-11-01 17:24:56 +00:00

157 lines
6.8 KiB
Go

package pipeline
import (
"context"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "failed", SessionID: "ses_original", Backend: "v2", Diagnostic: "offline"}}
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
t.Fatal("successor started while export was incomplete")
}
paused, err := s.GetRun(ctx, id)
if err != nil || paused.Status != "waiting" || paused.ExportPendingSequence != 1 || paused.FinishedAt != "" {
t.Fatalf("gate became terminal: %+v %v", paused, err)
}
if _, _, err := s.ResolveDecision(ctx, id, "one", "success", ""); err == nil {
t.Fatal("model export wait accepted human decision")
}
job, _ := s.SessionExportJob(ctx, id, 1)
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
if err != nil || claimed == nil {
t.Fatalf("missing export-only retry: %v", err)
}
if err := s.RecordSessionExportRepair(ctx, id, 1, &protocol.SessionExport{Status: "complete", SessionID: claimed.SessionID, Backend: claimed.Backend, Directory: claimed.Directory, Messages: 2, HTML: "session-repair-1-test.html", HTMLBytes: 100}); err != nil {
t.Fatal(err)
}
if err := s.FinishSessionExportJob(ctx, claimed, time.Now(), ""); err != nil {
t.Fatal(err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err == nil {
t.Fatal("duplicate continuation admitted")
}
// A crash after reserving the continuation but before starting its next
// executor must retain the checkpoint for startup's interrupted run.
for _, prior := range []string{"failed", "interrupted", "running", "queued"} {
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status=?,finished_at=CURRENT_TIMESTAMP WHERE id=?`, prior, id); err != nil {
t.Fatal(err)
}
if err := s.RecoverRuns(ctx); err != nil {
t.Fatal(err)
}
recovered, err := s.GetRun(ctx, id)
if err != nil || recovered.Status != "waiting" || recovered.ExportPendingSequence != 1 || recovered.FinishedAt != "" {
t.Fatalf("restart exposed a terminal export wait from %s: %+v %v", prior, recovered, err)
}
}
ready, err := s.ReadySessionExportGates(ctx)
if err != nil || ready[id] != 1 {
t.Fatalf("lost restart checkpoint: %v %v", ready, err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
t.Fatal(err)
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("successor did not consume checkpoint: %v %v", blocked, err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
t.Fatal("successor started twice")
}
}
// A recovery job wedged in running past its lease (or pending and never
// claimable) is bounded by the gate's overall wait; a live lease is not stolen.
func TestSessionExportGateWaitIsBoundedForStuckJob(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "partial", SessionID: "ses_original", Backend: "v2", Truncated: true}}
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
t.Fatal(err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
start := time.UnixMilli(job.NextAt)
if claimed, err := s.ClaimSessionExportJob(ctx, start, time.Hour); err != nil || claimed == nil {
t.Fatalf("claim: %+v %v", claimed, err)
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(5*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
t.Fatalf("young gate expired: %d %v", n, err)
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(20*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
t.Fatalf("live lease stolen: %d %v", n, err)
}
if ready, _ := s.ReadySessionExportGates(ctx); ready[id] != 0 {
t.Fatal("running job released its gate")
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(2*time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
t.Fatalf("stuck job not bounded: %d %v", n, err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "wait exceeded") {
t.Fatalf("job not exhausted: %+v", job)
}
ready, err := s.ReadySessionExportGates(ctx)
if err != nil || ready[id] != 1 {
t.Fatalf("exhausted gate not ready: %v %v", ready, err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("terminal finish kept exhausted gate: %v %v", blocked, err)
}
d, _ := s.GetRun(ctx, id)
if d.Status != "succeeded" || d.Steps[0].SessionExport == nil || d.Steps[0].SessionExport.Status != "partial" || len(d.Steps[0].SessionExportRepairs) != 0 {
t.Fatalf("partial evidence not retained as-is: %+v", d)
}
}
// A gate whose visit never got a recovery job (no exact owner) is given an
// exhausted job after the bounded wait, and the release records why.
func TestSessionExportGateWithoutJobIsReleasedAfterWait(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
if _, err := s.db.Exec(`DELETE FROM run_session_owners WHERE run_id=?`, id); err != nil {
t.Fatal(err)
}
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}}
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
t.Fatal(err)
}
if job, _ := s.SessionExportJob(ctx, id, 1); job != nil {
t.Fatalf("fixture unexpectedly has a job: %+v", job)
}
if n, err := s.ExpireSessionExportGates(ctx, time.Now().Add(time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
t.Fatalf("jobless gate not bounded: %d %v", n, err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
t.Fatalf("successor refused after release: %v", err)
}
d, _ := s.GetRun(ctx, id)
repairs := d.Steps[0].SessionExportRepairs
if len(repairs) != 1 || repairs[0].Status != "unavailable" || !strings.Contains(repairs[0].Diagnostic, "No automatic session export recovery") {
t.Fatalf("release did not record unavailable evidence: %+v", repairs)
}
}