package pipeline import ( "context" "strings" "testing" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) { s, id, _ := exportJobFixture(t) ctx := context.Background() step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "failed", SessionID: "ses_original", Backend: "v2", Diagnostic: "offline"}} if err := s.FinishStepExportBlocked(ctx, id, step); err != nil { t.Fatal(err) } if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil { t.Fatal("successor started while export was incomplete") } paused, err := s.GetRun(ctx, id) if err != nil || paused.Status != "waiting" || paused.ExportPendingSequence != 1 || paused.FinishedAt != "" { t.Fatalf("gate became terminal: %+v %v", paused, err) } if _, _, err := s.ResolveDecision(ctx, id, "one", "success", ""); err == nil { t.Fatal("model export wait accepted human decision") } job, _ := s.SessionExportJob(ctx, id, 1) claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute) if err != nil || claimed == nil { t.Fatalf("missing export-only retry: %v", err) } if err := s.RecordSessionExportRepair(ctx, id, 1, &protocol.SessionExport{Status: "complete", SessionID: claimed.SessionID, Backend: claimed.Backend, Directory: claimed.Directory, Messages: 2, HTML: "session-repair-1-test.html", HTMLBytes: 100}); err != nil { t.Fatal(err) } if err := s.FinishSessionExportJob(ctx, claimed, time.Now(), ""); err != nil { t.Fatal(err) } if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil { t.Fatal(err) } if err := s.ResumeSessionExportGate(ctx, id, 1); err == nil { t.Fatal("duplicate continuation admitted") } // A crash after reserving the continuation but before starting its next // executor must retain the checkpoint for startup's interrupted run. for _, prior := range []string{"failed", "interrupted", "running", "queued"} { if _, err := s.db.Exec(`UPDATE pipeline_runs SET status=?,finished_at=CURRENT_TIMESTAMP WHERE id=?`, prior, id); err != nil { t.Fatal(err) } if err := s.RecoverRuns(ctx); err != nil { t.Fatal(err) } recovered, err := s.GetRun(ctx, id) if err != nil || recovered.Status != "waiting" || recovered.ExportPendingSequence != 1 || recovered.FinishedAt != "" { t.Fatalf("restart exposed a terminal export wait from %s: %+v %v", prior, recovered, err) } } ready, err := s.ReadySessionExportGates(ctx) if err != nil || ready[id] != 1 { t.Fatalf("lost restart checkpoint: %v %v", ready, err) } if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil { t.Fatal(err) } if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil { t.Fatal(err) } if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked { t.Fatalf("successor did not consume checkpoint: %v %v", blocked, err) } if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil { t.Fatal("successor started twice") } } // A recovery job wedged in running past its lease (or pending and never // claimable) is bounded by the gate's overall wait; a live lease is not stolen. func TestSessionExportGateWaitIsBoundedForStuckJob(t *testing.T) { s, id, _ := exportJobFixture(t) ctx := context.Background() step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "partial", SessionID: "ses_original", Backend: "v2", Truncated: true}} if err := s.FinishStepExportBlocked(ctx, id, step); err != nil { t.Fatal(err) } job, _ := s.SessionExportJob(ctx, id, 1) start := time.UnixMilli(job.NextAt) if claimed, err := s.ClaimSessionExportJob(ctx, start, time.Hour); err != nil || claimed == nil { t.Fatalf("claim: %+v %v", claimed, err) } if n, err := s.ExpireSessionExportGates(ctx, start.Add(5*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 { t.Fatalf("young gate expired: %d %v", n, err) } if n, err := s.ExpireSessionExportGates(ctx, start.Add(20*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 { t.Fatalf("live lease stolen: %d %v", n, err) } if ready, _ := s.ReadySessionExportGates(ctx); ready[id] != 0 { t.Fatal("running job released its gate") } if n, err := s.ExpireSessionExportGates(ctx, start.Add(2*time.Hour), SessionExportGateMaxWait); err != nil || n != 1 { t.Fatalf("stuck job not bounded: %d %v", n, err) } job, _ = s.SessionExportJob(ctx, id, 1) if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "wait exceeded") { t.Fatalf("job not exhausted: %+v", job) } ready, err := s.ReadySessionExportGates(ctx) if err != nil || ready[id] != 1 { t.Fatalf("exhausted gate not ready: %v %v", ready, err) } if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil { t.Fatal(err) } if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { t.Fatal(err) } if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked { t.Fatalf("terminal finish kept exhausted gate: %v %v", blocked, err) } d, _ := s.GetRun(ctx, id) if d.Status != "succeeded" || d.Steps[0].SessionExport == nil || d.Steps[0].SessionExport.Status != "partial" || len(d.Steps[0].SessionExportRepairs) != 0 { t.Fatalf("partial evidence not retained as-is: %+v", d) } } // A gate whose visit never got a recovery job (no exact owner) is given an // exhausted job after the bounded wait, and the release records why. func TestSessionExportGateWithoutJobIsReleasedAfterWait(t *testing.T) { s, id, _ := exportJobFixture(t) ctx := context.Background() if _, err := s.db.Exec(`DELETE FROM run_session_owners WHERE run_id=?`, id); err != nil { t.Fatal(err) } step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}} if err := s.FinishStepExportBlocked(ctx, id, step); err != nil { t.Fatal(err) } if job, _ := s.SessionExportJob(ctx, id, 1); job != nil { t.Fatalf("fixture unexpectedly has a job: %+v", job) } if n, err := s.ExpireSessionExportGates(ctx, time.Now().Add(time.Hour), SessionExportGateMaxWait); err != nil || n != 1 { t.Fatalf("jobless gate not bounded: %d %v", n, err) } if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil { t.Fatal(err) } if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil { t.Fatalf("successor refused after release: %v", err) } d, _ := s.GetRun(ctx, id) repairs := d.Steps[0].SessionExportRepairs if len(repairs) != 1 || repairs[0].Status != "unavailable" || !strings.Contains(repairs[0].Diagnostic, "No automatic session export recovery") { t.Fatalf("release did not record unavailable evidence: %+v", repairs) } }