Files
pipelines/internal/pipeline/session_export_jobs_test.go
T

318 lines
12 KiB
Go

package pipeline
import (
"context"
"path/filepath"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func exportJobFixture(t *testing.T) (*Store, int64, string) {
t.Helper()
ctx := context.Background()
path := filepath.Join(t.TempDir(), "jobs.db")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
if _, err := s.Submit(ctx, "fixture", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
definition, _ := s.Get(ctx, items[0].ID, 1)
id, _, err := s.CreateRun(ctx, definition, "jobs", "/original/directory", false, nil, map[string]any{"prompt": "fixture"})
if err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
if err := s.RunSession(ctx, id, 1, "ses_original", false, "v2", "original-reference"); err != nil {
t.Fatal(err)
}
return s, id, path
}
func finishExportJobStep(t *testing.T, s *Store, id int64) {
t.Helper()
if err := s.FinishStep(context.Background(), id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Output: map[string]any{}, SessionExport: &protocol.SessionExport{Status: "empty", Messages: 0, SessionID: "ses_original", Backend: "v2"}}); err != nil {
t.Fatal(err)
}
}
func TestSessionExportJobEnqueueIsAtomicAndWaitsForSettlement(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
if _, err := s.db.Exec(`CREATE TRIGGER fail_usage BEFORE INSERT ON step_usage BEGIN SELECT RAISE(ABORT,'fixture usage failure'); END`); err != nil {
t.Fatal(err)
}
err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Output: map[string]any{}, Usage: &protocol.StageUsage{}})
if err == nil {
t.Fatal("fixture did not fail")
}
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job != nil {
t.Fatalf("job escaped rolled-back step: %+v %v", job, err)
}
if _, err := s.db.Exec("DROP TRIGGER fail_usage"); err != nil {
t.Fatal(err)
}
finishExportJobStep(t, s, id)
job, err = s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil || job.SessionID != "ses_original" || job.Directory != "/original/directory" || job.Reference != "original-reference" {
t.Fatalf("snapshot missing: %+v %v", job, err)
}
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
t.Fatalf("unsettled run claimed: %+v %v", claimed, err)
}
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
if claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt-1), time.Minute); err != nil || claimed != nil {
t.Fatalf("backoff ignored: %+v %v", claimed, err)
}
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
if err != nil || claimed == nil || claimed.Attempts != 1 {
t.Fatalf("settled due job not claimed: %+v %v", claimed, err)
}
now := time.UnixMilli(job.NextAt)
if err := s.FinishSessionExportJob(ctx, claimed, now, "empty"); err != nil {
t.Fatal(err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Attempts != 1 || job.Status != "pending" || job.NextAt != now.Add(20*time.Second).UnixMilli() {
t.Fatalf("first backoff wrong: %+v", job)
}
}
func TestSessionExportJobBackoffRestartAndExhaustion(t *testing.T) {
s, id, path := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
if err := s.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil {
t.Fatal(err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
now := time.UnixMilli(job.NextAt)
claimed, err := s.ClaimSessionExportJob(ctx, now, time.Minute)
if err != nil || claimed == nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := s.ReconcileSessionExportJobs(ctx, now); err != nil {
t.Fatal(err)
}
if other, err := s.ClaimSessionExportJob(ctx, now.Add(59*time.Second), time.Minute); err != nil || other != nil {
t.Fatalf("live lease stolen: %+v %v", other, err)
}
recovered, err := s.ClaimSessionExportJob(ctx, now.Add(time.Minute), time.Minute)
if err != nil || recovered == nil || recovered.Attempts != 2 || recovered.Token == claimed.Token {
t.Fatalf("restart did not recover lease: %+v %v", recovered, err)
}
if err := s.FinishSessionExportJob(ctx, claimed, now, "stale worker"); err == nil {
t.Fatal("stale lease overwrote recovery")
}
if err := s.FinishSessionExportJob(ctx, recovered, now, "upstream unavailable"); err != nil {
t.Fatal(err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "pending" || job.NextAt != now.Add(60*time.Second).UnixMilli() {
t.Fatalf("second backoff wrong: %+v", job)
}
last, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
if err != nil || last == nil || last.Attempts != 3 {
t.Fatalf("third attempt missing: %+v %v", last, err)
}
if err := s.FinishSessionExportJob(ctx, last, now.Add(time.Minute), "session gone"); err != nil {
t.Fatal(err)
}
if err := s.ReconcileSessionExportJobs(ctx, now.Add(time.Hour)); err != nil {
t.Fatal(err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 3 || job.Diagnostic != "session gone" {
t.Fatalf("budget reset: %+v", job)
}
if next, err := s.ClaimSessionExportJob(ctx, now.Add(24*time.Hour), time.Minute); err != nil || next != nil {
t.Fatalf("infinite retry: %+v %v", next, err)
}
}
func TestSessionExportJobGuardsCleanupAgentsAndExactOwnership(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
if err := s.addColumnIfMissing("pipeline_runs", "cleanup_unconfirmed", "INTEGER NOT NULL DEFAULT 0"); err != nil {
t.Fatal(err)
}
for _, queries := range [][2]string{
{"UPDATE pipeline_runs SET cleanup_unconfirmed=1", "UPDATE pipeline_runs SET cleanup_unconfirmed=0"},
{"UPDATE run_steps SET agent_active=1", "UPDATE run_steps SET agent_active=0"},
} {
if _, err := s.db.Exec(queries[0]); err != nil {
t.Fatal(err)
}
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
t.Fatalf("unsafe job claimed: %+v %v", job, err)
}
if _, err := s.db.Exec(queries[1]); err != nil {
t.Fatal(err)
}
}
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='v1'"); err != nil {
t.Fatal(err)
}
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
t.Fatalf("ownership drift exported: %+v %v", job, err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 0 || job.Backend != "v2" {
t.Fatalf("snapshot overwritten: %+v", job)
}
}
func TestSessionExportJobRecoversPersistedSuccessWithoutAnotherRead(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute)
if err != nil || job == nil {
t.Fatal(err)
}
evidence := &protocol.SessionExport{Status: "complete", SessionID: "ses_wrong", Backend: job.Backend, Directory: job.Directory, HTML: "public/repaired.html", HTMLBytes: 100, Messages: 2}
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
t.Fatal(err)
}
if err := s.FinishSessionExportJob(ctx, job, time.Now(), "wrong identity"); err != nil {
t.Fatal(err)
}
state, _ := s.SessionExportJob(ctx, id, 1)
if state.Status == "complete" {
t.Fatal("wrong-session evidence marked clean")
}
evidence.SessionID = job.SessionID
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
t.Fatal(err)
}
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(24*time.Hour), time.Minute); err != nil || claimed != nil {
t.Fatalf("successful persisted repair repeated: %+v %v", claimed, err)
}
state, _ = s.SessionExportJob(ctx, id, 1)
if state.Status != "complete" || state.Attempts != 1 {
t.Fatalf("acknowledgement not recovered: %+v", state)
}
}
func TestSessionExportJobMigrationReconcilesMissingJobsWithoutReset(t *testing.T) {
s, id, path := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
original, err := s.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec("DROP TABLE session_export_jobs"); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := s.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
t.Fatal(err)
}
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil || job.SessionID != "ses_original" {
t.Fatalf("historical incomplete export not reconciled: %+v %v", job, err)
}
if err := s.ReconcileSessionExportJobs(ctx, time.Now().Add(time.Hour)); err != nil {
t.Fatal(err)
}
again, _ := s.SessionExportJob(ctx, id, 1)
if again.NextAt != job.NextAt || again.Attempts != job.Attempts {
t.Fatal("restart reset retry schedule")
}
current, err := s.GetRun(ctx, id)
if err != nil || current.Steps[0].SessionExport.Status != original.Steps[0].SessionExport.Status || current.Steps[0].SessionExport.Messages != 0 {
t.Fatal("migration changed original evidence")
}
}
func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) {
for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "pi-native", "wrong-session"} {
t.Run(status, func(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
evidence := &protocol.SessionExport{Status: status, SessionID: "ses_original", Backend: "v2", Messages: 2, HTML: "session.html", HTMLBytes: 100}
if status == "missing" {
evidence = nil
}
if status == "wrong-session" {
evidence.Status = "complete"
evidence.SessionID = "ses_wrong"
}
if status == "unsupported" {
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference=''"); err != nil {
t.Fatal(err)
}
}
if status == "pi-native" {
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference='/original/session.jsonl'"); err != nil {
t.Fatal(err)
}
evidence.Backend, evidence.Status = "pi", "failed"
}
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", SessionExport: evidence, Output: map[string]any{}}); err != nil {
t.Fatal(err)
}
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
want := status != "complete" && status != "unsupported"
if (job != nil) != want {
t.Fatalf("%s: %+v", status, job)
}
})
}
}
func TestSessionExportJobFinalCrashedLeaseDoesNotResetBudget(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
if err := s.FinishRun(ctx, id, "failed", "error", "", nil); err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec("UPDATE session_export_jobs SET status='running',attempts=3,lease_until=0,token='lost'"); err != nil {
t.Fatal(err)
}
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
t.Fatalf("final crash retried: %+v %v", job, err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 3 || job.NextAt != 0 {
t.Fatalf("final lease not exhausted: %+v", job)
}
}