package pipeline import ( "context" "path/filepath" "testing" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) func exportJobFixture(t *testing.T) (*Store, int64, string) { t.Helper() ctx := context.Background() path := filepath.Join(t.TempDir(), "jobs.db") s, err := Open(path) if err != nil { t.Fatal(err) } t.Cleanup(func() { s.Close() }) if _, err := s.Submit(ctx, "fixture", Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := s.List(ctx) definition, _ := s.Get(ctx, items[0].ID, 1) id, _, err := s.CreateRun(ctx, definition, "jobs", "/original/directory", false, nil, map[string]any{"prompt": "fixture"}) if err != nil { t.Fatal(err) } if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil { t.Fatal(err) } if err := s.RunSession(ctx, id, 1, "ses_original", false, "v2", "original-reference"); err != nil { t.Fatal(err) } return s, id, path } func finishExportJobStep(t *testing.T, s *Store, id int64) { t.Helper() if err := s.FinishStep(context.Background(), id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Output: map[string]any{}, SessionExport: &protocol.SessionExport{Status: "empty", Messages: 0, SessionID: "ses_original", Backend: "v2"}}); err != nil { t.Fatal(err) } } func TestSessionExportJobEnqueueIsAtomicAndWaitsForSettlement(t *testing.T) { s, id, _ := exportJobFixture(t) ctx := context.Background() if _, err := s.db.Exec(`CREATE TRIGGER fail_usage BEFORE INSERT ON step_usage BEGIN SELECT RAISE(ABORT,'fixture usage failure'); END`); err != nil { t.Fatal(err) } err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Output: map[string]any{}, Usage: &protocol.StageUsage{}}) if err == nil { t.Fatal("fixture did not fail") } job, err := s.SessionExportJob(ctx, id, 1) if err != nil || job != nil { t.Fatalf("job escaped rolled-back step: %+v %v", job, err) } if _, err := s.db.Exec("DROP TRIGGER fail_usage"); err != nil { t.Fatal(err) } finishExportJobStep(t, s, id) job, err = s.SessionExportJob(ctx, id, 1) if err != nil || job == nil || job.SessionID != "ses_original" || job.Directory != "/original/directory" || job.Reference != "original-reference" { t.Fatalf("snapshot missing: %+v %v", job, err) } if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil { t.Fatalf("unsettled run claimed: %+v %v", claimed, err) } if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { t.Fatal(err) } if claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt-1), time.Minute); err != nil || claimed != nil { t.Fatalf("backoff ignored: %+v %v", claimed, err) } claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute) if err != nil || claimed == nil || claimed.Attempts != 1 { t.Fatalf("settled due job not claimed: %+v %v", claimed, err) } now := time.UnixMilli(job.NextAt) if err := s.FinishSessionExportJob(ctx, claimed, now, "empty"); err != nil { t.Fatal(err) } job, _ = s.SessionExportJob(ctx, id, 1) if job.Attempts != 1 || job.Status != "pending" || job.NextAt != now.Add(20*time.Second).UnixMilli() { t.Fatalf("first backoff wrong: %+v", job) } } func TestSessionExportJobBackoffRestartAndExhaustion(t *testing.T) { s, id, path := exportJobFixture(t) ctx := context.Background() finishExportJobStep(t, s, id) if err := s.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil { t.Fatal(err) } job, _ := s.SessionExportJob(ctx, id, 1) now := time.UnixMilli(job.NextAt) claimed, err := s.ClaimSessionExportJob(ctx, now, time.Minute) if err != nil || claimed == nil { t.Fatal(err) } if err := s.Close(); err != nil { t.Fatal(err) } s, err = Open(path) if err != nil { t.Fatal(err) } defer s.Close() if err := s.ReconcileSessionExportJobs(ctx, now); err != nil { t.Fatal(err) } if other, err := s.ClaimSessionExportJob(ctx, now.Add(59*time.Second), time.Minute); err != nil || other != nil { t.Fatalf("live lease stolen: %+v %v", other, err) } recovered, err := s.ClaimSessionExportJob(ctx, now.Add(time.Minute), time.Minute) if err != nil || recovered == nil || recovered.Attempts != 2 || recovered.Token == claimed.Token { t.Fatalf("restart did not recover lease: %+v %v", recovered, err) } if err := s.FinishSessionExportJob(ctx, claimed, now, "stale worker"); err == nil { t.Fatal("stale lease overwrote recovery") } if err := s.FinishSessionExportJob(ctx, recovered, now, "upstream unavailable"); err != nil { t.Fatal(err) } job, _ = s.SessionExportJob(ctx, id, 1) if job.Status != "pending" || job.NextAt != now.Add(60*time.Second).UnixMilli() { t.Fatalf("second backoff wrong: %+v", job) } last, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute) if err != nil || last == nil || last.Attempts != 3 { t.Fatalf("third attempt missing: %+v %v", last, err) } if err := s.FinishSessionExportJob(ctx, last, now.Add(time.Minute), "session gone"); err != nil { t.Fatal(err) } if err := s.ReconcileSessionExportJobs(ctx, now.Add(time.Hour)); err != nil { t.Fatal(err) } job, _ = s.SessionExportJob(ctx, id, 1) if job.Status != "exhausted" || job.Attempts != 3 || job.Diagnostic != "session gone" { t.Fatalf("budget reset: %+v", job) } if next, err := s.ClaimSessionExportJob(ctx, now.Add(24*time.Hour), time.Minute); err != nil || next != nil { t.Fatalf("infinite retry: %+v %v", next, err) } } func TestSessionExportJobGuardsCleanupAgentsAndExactOwnership(t *testing.T) { s, id, _ := exportJobFixture(t) ctx := context.Background() finishExportJobStep(t, s, id) if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { t.Fatal(err) } if err := s.addColumnIfMissing("pipeline_runs", "cleanup_unconfirmed", "INTEGER NOT NULL DEFAULT 0"); err != nil { t.Fatal(err) } for _, queries := range [][2]string{ {"UPDATE pipeline_runs SET cleanup_unconfirmed=1", "UPDATE pipeline_runs SET cleanup_unconfirmed=0"}, {"UPDATE run_steps SET agent_active=1", "UPDATE run_steps SET agent_active=0"}, } { if _, err := s.db.Exec(queries[0]); err != nil { t.Fatal(err) } if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil { t.Fatalf("unsafe job claimed: %+v %v", job, err) } if _, err := s.db.Exec(queries[1]); err != nil { t.Fatal(err) } } if _, err := s.db.Exec("UPDATE run_session_owners SET backend='v1'"); err != nil { t.Fatal(err) } if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil { t.Fatalf("ownership drift exported: %+v %v", job, err) } job, _ := s.SessionExportJob(ctx, id, 1) if job.Status != "exhausted" || job.Attempts != 0 || job.Backend != "v2" { t.Fatalf("snapshot overwritten: %+v", job) } } func TestSessionExportJobRecoversPersistedSuccessWithoutAnotherRead(t *testing.T) { s, id, _ := exportJobFixture(t) ctx := context.Background() finishExportJobStep(t, s, id) if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { t.Fatal(err) } job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute) if err != nil || job == nil { t.Fatal(err) } evidence := &protocol.SessionExport{Status: "complete", SessionID: "ses_wrong", Backend: job.Backend, Directory: job.Directory, HTML: "public/repaired.html", HTMLBytes: 100, Messages: 2} if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil { t.Fatal(err) } if err := s.FinishSessionExportJob(ctx, job, time.Now(), "wrong identity"); err != nil { t.Fatal(err) } state, _ := s.SessionExportJob(ctx, id, 1) if state.Status == "complete" { t.Fatal("wrong-session evidence marked clean") } evidence.SessionID = job.SessionID if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil { t.Fatal(err) } if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(24*time.Hour), time.Minute); err != nil || claimed != nil { t.Fatalf("successful persisted repair repeated: %+v %v", claimed, err) } state, _ = s.SessionExportJob(ctx, id, 1) if state.Status != "complete" || state.Attempts != 1 { t.Fatalf("acknowledgement not recovered: %+v", state) } } func TestSessionExportJobMigrationReconcilesMissingJobsWithoutReset(t *testing.T) { s, id, path := exportJobFixture(t) ctx := context.Background() finishExportJobStep(t, s, id) original, err := s.GetRun(ctx, id) if err != nil { t.Fatal(err) } if _, err := s.db.Exec("DROP TABLE session_export_jobs"); err != nil { t.Fatal(err) } if err := s.Close(); err != nil { t.Fatal(err) } s, err = Open(path) if err != nil { t.Fatal(err) } defer s.Close() if err := s.ReconcileSessionExportJobs(ctx, time.Now()); err != nil { t.Fatal(err) } job, err := s.SessionExportJob(ctx, id, 1) if err != nil || job == nil || job.SessionID != "ses_original" { t.Fatalf("historical incomplete export not reconciled: %+v %v", job, err) } if err := s.ReconcileSessionExportJobs(ctx, time.Now().Add(time.Hour)); err != nil { t.Fatal(err) } again, _ := s.SessionExportJob(ctx, id, 1) if again.NextAt != job.NextAt || again.Attempts != job.Attempts { t.Fatal("restart reset retry schedule") } current, err := s.GetRun(ctx, id) if err != nil || current.Steps[0].SessionExport.Status != original.Steps[0].SessionExport.Status || current.Steps[0].SessionExport.Messages != 0 { t.Fatal("migration changed original evidence") } } func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) { for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "pi-native", "wrong-session"} { t.Run(status, func(t *testing.T) { s, id, _ := exportJobFixture(t) ctx := context.Background() evidence := &protocol.SessionExport{Status: status, SessionID: "ses_original", Backend: "v2", Messages: 2, HTML: "session.html", HTMLBytes: 100} if status == "missing" { evidence = nil } if status == "wrong-session" { evidence.Status = "complete" evidence.SessionID = "ses_wrong" } if status == "unsupported" { if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference=''"); err != nil { t.Fatal(err) } } if status == "pi-native" { if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference='/original/session.jsonl'"); err != nil { t.Fatal(err) } evidence.Backend, evidence.Status = "pi", "failed" } if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", SessionExport: evidence, Output: map[string]any{}}); err != nil { t.Fatal(err) } job, err := s.SessionExportJob(ctx, id, 1) if err != nil { t.Fatal(err) } want := status != "complete" && status != "unsupported" if (job != nil) != want { t.Fatalf("%s: %+v", status, job) } }) } } func TestSessionExportJobFinalCrashedLeaseDoesNotResetBudget(t *testing.T) { s, id, _ := exportJobFixture(t) ctx := context.Background() finishExportJobStep(t, s, id) if err := s.FinishRun(ctx, id, "failed", "error", "", nil); err != nil { t.Fatal(err) } if _, err := s.db.Exec("UPDATE session_export_jobs SET status='running',attempts=3,lease_until=0,token='lost'"); err != nil { t.Fatal(err) } if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil { t.Fatalf("final crash retried: %+v %v", job, err) } job, _ := s.SessionExportJob(ctx, id, 1) if job.Status != "exhausted" || job.Attempts != 3 || job.NextAt != 0 { t.Fatalf("final lease not exhausted: %+v", job) } }