318 lines
12 KiB
Go
318 lines
12 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func exportJobFixture(t *testing.T) (*Store, int64, string) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "jobs.db")
|
|
s, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { s.Close() })
|
|
if _, err := s.Submit(ctx, "fixture", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
definition, _ := s.Get(ctx, items[0].ID, 1)
|
|
id, _, err := s.CreateRun(ctx, definition, "jobs", "/original/directory", false, nil, map[string]any{"prompt": "fixture"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.RunSession(ctx, id, 1, "ses_original", false, "v2", "original-reference"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s, id, path
|
|
}
|
|
|
|
func finishExportJobStep(t *testing.T, s *Store, id int64) {
|
|
t.Helper()
|
|
if err := s.FinishStep(context.Background(), id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Output: map[string]any{}, SessionExport: &protocol.SessionExport{Status: "empty", Messages: 0, SessionID: "ses_original", Backend: "v2"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportJobEnqueueIsAtomicAndWaitsForSettlement(t *testing.T) {
|
|
s, id, _ := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
if _, err := s.db.Exec(`CREATE TRIGGER fail_usage BEFORE INSERT ON step_usage BEGIN SELECT RAISE(ABORT,'fixture usage failure'); END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Output: map[string]any{}, Usage: &protocol.StageUsage{}})
|
|
if err == nil {
|
|
t.Fatal("fixture did not fail")
|
|
}
|
|
job, err := s.SessionExportJob(ctx, id, 1)
|
|
if err != nil || job != nil {
|
|
t.Fatalf("job escaped rolled-back step: %+v %v", job, err)
|
|
}
|
|
if _, err := s.db.Exec("DROP TRIGGER fail_usage"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
finishExportJobStep(t, s, id)
|
|
job, err = s.SessionExportJob(ctx, id, 1)
|
|
if err != nil || job == nil || job.SessionID != "ses_original" || job.Directory != "/original/directory" || job.Reference != "original-reference" {
|
|
t.Fatalf("snapshot missing: %+v %v", job, err)
|
|
}
|
|
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
|
|
t.Fatalf("unsettled run claimed: %+v %v", claimed, err)
|
|
}
|
|
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt-1), time.Minute); err != nil || claimed != nil {
|
|
t.Fatalf("backoff ignored: %+v %v", claimed, err)
|
|
}
|
|
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
|
|
if err != nil || claimed == nil || claimed.Attempts != 1 {
|
|
t.Fatalf("settled due job not claimed: %+v %v", claimed, err)
|
|
}
|
|
now := time.UnixMilli(job.NextAt)
|
|
if err := s.FinishSessionExportJob(ctx, claimed, now, "empty"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job, _ = s.SessionExportJob(ctx, id, 1)
|
|
if job.Attempts != 1 || job.Status != "pending" || job.NextAt != now.Add(20*time.Second).UnixMilli() {
|
|
t.Fatalf("first backoff wrong: %+v", job)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportJobBackoffRestartAndExhaustion(t *testing.T) {
|
|
s, id, path := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
finishExportJobStep(t, s, id)
|
|
if err := s.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
now := time.UnixMilli(job.NextAt)
|
|
claimed, err := s.ClaimSessionExportJob(ctx, now, time.Minute)
|
|
if err != nil || claimed == nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
if err := s.ReconcileSessionExportJobs(ctx, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if other, err := s.ClaimSessionExportJob(ctx, now.Add(59*time.Second), time.Minute); err != nil || other != nil {
|
|
t.Fatalf("live lease stolen: %+v %v", other, err)
|
|
}
|
|
recovered, err := s.ClaimSessionExportJob(ctx, now.Add(time.Minute), time.Minute)
|
|
if err != nil || recovered == nil || recovered.Attempts != 2 || recovered.Token == claimed.Token {
|
|
t.Fatalf("restart did not recover lease: %+v %v", recovered, err)
|
|
}
|
|
if err := s.FinishSessionExportJob(ctx, claimed, now, "stale worker"); err == nil {
|
|
t.Fatal("stale lease overwrote recovery")
|
|
}
|
|
if err := s.FinishSessionExportJob(ctx, recovered, now, "upstream unavailable"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job, _ = s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "pending" || job.NextAt != now.Add(60*time.Second).UnixMilli() {
|
|
t.Fatalf("second backoff wrong: %+v", job)
|
|
}
|
|
last, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
|
|
if err != nil || last == nil || last.Attempts != 3 {
|
|
t.Fatalf("third attempt missing: %+v %v", last, err)
|
|
}
|
|
if err := s.FinishSessionExportJob(ctx, last, now.Add(time.Minute), "session gone"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.ReconcileSessionExportJobs(ctx, now.Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job, _ = s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "exhausted" || job.Attempts != 3 || job.Diagnostic != "session gone" {
|
|
t.Fatalf("budget reset: %+v", job)
|
|
}
|
|
if next, err := s.ClaimSessionExportJob(ctx, now.Add(24*time.Hour), time.Minute); err != nil || next != nil {
|
|
t.Fatalf("infinite retry: %+v %v", next, err)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportJobGuardsCleanupAgentsAndExactOwnership(t *testing.T) {
|
|
s, id, _ := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
finishExportJobStep(t, s, id)
|
|
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.addColumnIfMissing("pipeline_runs", "cleanup_unconfirmed", "INTEGER NOT NULL DEFAULT 0"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, queries := range [][2]string{
|
|
{"UPDATE pipeline_runs SET cleanup_unconfirmed=1", "UPDATE pipeline_runs SET cleanup_unconfirmed=0"},
|
|
{"UPDATE run_steps SET agent_active=1", "UPDATE run_steps SET agent_active=0"},
|
|
} {
|
|
if _, err := s.db.Exec(queries[0]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
|
|
t.Fatalf("unsafe job claimed: %+v %v", job, err)
|
|
}
|
|
if _, err := s.db.Exec(queries[1]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='v1'"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
|
|
t.Fatalf("ownership drift exported: %+v %v", job, err)
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "exhausted" || job.Attempts != 0 || job.Backend != "v2" {
|
|
t.Fatalf("snapshot overwritten: %+v", job)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportJobRecoversPersistedSuccessWithoutAnotherRead(t *testing.T) {
|
|
s, id, _ := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
finishExportJobStep(t, s, id)
|
|
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute)
|
|
if err != nil || job == nil {
|
|
t.Fatal(err)
|
|
}
|
|
evidence := &protocol.SessionExport{Status: "complete", SessionID: "ses_wrong", Backend: job.Backend, Directory: job.Directory, HTML: "public/repaired.html", HTMLBytes: 100, Messages: 2}
|
|
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.FinishSessionExportJob(ctx, job, time.Now(), "wrong identity"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state, _ := s.SessionExportJob(ctx, id, 1)
|
|
if state.Status == "complete" {
|
|
t.Fatal("wrong-session evidence marked clean")
|
|
}
|
|
evidence.SessionID = job.SessionID
|
|
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(24*time.Hour), time.Minute); err != nil || claimed != nil {
|
|
t.Fatalf("successful persisted repair repeated: %+v %v", claimed, err)
|
|
}
|
|
state, _ = s.SessionExportJob(ctx, id, 1)
|
|
if state.Status != "complete" || state.Attempts != 1 {
|
|
t.Fatalf("acknowledgement not recovered: %+v", state)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportJobMigrationReconcilesMissingJobsWithoutReset(t *testing.T) {
|
|
s, id, path := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
finishExportJobStep(t, s, id)
|
|
original, err := s.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.db.Exec("DROP TABLE session_export_jobs"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
if err := s.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job, err := s.SessionExportJob(ctx, id, 1)
|
|
if err != nil || job == nil || job.SessionID != "ses_original" {
|
|
t.Fatalf("historical incomplete export not reconciled: %+v %v", job, err)
|
|
}
|
|
if err := s.ReconcileSessionExportJobs(ctx, time.Now().Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again, _ := s.SessionExportJob(ctx, id, 1)
|
|
if again.NextAt != job.NextAt || again.Attempts != job.Attempts {
|
|
t.Fatal("restart reset retry schedule")
|
|
}
|
|
current, err := s.GetRun(ctx, id)
|
|
if err != nil || current.Steps[0].SessionExport.Status != original.Steps[0].SessionExport.Status || current.Steps[0].SessionExport.Messages != 0 {
|
|
t.Fatal("migration changed original evidence")
|
|
}
|
|
}
|
|
|
|
func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) {
|
|
for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "pi-native", "wrong-session"} {
|
|
t.Run(status, func(t *testing.T) {
|
|
s, id, _ := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
evidence := &protocol.SessionExport{Status: status, SessionID: "ses_original", Backend: "v2", Messages: 2, HTML: "session.html", HTMLBytes: 100}
|
|
if status == "missing" {
|
|
evidence = nil
|
|
}
|
|
if status == "wrong-session" {
|
|
evidence.Status = "complete"
|
|
evidence.SessionID = "ses_wrong"
|
|
}
|
|
if status == "unsupported" {
|
|
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference=''"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if status == "pi-native" {
|
|
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference='/original/session.jsonl'"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
evidence.Backend, evidence.Status = "pi", "failed"
|
|
}
|
|
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", SessionExport: evidence, Output: map[string]any{}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job, err := s.SessionExportJob(ctx, id, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := status != "complete" && status != "unsupported"
|
|
if (job != nil) != want {
|
|
t.Fatalf("%s: %+v", status, job)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSessionExportJobFinalCrashedLeaseDoesNotResetBudget(t *testing.T) {
|
|
s, id, _ := exportJobFixture(t)
|
|
ctx := context.Background()
|
|
finishExportJobStep(t, s, id)
|
|
if err := s.FinishRun(ctx, id, "failed", "error", "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := s.db.Exec("UPDATE session_export_jobs SET status='running',attempts=3,lease_until=0,token='lost'"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
|
|
t.Fatalf("final crash retried: %+v %v", job, err)
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "exhausted" || job.Attempts != 3 || job.NextAt != 0 {
|
|
t.Fatalf("final lease not exhausted: %+v", job)
|
|
}
|
|
}
|