package server import ( "context" "encoding/json" "errors" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/stage" ) func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) { t.Helper() h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) { return stage.SessionExportResult{}, errors.New("initial readback unavailable") }) // Synthetic executor has no agent process; model confirmed runtime cleanup. if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil { t.Fatal(err) } h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { t.Error("export recovery reran a stage") return stage.Result{}, errors.New("forbidden rerun") } return h, s, id } func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) { h, s, id := recoveryRunner(t) ctx := context.Background() original, _ := s.GetRun(ctx, id) before, _ := json.Marshal(original.Steps[0].SessionExport) owner, _ := s.RepairSessionOwner(ctx, id, 1) var calls atomic.Int32 h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) { if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory { t.Errorf("ownership changed: %+v", request) } if calls.Add(1) == 1 { return stage.SessionExportResult{}, errors.New("temporary upstream read failure") } body := []byte("

recorded messages

") if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil { return stage.SessionExportResult{}, err } return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil } for i := 0; i < 2; i++ { job, _ := s.SessionExportJob(ctx, id, 1) worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)) if err != nil || !worked { t.Fatalf("attempt %d: %v %v", i, worked, err) } } job, _ := s.SessionExportJob(ctx, id, 1) if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 { t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load()) } detail, _ := s.GetRun(ctx, id) after, _ := json.Marshal(detail.Steps[0].SessionExport) if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 { t.Fatalf("original evidence/execution changed: %+v", detail) } if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" { t.Fatal("recovery state not visible") } } func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) { h, s, id := recoveryRunner(t) ctx := context.Background() var calls atomic.Int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, password, ok := r.BasicAuth() if !ok || user != "opencode" || password != "fixture-password" { w.WriteHeader(http.StatusUnauthorized) return } if r.Method != "GET" || r.URL.Path != "/api/experimental/session/ses_run/export" { w.WriteHeader(http.StatusNotFound) return } calls.Add(1) w.Header().Set("Content-Type", "application/json") w.Write([]byte(`{"info":{"id":"ses_wrong"},"messages":[{"info":{"role":"assistant"},"parts":[{"type":"text","text":"wrong"}]}]}`)) })) t.Cleanup(srv.Close) localExportService(t, srv.URL) h.runner.exportSession = stage.ExportSession for range 3 { job, _ := s.SessionExportJob(ctx, id, 1) if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked { t.Fatalf("job failed: %v %v", worked, err) } } job, _ := s.SessionExportJob(ctx, id, 1) if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") { t.Fatalf("wrong session looks clean: %+v", job) } if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked { t.Fatalf("unbounded recovery: %v %v", worked, err) } if calls.Load() != 3 { t.Fatalf("wrong-session read was retried internally: %d", calls.Load()) } detail, _ := s.GetRun(ctx, id) if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 { t.Fatalf("lost failed attempts: %+v", detail.Steps) } for _, repair := range detail.Steps[0].SessionExportRepairs { if repair.Status != "failed" || repair.HTML != "" { t.Fatalf("wrong session marked clean: %+v", repair) } } } func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) { h, s, id := recoveryRunner(t) ctx := context.Background() h.runner.mu.Lock() h.runner.active[id] = activeRun{} h.runner.mu.Unlock() defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }() job, _ := s.SessionExportJob(ctx, id, 1) if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked { t.Fatalf("deferral failed: %v %v", worked, err) } job, _ = s.SessionExportJob(ctx, id, 1) if job.Status != "pending" || job.Attempts != 0 { t.Fatalf("cleanup spent attempt: %+v", job) } } func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) { for _, closeStore := range []bool{false, true} { ctx, cancel := context.WithCancel(context.Background()) s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db")) if err != nil { t.Fatal(err) } r, err := newRunner(ctx, s) if err != nil { t.Fatal(err) } if closeStore { s.Close() } else { cancel() } done := make(chan struct{}) go func() { r.schedulerWG.Wait(); close(done) }() select { case <-done: case <-time.After(3 * time.Second): t.Fatal("export worker did not stop") } cancel() s.Close() } } func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) { h, s, id := recoveryRunner(t) root := h.runner.artifactsRoot path := filepath.Join(s.Directory(), "session.db") original, _ := s.GetRun(context.Background(), id) before, _ := json.Marshal(original.Steps[0].SessionExport) if err := s.Close(); err != nil { t.Fatal(err) } s, err := pipeline.Open(path) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() sessionExportCLI(t) // read-only native CLI fixture, never a live model/run r, err := newRunner(ctx, s) if err != nil { t.Fatal(err) } defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }() r.artifactsRoot = root r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { t.Error("restart recovery reran a stage") return stage.Result{}, errors.New("forbidden rerun") } deadline := time.Now().Add(9 * time.Second) for { job, err := s.SessionExportJob(ctx, id, 1) if err != nil { t.Fatal(err) } settled, _ := s.GetRun(ctx, id) if job != nil && job.Status == "complete" && settled.Status == "succeeded" { break } if time.Now().After(deadline) { t.Fatalf("persisted job did not auto-recover: %+v", job) } time.Sleep(25 * time.Millisecond) } detail, err := s.GetRun(ctx, id) if err != nil { t.Fatal(err) } after, _ := json.Marshal(detail.Steps[0].SessionExport) if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" { t.Fatalf("recovery overwrote history or reran stage: %+v", detail) } } func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) { done := make(chan struct{}) ctx, cancel := sessionExportWorkerContext(context.Background(), done) defer cancel() exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout) defer stop() close(done) select { case <-exportCtx.Done(): case <-time.After(time.Second): t.Fatal("Store.Done did not cancel in-flight readback context") } }