package server import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/stage" ) func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) { t.Helper() h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) { return stage.SessionExportResult{}, errors.New("initial readback unavailable") }) // Synthetic executor has no agent process; model confirmed runtime cleanup. if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil { t.Fatal(err) } h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { t.Error("export recovery reran a stage") return stage.Result{}, errors.New("forbidden rerun") } return h, s, id } func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) { h, s, id := recoveryRunner(t) ctx := context.Background() original, _ := s.GetRun(ctx, id) before, _ := json.Marshal(original.Steps[0].SessionExport) owner, _ := s.RepairSessionOwner(ctx, id, 1) var calls atomic.Int32 h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) { if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory { t.Errorf("ownership changed: %+v", request) } if calls.Add(1) == 1 { return stage.SessionExportResult{}, errors.New("temporary upstream read failure") } body := []byte("

recorded messages

") if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil { return stage.SessionExportResult{}, err } return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil } for i := 0; i < 2; i++ { job, _ := s.SessionExportJob(ctx, id, 1) worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)) if err != nil || !worked { t.Fatalf("attempt %d: %v %v", i, worked, err) } } job, _ := s.SessionExportJob(ctx, id, 1) if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 { t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load()) } detail, _ := s.GetRun(ctx, id) after, _ := json.Marshal(detail.Steps[0].SessionExport) if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 { t.Fatalf("original evidence/execution changed: %+v", detail) } if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" { t.Fatal("recovery state not visible") } } func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) { h, s, id := recoveryRunner(t) ctx := context.Background() dir := t.TempDir() calls := filepath.Join(dir, "calls") bin := filepath.Join(dir, "opencode") script := fmt.Sprintf("#!/bin/sh\necho read >> %q\necho '{\"info\":{\"id\":\"ses_wrong\"},\"messages\":[{\"info\":{\"role\":\"assistant\"},\"parts\":[{\"type\":\"text\",\"text\":\"wrong\"}]}]}'\n", calls) if err := os.WriteFile(bin, []byte(script), 0700); err != nil { t.Fatal(err) } t.Setenv("PIPELINES_OPENCODE", bin) h.runner.exportSession = stage.ExportSession for range 3 { job, _ := s.SessionExportJob(ctx, id, 1) if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked { t.Fatalf("job failed: %v %v", worked, err) } } job, _ := s.SessionExportJob(ctx, id, 1) if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") { t.Fatalf("wrong session looks clean: %+v", job) } if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked { t.Fatalf("unbounded recovery: %v %v", worked, err) } body, _ := os.ReadFile(calls) if strings.Count(string(body), "read") != 3 { t.Fatalf("wrong-session read was retried internally: %s", body) } detail, _ := s.GetRun(ctx, id) if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 { t.Fatalf("lost failed attempts: %+v", detail.Steps) } for _, repair := range detail.Steps[0].SessionExportRepairs { if repair.Status != "failed" || repair.HTML != "" { t.Fatalf("wrong session marked clean: %+v", repair) } } } func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) { h, s, id := recoveryRunner(t) ctx := context.Background() h.runner.mu.Lock() h.runner.active[id] = activeRun{} h.runner.mu.Unlock() defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }() job, _ := s.SessionExportJob(ctx, id, 1) if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked { t.Fatalf("deferral failed: %v %v", worked, err) } job, _ = s.SessionExportJob(ctx, id, 1) if job.Status != "pending" || job.Attempts != 0 { t.Fatalf("cleanup spent attempt: %+v", job) } } func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) { for _, closeStore := range []bool{false, true} { ctx, cancel := context.WithCancel(context.Background()) s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db")) if err != nil { t.Fatal(err) } r, err := newRunner(ctx, s) if err != nil { t.Fatal(err) } if closeStore { s.Close() } else { cancel() } done := make(chan struct{}) go func() { r.schedulerWG.Wait(); close(done) }() select { case <-done: case <-time.After(3 * time.Second): t.Fatal("export worker did not stop") } cancel() s.Close() } } func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) { h, s, id := recoveryRunner(t) root := h.runner.artifactsRoot path := filepath.Join(s.Directory(), "session.db") original, _ := s.GetRun(context.Background(), id) before, _ := json.Marshal(original.Steps[0].SessionExport) if err := s.Close(); err != nil { t.Fatal(err) } s, err := pipeline.Open(path) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() sessionExportCLI(t) // read-only native CLI fixture, never a live model/run r, err := newRunner(ctx, s) if err != nil { t.Fatal(err) } defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }() r.artifactsRoot = root r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { t.Error("restart recovery reran a stage") return stage.Result{}, errors.New("forbidden rerun") } deadline := time.Now().Add(9 * time.Second) for { job, err := s.SessionExportJob(ctx, id, 1) if err != nil { t.Fatal(err) } if job != nil && job.Status == "complete" { break } if time.Now().After(deadline) { t.Fatalf("persisted job did not auto-recover: %+v", job) } time.Sleep(25 * time.Millisecond) } detail, err := s.GetRun(ctx, id) if err != nil { t.Fatal(err) } after, _ := json.Marshal(detail.Steps[0].SessionExport) if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != original.Status { t.Fatalf("recovery overwrote history or reran stage: %+v", detail) } } func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) { done := make(chan struct{}) ctx, cancel := sessionExportWorkerContext(context.Background(), done) defer cancel() exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout) defer stop() close(done) select { case <-exportCtx.Done(): case <-time.After(time.Second): t.Fatal("Store.Done did not cancel in-flight readback context") } }