Files
pipelines/internal/server/session_export_jobs_test.go
T

234 lines
8.1 KiB
Go

package server
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) {
t.Helper()
h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("initial readback unavailable")
})
// Synthetic executor has no agent process; model confirmed runtime cleanup.
if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil {
t.Fatal(err)
}
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("export recovery reran a stage")
return stage.Result{}, errors.New("forbidden rerun")
}
return h, s, id
}
func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
original, _ := s.GetRun(ctx, id)
before, _ := json.Marshal(original.Steps[0].SessionExport)
owner, _ := s.RepairSessionOwner(ctx, id, 1)
var calls atomic.Int32
h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) {
if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory {
t.Errorf("ownership changed: %+v", request)
}
if calls.Add(1) == 1 {
return stage.SessionExportResult{}, errors.New("temporary upstream read failure")
}
body := []byte("<!doctype html><p>recorded messages</p>")
if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil {
return stage.SessionExportResult{}, err
}
return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil
}
for i := 0; i < 2; i++ {
job, _ := s.SessionExportJob(ctx, id, 1)
worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt))
if err != nil || !worked {
t.Fatalf("attempt %d: %v %v", i, worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 {
t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load())
}
detail, _ := s.GetRun(ctx, id)
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 {
t.Fatalf("original evidence/execution changed: %+v", detail)
}
if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" {
t.Fatal("recovery state not visible")
}
}
func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, password, ok := r.BasicAuth()
if !ok || user != "opencode" || password != "fixture-password" {
w.WriteHeader(http.StatusUnauthorized)
return
}
if r.Method != "GET" || r.URL.Path != "/api/experimental/session/ses_run/export" {
w.WriteHeader(http.StatusNotFound)
return
}
calls.Add(1)
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"info":{"id":"ses_wrong"},"messages":[{"info":{"role":"assistant"},"parts":[{"type":"text","text":"wrong"}]}]}`))
}))
t.Cleanup(srv.Close)
localExportService(t, srv.URL)
h.runner.exportSession = stage.ExportSession
for range 3 {
job, _ := s.SessionExportJob(ctx, id, 1)
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("job failed: %v %v", worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") {
t.Fatalf("wrong session looks clean: %+v", job)
}
if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked {
t.Fatalf("unbounded recovery: %v %v", worked, err)
}
if calls.Load() != 3 {
t.Fatalf("wrong-session read was retried internally: %d", calls.Load())
}
detail, _ := s.GetRun(ctx, id)
if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 {
t.Fatalf("lost failed attempts: %+v", detail.Steps)
}
for _, repair := range detail.Steps[0].SessionExportRepairs {
if repair.Status != "failed" || repair.HTML != "" {
t.Fatalf("wrong session marked clean: %+v", repair)
}
}
}
func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
h.runner.mu.Lock()
h.runner.active[id] = activeRun{}
h.runner.mu.Unlock()
defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }()
job, _ := s.SessionExportJob(ctx, id, 1)
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("deferral failed: %v %v", worked, err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "pending" || job.Attempts != 0 {
t.Fatalf("cleanup spent attempt: %+v", job)
}
}
func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) {
for _, closeStore := range []bool{false, true} {
ctx, cancel := context.WithCancel(context.Background())
s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db"))
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
if closeStore {
s.Close()
} else {
cancel()
}
done := make(chan struct{})
go func() { r.schedulerWG.Wait(); close(done) }()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("export worker did not stop")
}
cancel()
s.Close()
}
}
func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) {
h, s, id := recoveryRunner(t)
root := h.runner.artifactsRoot
path := filepath.Join(s.Directory(), "session.db")
original, _ := s.GetRun(context.Background(), id)
before, _ := json.Marshal(original.Steps[0].SessionExport)
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessionExportCLI(t) // read-only native CLI fixture, never a live model/run
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }()
r.artifactsRoot = root
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("restart recovery reran a stage")
return stage.Result{}, errors.New("forbidden rerun")
}
deadline := time.Now().Add(9 * time.Second)
for {
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
settled, _ := s.GetRun(ctx, id)
if job != nil && job.Status == "complete" && settled.Status == "succeeded" {
break
}
if time.Now().After(deadline) {
t.Fatalf("persisted job did not auto-recover: %+v", job)
}
time.Sleep(25 * time.Millisecond)
}
detail, err := s.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
}
}
func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) {
done := make(chan struct{})
ctx, cancel := sessionExportWorkerContext(context.Background(), done)
defer cancel()
exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout)
defer stop()
close(done)
select {
case <-exportCtx.Done():
case <-time.After(time.Second):
t.Fatal("Store.Done did not cancel in-flight readback context")
}
}