234 lines
8.1 KiB
Go
234 lines
8.1 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) {
|
|
t.Helper()
|
|
h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("initial readback unavailable")
|
|
})
|
|
// Synthetic executor has no agent process; model confirmed runtime cleanup.
|
|
if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
t.Error("export recovery reran a stage")
|
|
return stage.Result{}, errors.New("forbidden rerun")
|
|
}
|
|
return h, s, id
|
|
}
|
|
|
|
func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) {
|
|
h, s, id := recoveryRunner(t)
|
|
ctx := context.Background()
|
|
original, _ := s.GetRun(ctx, id)
|
|
before, _ := json.Marshal(original.Steps[0].SessionExport)
|
|
owner, _ := s.RepairSessionOwner(ctx, id, 1)
|
|
var calls atomic.Int32
|
|
h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory {
|
|
t.Errorf("ownership changed: %+v", request)
|
|
}
|
|
if calls.Add(1) == 1 {
|
|
return stage.SessionExportResult{}, errors.New("temporary upstream read failure")
|
|
}
|
|
body := []byte("<!doctype html><p>recorded messages</p>")
|
|
if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt))
|
|
if err != nil || !worked {
|
|
t.Fatalf("attempt %d: %v %v", i, worked, err)
|
|
}
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 {
|
|
t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load())
|
|
}
|
|
detail, _ := s.GetRun(ctx, id)
|
|
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
|
if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 {
|
|
t.Fatalf("original evidence/execution changed: %+v", detail)
|
|
}
|
|
if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" {
|
|
t.Fatal("recovery state not visible")
|
|
}
|
|
}
|
|
|
|
func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) {
|
|
h, s, id := recoveryRunner(t)
|
|
ctx := context.Background()
|
|
var calls atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
user, password, ok := r.BasicAuth()
|
|
if !ok || user != "opencode" || password != "fixture-password" {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if r.Method != "GET" || r.URL.Path != "/api/experimental/session/ses_run/export" {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
calls.Add(1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(`{"info":{"id":"ses_wrong"},"messages":[{"info":{"role":"assistant"},"parts":[{"type":"text","text":"wrong"}]}]}`))
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
localExportService(t, srv.URL)
|
|
h.runner.exportSession = stage.ExportSession
|
|
for range 3 {
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
|
t.Fatalf("job failed: %v %v", worked, err)
|
|
}
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") {
|
|
t.Fatalf("wrong session looks clean: %+v", job)
|
|
}
|
|
if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked {
|
|
t.Fatalf("unbounded recovery: %v %v", worked, err)
|
|
}
|
|
if calls.Load() != 3 {
|
|
t.Fatalf("wrong-session read was retried internally: %d", calls.Load())
|
|
}
|
|
detail, _ := s.GetRun(ctx, id)
|
|
if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 {
|
|
t.Fatalf("lost failed attempts: %+v", detail.Steps)
|
|
}
|
|
for _, repair := range detail.Steps[0].SessionExportRepairs {
|
|
if repair.Status != "failed" || repair.HTML != "" {
|
|
t.Fatalf("wrong session marked clean: %+v", repair)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) {
|
|
h, s, id := recoveryRunner(t)
|
|
ctx := context.Background()
|
|
h.runner.mu.Lock()
|
|
h.runner.active[id] = activeRun{}
|
|
h.runner.mu.Unlock()
|
|
defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }()
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
|
t.Fatalf("deferral failed: %v %v", worked, err)
|
|
}
|
|
job, _ = s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "pending" || job.Attempts != 0 {
|
|
t.Fatalf("cleanup spent attempt: %+v", job)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) {
|
|
for _, closeStore := range []bool{false, true} {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if closeStore {
|
|
s.Close()
|
|
} else {
|
|
cancel()
|
|
}
|
|
done := make(chan struct{})
|
|
go func() { r.schedulerWG.Wait(); close(done) }()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("export worker did not stop")
|
|
}
|
|
cancel()
|
|
s.Close()
|
|
}
|
|
}
|
|
|
|
func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) {
|
|
h, s, id := recoveryRunner(t)
|
|
root := h.runner.artifactsRoot
|
|
path := filepath.Join(s.Directory(), "session.db")
|
|
original, _ := s.GetRun(context.Background(), id)
|
|
before, _ := json.Marshal(original.Steps[0].SessionExport)
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
sessionExportCLI(t) // read-only native CLI fixture, never a live model/run
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }()
|
|
r.artifactsRoot = root
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
t.Error("restart recovery reran a stage")
|
|
return stage.Result{}, errors.New("forbidden rerun")
|
|
}
|
|
deadline := time.Now().Add(9 * time.Second)
|
|
for {
|
|
job, err := s.SessionExportJob(ctx, id, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
settled, _ := s.GetRun(ctx, id)
|
|
if job != nil && job.Status == "complete" && settled.Status == "succeeded" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("persisted job did not auto-recover: %+v", job)
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
detail, err := s.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
|
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
|
|
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) {
|
|
done := make(chan struct{})
|
|
ctx, cancel := sessionExportWorkerContext(context.Background(), done)
|
|
defer cancel()
|
|
exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout)
|
|
defer stop()
|
|
close(done)
|
|
select {
|
|
case <-exportCtx.Done():
|
|
case <-time.After(time.Second):
|
|
t.Fatal("Store.Done did not cancel in-flight readback context")
|
|
}
|
|
}
|