Files
pipelines/internal/server/session_export_jobs_test.go
alan 3bfb3d6b44 Fix decide-after-finish refusal and run size loop data race
A run stays in r.active after finishRunState persists its terminal status
until its goroutine finishes notifying, so a decision in that window was
refused as "already active". decide now refuses only runs whose terminal
state is not yet durable; a finished run gets ResolveDecision's
already-decided / not-waiting answer, and an executing run is still refused.

Tests overwrote runner.artifactsRoot after Handler/newRunner had started
background goroutines (sizeLoop, session export jobs) that read it. The
root is now fixed at construction through newRunnerAt, which
HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it
there instead of mutating the field afterwards.
2026-11-01 18:58:14 +00:00

233 lines
8.1 KiB
Go

package server
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) {
t.Helper()
h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("initial readback unavailable")
})
// Synthetic executor has no agent process; model confirmed runtime cleanup.
if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil {
t.Fatal(err)
}
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("export recovery reran a stage")
return stage.Result{}, errors.New("forbidden rerun")
}
return h, s, id
}
func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
original, _ := s.GetRun(ctx, id)
before, _ := json.Marshal(original.Steps[0].SessionExport)
owner, _ := s.RepairSessionOwner(ctx, id, 1)
var calls atomic.Int32
h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) {
if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory {
t.Errorf("ownership changed: %+v", request)
}
if calls.Add(1) == 1 {
return stage.SessionExportResult{}, errors.New("temporary upstream read failure")
}
body := []byte("<!doctype html><p>recorded messages</p>")
if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil {
return stage.SessionExportResult{}, err
}
return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil
}
for i := 0; i < 2; i++ {
job, _ := s.SessionExportJob(ctx, id, 1)
worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt))
if err != nil || !worked {
t.Fatalf("attempt %d: %v %v", i, worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 {
t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load())
}
detail, _ := s.GetRun(ctx, id)
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 {
t.Fatalf("original evidence/execution changed: %+v", detail)
}
if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" {
t.Fatal("recovery state not visible")
}
}
func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, password, ok := r.BasicAuth()
if !ok || user != "opencode" || password != "fixture-password" {
w.WriteHeader(http.StatusUnauthorized)
return
}
if r.Method != "GET" || r.URL.Path != "/api/experimental/session/ses_run/export" {
w.WriteHeader(http.StatusNotFound)
return
}
calls.Add(1)
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"info":{"id":"ses_wrong"},"messages":[{"info":{"role":"assistant"},"parts":[{"type":"text","text":"wrong"}]}]}`))
}))
t.Cleanup(srv.Close)
localExportService(t, srv.URL)
h.runner.exportSession = stage.ExportSession
for range 3 {
job, _ := s.SessionExportJob(ctx, id, 1)
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("job failed: %v %v", worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") {
t.Fatalf("wrong session looks clean: %+v", job)
}
if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked {
t.Fatalf("unbounded recovery: %v %v", worked, err)
}
if calls.Load() != 3 {
t.Fatalf("wrong-session read was retried internally: %d", calls.Load())
}
detail, _ := s.GetRun(ctx, id)
if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 {
t.Fatalf("lost failed attempts: %+v", detail.Steps)
}
for _, repair := range detail.Steps[0].SessionExportRepairs {
if repair.Status != "failed" || repair.HTML != "" {
t.Fatalf("wrong session marked clean: %+v", repair)
}
}
}
func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
h.runner.mu.Lock()
h.runner.active[id] = activeRun{}
h.runner.mu.Unlock()
defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }()
job, _ := s.SessionExportJob(ctx, id, 1)
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("deferral failed: %v %v", worked, err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "pending" || job.Attempts != 0 {
t.Fatalf("cleanup spent attempt: %+v", job)
}
}
func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) {
for _, closeStore := range []bool{false, true} {
ctx, cancel := context.WithCancel(context.Background())
s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db"))
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
if closeStore {
s.Close()
} else {
cancel()
}
done := make(chan struct{})
go func() { r.schedulerWG.Wait(); close(done) }()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("export worker did not stop")
}
cancel()
s.Close()
}
}
func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) {
h, s, id := recoveryRunner(t)
root := h.runner.artifactsRoot
path := filepath.Join(s.Directory(), "session.db")
original, _ := s.GetRun(context.Background(), id)
before, _ := json.Marshal(original.Steps[0].SessionExport)
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessionExportCLI(t) // read-only native CLI fixture, never a live model/run
r, err := newRunnerAt(ctx, s, root)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }()
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("restart recovery reran a stage")
return stage.Result{}, errors.New("forbidden rerun")
}
deadline := time.Now().Add(9 * time.Second)
for {
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
settled, _ := s.GetRun(ctx, id)
if job != nil && job.Status == "complete" && settled.Status == "succeeded" {
break
}
if time.Now().After(deadline) {
t.Fatalf("persisted job did not auto-recover: %+v", job)
}
time.Sleep(25 * time.Millisecond)
}
detail, err := s.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
}
}
func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) {
done := make(chan struct{})
ctx, cancel := sessionExportWorkerContext(context.Background(), done)
defer cancel()
exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout)
defer stop()
close(done)
select {
case <-exportCtx.Done():
case <-time.After(time.Second):
t.Fatal("Store.Done did not cancel in-flight readback context")
}
}