3bfb3d6b44
A run stays in r.active after finishRunState persists its terminal status until its goroutine finishes notifying, so a decision in that window was refused as "already active". decide now refuses only runs whose terminal state is not yet durable; a finished run gets ResolveDecision's already-decided / not-waiting answer, and an executing run is still refused. Tests overwrote runner.artifactsRoot after Handler/newRunner had started background goroutines (sizeLoop, session export jobs) that read it. The root is now fixed at construction through newRunnerAt, which HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it there instead of mutating the field afterwards.
233 lines
8.1 KiB
Go
233 lines
8.1 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) {
|
|
t.Helper()
|
|
h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("initial readback unavailable")
|
|
})
|
|
// Synthetic executor has no agent process; model confirmed runtime cleanup.
|
|
if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
t.Error("export recovery reran a stage")
|
|
return stage.Result{}, errors.New("forbidden rerun")
|
|
}
|
|
return h, s, id
|
|
}
|
|
|
|
func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) {
|
|
h, s, id := recoveryRunner(t)
|
|
ctx := context.Background()
|
|
original, _ := s.GetRun(ctx, id)
|
|
before, _ := json.Marshal(original.Steps[0].SessionExport)
|
|
owner, _ := s.RepairSessionOwner(ctx, id, 1)
|
|
var calls atomic.Int32
|
|
h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory {
|
|
t.Errorf("ownership changed: %+v", request)
|
|
}
|
|
if calls.Add(1) == 1 {
|
|
return stage.SessionExportResult{}, errors.New("temporary upstream read failure")
|
|
}
|
|
body := []byte("<!doctype html><p>recorded messages</p>")
|
|
if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt))
|
|
if err != nil || !worked {
|
|
t.Fatalf("attempt %d: %v %v", i, worked, err)
|
|
}
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 {
|
|
t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load())
|
|
}
|
|
detail, _ := s.GetRun(ctx, id)
|
|
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
|
if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 {
|
|
t.Fatalf("original evidence/execution changed: %+v", detail)
|
|
}
|
|
if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" {
|
|
t.Fatal("recovery state not visible")
|
|
}
|
|
}
|
|
|
|
func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) {
|
|
h, s, id := recoveryRunner(t)
|
|
ctx := context.Background()
|
|
var calls atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
user, password, ok := r.BasicAuth()
|
|
if !ok || user != "opencode" || password != "fixture-password" {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
return
|
|
}
|
|
if r.Method != "GET" || r.URL.Path != "/api/experimental/session/ses_run/export" {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
calls.Add(1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(`{"info":{"id":"ses_wrong"},"messages":[{"info":{"role":"assistant"},"parts":[{"type":"text","text":"wrong"}]}]}`))
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
localExportService(t, srv.URL)
|
|
h.runner.exportSession = stage.ExportSession
|
|
for range 3 {
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
|
t.Fatalf("job failed: %v %v", worked, err)
|
|
}
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") {
|
|
t.Fatalf("wrong session looks clean: %+v", job)
|
|
}
|
|
if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked {
|
|
t.Fatalf("unbounded recovery: %v %v", worked, err)
|
|
}
|
|
if calls.Load() != 3 {
|
|
t.Fatalf("wrong-session read was retried internally: %d", calls.Load())
|
|
}
|
|
detail, _ := s.GetRun(ctx, id)
|
|
if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 {
|
|
t.Fatalf("lost failed attempts: %+v", detail.Steps)
|
|
}
|
|
for _, repair := range detail.Steps[0].SessionExportRepairs {
|
|
if repair.Status != "failed" || repair.HTML != "" {
|
|
t.Fatalf("wrong session marked clean: %+v", repair)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) {
|
|
h, s, id := recoveryRunner(t)
|
|
ctx := context.Background()
|
|
h.runner.mu.Lock()
|
|
h.runner.active[id] = activeRun{}
|
|
h.runner.mu.Unlock()
|
|
defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }()
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
|
t.Fatalf("deferral failed: %v %v", worked, err)
|
|
}
|
|
job, _ = s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "pending" || job.Attempts != 0 {
|
|
t.Fatalf("cleanup spent attempt: %+v", job)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) {
|
|
for _, closeStore := range []bool{false, true} {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := newRunner(ctx, s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if closeStore {
|
|
s.Close()
|
|
} else {
|
|
cancel()
|
|
}
|
|
done := make(chan struct{})
|
|
go func() { r.schedulerWG.Wait(); close(done) }()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("export worker did not stop")
|
|
}
|
|
cancel()
|
|
s.Close()
|
|
}
|
|
}
|
|
|
|
func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) {
|
|
h, s, id := recoveryRunner(t)
|
|
root := h.runner.artifactsRoot
|
|
path := filepath.Join(s.Directory(), "session.db")
|
|
original, _ := s.GetRun(context.Background(), id)
|
|
before, _ := json.Marshal(original.Steps[0].SessionExport)
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
sessionExportCLI(t) // read-only native CLI fixture, never a live model/run
|
|
r, err := newRunnerAt(ctx, s, root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }()
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
t.Error("restart recovery reran a stage")
|
|
return stage.Result{}, errors.New("forbidden rerun")
|
|
}
|
|
deadline := time.Now().Add(9 * time.Second)
|
|
for {
|
|
job, err := s.SessionExportJob(ctx, id, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
settled, _ := s.GetRun(ctx, id)
|
|
if job != nil && job.Status == "complete" && settled.Status == "succeeded" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("persisted job did not auto-recover: %+v", job)
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
detail, err := s.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
|
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
|
|
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) {
|
|
done := make(chan struct{})
|
|
ctx, cancel := sessionExportWorkerContext(context.Background(), done)
|
|
defer cancel()
|
|
exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout)
|
|
defer stop()
|
|
close(done)
|
|
select {
|
|
case <-exportCtx.Done():
|
|
case <-time.After(time.Second):
|
|
t.Fatal("Store.Done did not cancel in-flight readback context")
|
|
}
|
|
}
|