Files
pipelines/internal/server/session_export_gate_test.go
T

340 lines
12 KiB
Go

package server
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
type gateTestOptions struct {
missingOwner bool
waitForCancel bool
}
func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error), active bool, options ...gateTestOptions) (*runHandler, *pipeline.Store, int64, *atomic.Int32, *atomic.Int32) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
s, err := pipeline.Open(filepath.Join(t.TempDir(), "gate.db"))
if err != nil {
t.Fatal(err)
}
h := Handler(ctx, s).(*runHandler)
t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() })
h.runner.artifactsRoot = t.TempDir()
h.runner.exportSession = export
if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
ready := make(chan struct{})
var id int64
first, second := &atomic.Int32{}, &atomic.Int32{}
h.runner.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
<-ready
if def.Key == "one" {
first.Add(1)
metadata := []string{"v2", "exact-ref"}
if len(options) > 0 && options[0].missingOwner {
metadata = nil
}
if err := s.RunSession(ctx, id, 1, "ses_exact", active, metadata...); err != nil {
return stage.Result{}, err
}
if len(options) > 0 && options[0].waitForCancel {
<-ctx.Done()
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "computed output"}}, nil
}
second.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
d, err := h.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "gate", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
id = d.ID
close(ready)
return h, s, id, first, second
}
func validGateExport(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil {
return stage.SessionExportResult{}, err
}
name := req.FileName
if name == "" {
name = stage.SessionHTMLFile
}
body := []byte("<!doctype html><html>exact session</html>")
if err := os.WriteFile(filepath.Join(req.ArtifactsDir, name), body, 0600); err != nil {
return stage.SessionExportResult{}, err
}
return stage.SessionExportResult{Source: "native_export", Status: "complete", HTML: name, HTMLBytes: int64(len(body)), Messages: 2}, nil
}
func TestSessionExportGateOrdersExecutors(t *testing.T) {
entered, release := make(chan struct{}), make(chan struct{})
h, s, id, first, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if req.SessionID != "ses_exact" || req.Backend != "v2" || req.Reference != "exact-ref" {
t.Errorf("wrong owner: %+v", req)
}
close(entered)
<-release
return validGateExport(ctx, req)
}, false)
<-entered
if second.Load() != 0 {
t.Fatal("next executor ran before verified export")
}
close(release)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
t.Fatalf("handoff failed: %+v", d)
}
}
func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("export offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
t.Fatalf("lost computation/advanced: %+v", d)
}
h.runner.exportSession = validGateExport
job, _ := s.SessionExportJob(ctx, id, 1)
if job == nil {
t.Fatal("missing durable recovery job")
}
if _, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
t.Fatal(err)
}
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ = s.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || len(d.Steps) != 2 || d.Status != "succeeded" {
t.Fatalf("export retry replayed execution: %+v (%d/%d)", d, first.Load(), second.Load())
}
}
func TestSessionExportGateCleanupUnconfirmedDoesNotExportOrAdvance(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, true)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
t.Fatalf("unconfirmed cleanup advanced/exported: %+v", d)
}
}
func TestSessionExportGateCanceledCleanupUnconfirmedDoesNotAdvance(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, true, gateTestOptions{waitForCancel: true})
deadline := time.Now().Add(3 * time.Second)
for {
d, _ := s.GetRun(context.Background(), id)
if len(d.Steps) > 0 && d.Steps[0].AgentActive {
break
}
if time.Now().After(deadline) {
t.Fatal("executor did not acquire session")
}
time.Sleep(time.Millisecond)
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if !d.CancelRequested || !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
t.Fatalf("canceled unconfirmed owner advanced: %+v", d)
}
}
func TestSessionExportGateCanceledExecutionStillReadsExactOwnerAndExports(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if ctx.Err() != nil || req.SessionID != "ses_exact" || req.Reference != "exact-ref" {
t.Errorf("lost bounded uncanceled exact owner: %+v %v", req, ctx.Err())
}
exports.Add(1)
return validGateExport(ctx, req)
}, false, gateTestOptions{waitForCancel: true})
deadline := time.Now().Add(3 * time.Second)
for {
d, _ := s.GetRun(context.Background(), id)
if len(d.Steps) > 0 && d.Steps[0].SessionID != "" {
break
}
if time.Now().After(deadline) {
t.Fatal("executor did not record session")
}
time.Sleep(time.Millisecond)
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "canceled" || second.Load() != 0 || exports.Load() != 1 || d.Steps[0].SessionExport == nil {
t.Fatalf("canceled ownership/export was silently lost: %+v", d)
}
}
func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, false, gateTestOptions{missingOwner: true})
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
t.Fatalf("missing owner did not fail closed: %+v", d)
}
}
func TestSessionExportGateReservesWorkspaceWithoutExecutingAndCanCancel(t *testing.T) {
h, s, id, _, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
d, err := h.runner.getRun(id)
if err != nil || d.Executing {
t.Fatalf("parked export mislabeled executing: %+v %v", d, err)
}
pending, err := s.PendingSessionExportGate(ctx, id, 1)
if err != nil || !pending {
t.Fatalf("quiescent pending visit not exposed: %v %v", pending, err)
}
request := protocol.Message{PipelineID: d.PipelineID, Version: 1, RequestKey: "overlap", Workspace: d.Workspace, Input: map[string]any{"prompt": "another task"}}
if _, err := h.runner.start(request); err == nil {
t.Fatal("parked export released workspace")
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
pending, err = s.PendingSessionExportGate(ctx, id, 1)
if err != nil || pending {
t.Fatalf("canceled gate remains resumable: %v %v", pending, err)
}
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
request.RequestKey = "after-cancel"
if _, err := h.runner.start(request); err != nil {
t.Fatalf("cancel did not release workspace: %v", err)
}
h.runner.wg.Wait()
if second.Load() != 0 {
t.Fatal("canceled gate advanced")
}
}
func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
for _, mode := range []string{"missing", "empty", "partial", "wrong-size", "symlink"} {
t.Run(mode, func(t *testing.T) {
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
result, err := validGateExport(ctx, req)
path := filepath.Join(req.ArtifactsDir, result.HTML)
switch mode {
case "missing":
err = os.Remove(path)
case "empty":
result.Status = "empty"
case "partial":
result.Status = "partial"
case "wrong-size":
result.HTMLBytes++
case "symlink":
if err = os.Remove(path); err == nil {
err = os.Symlink("input.md", path)
}
}
return result, err
}, false)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" {
t.Fatalf("invalid export advanced: %+v", d)
}
})
}
}
func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
if err := s.Close(); err != nil {
t.Fatal(err)
}
reopened, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunner(ctx, reopened)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.artifactsRoot = root
r.exportSession = validGateExport
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
t.Errorf("replayed owner: %s", def.Key)
}
second.Add(1)
body, err := os.ReadFile(def.Turn.Input)
if err != nil || !strings.Contains(string(body), "computed output") {
t.Errorf("lost computed handoff: %q %v", body, err)
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
job, _ := reopened.SessionExportJob(ctx, id, 1)
if _, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
t.Fatal(err)
}
if err := r.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
r.wg.Wait()
if err := r.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
r.wg.Wait()
d, _ := reopened.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
t.Fatalf("restart replayed or lost successor: %+v", d)
}
}