340 lines
12 KiB
Go
340 lines
12 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
type gateTestOptions struct {
|
|
missingOwner bool
|
|
waitForCancel bool
|
|
}
|
|
|
|
func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error), active bool, options ...gateTestOptions) (*runHandler, *pipeline.Store, int64, *atomic.Int32, *atomic.Int32) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "gate.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := Handler(ctx, s).(*runHandler)
|
|
t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() })
|
|
h.runner.artifactsRoot = t.TempDir()
|
|
h.runner.exportSession = export
|
|
if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
ready := make(chan struct{})
|
|
var id int64
|
|
first, second := &atomic.Int32{}, &atomic.Int32{}
|
|
h.runner.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
|
<-ready
|
|
if def.Key == "one" {
|
|
first.Add(1)
|
|
metadata := []string{"v2", "exact-ref"}
|
|
if len(options) > 0 && options[0].missingOwner {
|
|
metadata = nil
|
|
}
|
|
if err := s.RunSession(ctx, id, 1, "ses_exact", active, metadata...); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
if len(options) > 0 && options[0].waitForCancel {
|
|
<-ctx.Done()
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "computed output"}}, nil
|
|
}
|
|
second.Add(1)
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
|
}
|
|
d, err := h.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "gate", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id = d.ID
|
|
close(ready)
|
|
return h, s, id, first, second
|
|
}
|
|
|
|
func validGateExport(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
name := req.FileName
|
|
if name == "" {
|
|
name = stage.SessionHTMLFile
|
|
}
|
|
body := []byte("<!doctype html><html>exact session</html>")
|
|
if err := os.WriteFile(filepath.Join(req.ArtifactsDir, name), body, 0600); err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
return stage.SessionExportResult{Source: "native_export", Status: "complete", HTML: name, HTMLBytes: int64(len(body)), Messages: 2}, nil
|
|
}
|
|
|
|
func TestSessionExportGateOrdersExecutors(t *testing.T) {
|
|
entered, release := make(chan struct{}), make(chan struct{})
|
|
h, s, id, first, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if req.SessionID != "ses_exact" || req.Backend != "v2" || req.Reference != "exact-ref" {
|
|
t.Errorf("wrong owner: %+v", req)
|
|
}
|
|
close(entered)
|
|
<-release
|
|
return validGateExport(ctx, req)
|
|
}, false)
|
|
<-entered
|
|
if second.Load() != 0 {
|
|
t.Fatal("next executor ran before verified export")
|
|
}
|
|
close(release)
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
|
|
t.Fatalf("handoff failed: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("export offline")
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
ctx := context.Background()
|
|
d, _ := s.GetRun(ctx, id)
|
|
if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
|
|
t.Fatalf("lost computation/advanced: %+v", d)
|
|
}
|
|
h.runner.exportSession = validGateExport
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job == nil {
|
|
t.Fatal("missing durable recovery job")
|
|
}
|
|
if _, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := h.runner.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
if err := h.runner.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ = s.GetRun(ctx, id)
|
|
if first.Load() != 1 || second.Load() != 1 || len(d.Steps) != 2 || d.Status != "succeeded" {
|
|
t.Fatalf("export retry replayed execution: %+v (%d/%d)", d, first.Load(), second.Load())
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateCleanupUnconfirmedDoesNotExportOrAdvance(t *testing.T) {
|
|
var exports atomic.Int32
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
exports.Add(1)
|
|
return validGateExport(ctx, req)
|
|
}, true)
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
|
|
t.Fatalf("unconfirmed cleanup advanced/exported: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateCanceledCleanupUnconfirmedDoesNotAdvance(t *testing.T) {
|
|
var exports atomic.Int32
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
exports.Add(1)
|
|
return validGateExport(ctx, req)
|
|
}, true, gateTestOptions{waitForCancel: true})
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for {
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if len(d.Steps) > 0 && d.Steps[0].AgentActive {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("executor did not acquire session")
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if err := h.runner.cancel(id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if !d.CancelRequested || !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
|
|
t.Fatalf("canceled unconfirmed owner advanced: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateCanceledExecutionStillReadsExactOwnerAndExports(t *testing.T) {
|
|
var exports atomic.Int32
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if ctx.Err() != nil || req.SessionID != "ses_exact" || req.Reference != "exact-ref" {
|
|
t.Errorf("lost bounded uncanceled exact owner: %+v %v", req, ctx.Err())
|
|
}
|
|
exports.Add(1)
|
|
return validGateExport(ctx, req)
|
|
}, false, gateTestOptions{waitForCancel: true})
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for {
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if len(d.Steps) > 0 && d.Steps[0].SessionID != "" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("executor did not record session")
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if err := h.runner.cancel(id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if d.Status != "canceled" || second.Load() != 0 || exports.Load() != 1 || d.Steps[0].SessionExport == nil {
|
|
t.Fatalf("canceled ownership/export was silently lost: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
|
|
var exports atomic.Int32
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
exports.Add(1)
|
|
return validGateExport(ctx, req)
|
|
}, false, gateTestOptions{missingOwner: true})
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
|
|
t.Fatalf("missing owner did not fail closed: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateReservesWorkspaceWithoutExecutingAndCanCancel(t *testing.T) {
|
|
h, s, id, _, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("offline")
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
ctx := context.Background()
|
|
d, err := h.runner.getRun(id)
|
|
if err != nil || d.Executing {
|
|
t.Fatalf("parked export mislabeled executing: %+v %v", d, err)
|
|
}
|
|
pending, err := s.PendingSessionExportGate(ctx, id, 1)
|
|
if err != nil || !pending {
|
|
t.Fatalf("quiescent pending visit not exposed: %v %v", pending, err)
|
|
}
|
|
request := protocol.Message{PipelineID: d.PipelineID, Version: 1, RequestKey: "overlap", Workspace: d.Workspace, Input: map[string]any{"prompt": "another task"}}
|
|
if _, err := h.runner.start(request); err == nil {
|
|
t.Fatal("parked export released workspace")
|
|
}
|
|
if err := h.runner.cancel(id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pending, err = s.PendingSessionExportGate(ctx, id, 1)
|
|
if err != nil || pending {
|
|
t.Fatalf("canceled gate remains resumable: %v %v", pending, err)
|
|
}
|
|
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
request.RequestKey = "after-cancel"
|
|
if _, err := h.runner.start(request); err != nil {
|
|
t.Fatalf("cancel did not release workspace: %v", err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
if second.Load() != 0 {
|
|
t.Fatal("canceled gate advanced")
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
|
|
for _, mode := range []string{"missing", "empty", "partial", "wrong-size", "symlink"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
result, err := validGateExport(ctx, req)
|
|
path := filepath.Join(req.ArtifactsDir, result.HTML)
|
|
switch mode {
|
|
case "missing":
|
|
err = os.Remove(path)
|
|
case "empty":
|
|
result.Status = "empty"
|
|
case "partial":
|
|
result.Status = "partial"
|
|
case "wrong-size":
|
|
result.HTMLBytes++
|
|
case "symlink":
|
|
if err = os.Remove(path); err == nil {
|
|
err = os.Symlink("input.md", path)
|
|
}
|
|
}
|
|
return result, err
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" {
|
|
t.Fatalf("invalid export advanced: %+v", d)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("offline")
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reopened, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
r, err := newRunner(ctx, reopened)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
|
r.artifactsRoot = root
|
|
r.exportSession = validGateExport
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
if def.Key != "two" {
|
|
t.Errorf("replayed owner: %s", def.Key)
|
|
}
|
|
second.Add(1)
|
|
body, err := os.ReadFile(def.Turn.Input)
|
|
if err != nil || !strings.Contains(string(body), "computed output") {
|
|
t.Errorf("lost computed handoff: %q %v", body, err)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
|
}
|
|
job, _ := reopened.SessionExportJob(ctx, id, 1)
|
|
if _, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if err := r.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
d, _ := reopened.GetRun(ctx, id)
|
|
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
|
|
t.Fatalf("restart replayed or lost successor: %+v", d)
|
|
}
|
|
}
|