package server import ( "context" "errors" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) type gateTestOptions struct { missingOwner bool waitForCancel bool } func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error), active bool, options ...gateTestOptions) (*runHandler, *pipeline.Store, int64, *atomic.Int32, *atomic.Int32) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) s, err := pipeline.Open(filepath.Join(t.TempDir(), "gate.db")) if err != nil { t.Fatal(err) } h := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, s).(*runHandler) t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() }) h.runner.exportSession = export if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil { t.Fatal(err) } items, _ := s.List(ctx) ready := make(chan struct{}) var id int64 first, second := &atomic.Int32{}, &atomic.Int32{} h.runner.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) { <-ready if def.Key == "one" { first.Add(1) metadata := []string{"v2", "exact-ref"} if len(options) > 0 && options[0].missingOwner { metadata = nil } if err := s.RunSession(ctx, id, 1, "ses_exact", active, metadata...); err != nil { return stage.Result{}, err } if len(options) > 0 && options[0].waitForCancel { <-ctx.Done() return stage.Result{}, ctx.Err() } return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "computed output"}}, nil } second.Add(1) return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil } d, err := h.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "gate", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}}) if err != nil { t.Fatal(err) } id = d.ID close(ready) return h, s, id, first, second } func validGateExport(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil { return stage.SessionExportResult{}, err } name := req.FileName if name == "" { name = stage.SessionHTMLFile } body := []byte("exact session") if err := os.WriteFile(filepath.Join(req.ArtifactsDir, name), body, 0600); err != nil { return stage.SessionExportResult{}, err } return stage.SessionExportResult{Source: "native_export", Status: "complete", HTML: name, HTMLBytes: int64(len(body)), Messages: 2}, nil } func TestSessionExportGateOrdersExecutors(t *testing.T) { entered, release := make(chan struct{}), make(chan struct{}) h, s, id, first, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { if req.SessionID != "ses_exact" || req.Backend != "v2" || req.Reference != "exact-ref" { t.Errorf("wrong owner: %+v", req) } close(entered) <-release return validGateExport(ctx, req) }, false) <-entered if second.Load() != 0 { t.Fatal("next executor ran before verified export") } close(release) h.runner.wg.Wait() d, _ := s.GetRun(context.Background(), id) if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" { t.Fatalf("handoff failed: %+v", d) } } func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) { h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) { return stage.SessionExportResult{}, errors.New("export offline") }, false) h.runner.wg.Wait() ctx := context.Background() d, _ := s.GetRun(ctx, id) if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 { t.Fatalf("lost computation/advanced: %+v", d) } h.runner.exportSession = validGateExport job, _ := s.SessionExportJob(ctx, id, 1) if job == nil { t.Fatal("missing durable recovery job") } if _, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil { t.Fatal(err) } if err := h.runner.resumeExportGates(ctx); err != nil { t.Fatal(err) } h.runner.wg.Wait() if err := h.runner.resumeExportGates(ctx); err != nil { t.Fatal(err) } h.runner.wg.Wait() d, _ = s.GetRun(ctx, id) if first.Load() != 1 || second.Load() != 1 || len(d.Steps) != 2 || d.Status != "succeeded" { t.Fatalf("export retry replayed execution: %+v (%d/%d)", d, first.Load(), second.Load()) } } // TestSessionExportGateAcceptsRepairAboveOldRenderCap keeps the handoff gate's // versioned-repair verification on the same larger bound as normal export. func TestSessionExportGateAcceptsRepairAboveOldRenderCap(t *testing.T) { h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) { return stage.SessionExportResult{}, errors.New("export offline") }, false) h.runner.wg.Wait() ctx := context.Background() if err := s.ConfirmRecoveredSession(ctx, pipeline.SessionOwner{RunID: id, ID: "ses_exact"}); err != nil { t.Fatal(err) } size := int64(2<<20) + 4096 h.runner.exportSession = func(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil { return stage.SessionExportResult{}, err } f, err := os.Create(filepath.Join(req.ArtifactsDir, req.FileName)) if err != nil { return stage.SessionExportResult{}, err } if err := f.Truncate(size); err != nil { f.Close() return stage.SessionExportResult{}, err } if err := f.Close(); err != nil { return stage.SessionExportResult{}, err } return stage.SessionExportResult{Source: "native_export", HTML: req.FileName, HTMLBytes: size, Messages: 2}, nil } if _, err := h.runner.repairSessionExport(ctx, id, 1); err != nil { t.Fatal(err) } // The worker promotes a persisted complete repair before claiming a lease. job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Second), time.Minute) if err != nil { t.Fatal(err) } if job != nil { t.Fatalf("complete repair still claimed as a recovery job: %+v", job) } if err := h.runner.resumeExportGates(ctx); err != nil { t.Fatal(err) } h.runner.wg.Wait() d, _ := s.GetRun(ctx, id) if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 { t.Fatalf("large repair did not advance the gate: %+v (%d/%d)", d, first.Load(), second.Load()) } } func TestSessionExportGateCleanupUnconfirmedDoesNotExportOrAdvance(t *testing.T) { var exports atomic.Int32 h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { exports.Add(1) return validGateExport(ctx, req) }, true) h.runner.wg.Wait() d, _ := s.GetRun(context.Background(), id) if !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 { t.Fatalf("unconfirmed cleanup advanced/exported: %+v", d) } } func TestSessionExportGateCanceledCleanupUnconfirmedDoesNotAdvance(t *testing.T) { var exports atomic.Int32 h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { exports.Add(1) return validGateExport(ctx, req) }, true, gateTestOptions{waitForCancel: true}) deadline := time.Now().Add(3 * time.Second) for { d, _ := s.GetRun(context.Background(), id) if len(d.Steps) > 0 && d.Steps[0].AgentActive { break } if time.Now().After(deadline) { t.Fatal("executor did not acquire session") } time.Sleep(time.Millisecond) } if err := h.runner.cancel(id); err != nil { t.Fatal(err) } h.runner.wg.Wait() d, _ := s.GetRun(context.Background(), id) if !d.CancelRequested || !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 { t.Fatalf("canceled unconfirmed owner advanced: %+v", d) } } func TestSessionExportGateCanceledExecutionStillReadsExactOwnerAndExports(t *testing.T) { var exports atomic.Int32 h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { if ctx.Err() != nil || req.SessionID != "ses_exact" || req.Reference != "exact-ref" { t.Errorf("lost bounded uncanceled exact owner: %+v %v", req, ctx.Err()) } exports.Add(1) return validGateExport(ctx, req) }, false, gateTestOptions{waitForCancel: true}) deadline := time.Now().Add(3 * time.Second) for { d, _ := s.GetRun(context.Background(), id) if len(d.Steps) > 0 && d.Steps[0].SessionID != "" { break } if time.Now().After(deadline) { t.Fatal("executor did not record session") } time.Sleep(time.Millisecond) } if err := h.runner.cancel(id); err != nil { t.Fatal(err) } h.runner.wg.Wait() d, _ := s.GetRun(context.Background(), id) if d.Status != "canceled" || second.Load() != 0 || exports.Load() != 1 || d.Steps[0].SessionExport == nil { t.Fatalf("canceled ownership/export was silently lost: %+v", d) } } func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) { var exports atomic.Int32 h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { exports.Add(1) return validGateExport(ctx, req) }, false, gateTestOptions{missingOwner: true}) h.runner.wg.Wait() d, _ := s.GetRun(context.Background(), id) if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" { t.Fatalf("missing owner did not fail closed: %+v", d) } } func TestSessionExportGateReservesWorkspaceWithoutExecutingAndCanCancel(t *testing.T) { h, s, id, _, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) { return stage.SessionExportResult{}, errors.New("offline") }, false) h.runner.wg.Wait() ctx := context.Background() d, err := h.runner.getRun(id) if err != nil || d.Executing { t.Fatalf("parked export mislabeled executing: %+v %v", d, err) } pending, err := s.PendingSessionExportGate(ctx, id, 1) if err != nil || !pending { t.Fatalf("quiescent pending visit not exposed: %v %v", pending, err) } request := protocol.Message{PipelineID: d.PipelineID, Version: 1, RequestKey: "overlap", Workspace: d.Workspace, Input: map[string]any{"prompt": "another task"}} if _, err := h.runner.start(request); err == nil { t.Fatal("parked export released workspace") } if err := h.runner.cancel(id); err != nil { t.Fatal(err) } pending, err = s.PendingSessionExportGate(ctx, id, 1) if err != nil || pending { t.Fatalf("canceled gate remains resumable: %v %v", pending, err) } h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } request.RequestKey = "after-cancel" if _, err := h.runner.start(request); err != nil { t.Fatalf("cancel did not release workspace: %v", err) } h.runner.wg.Wait() if second.Load() != 0 { t.Fatal("canceled gate advanced") } } func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) { for _, mode := range []string{"missing", "empty", "partial", "wrong-size", "symlink"} { t.Run(mode, func(t *testing.T) { h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { result, err := validGateExport(ctx, req) path := filepath.Join(req.ArtifactsDir, result.HTML) switch mode { case "missing": err = os.Remove(path) case "empty": result.Status = "empty" case "partial": result.Status = "partial" case "wrong-size": result.HTMLBytes++ case "symlink": if err = os.Remove(path); err == nil { err = os.Symlink("input.md", path) } } return result, err }, false) h.runner.wg.Wait() d, _ := s.GetRun(context.Background(), id) if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" { t.Fatalf("invalid export advanced: %+v", d) } }) } } func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) { h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) { return stage.SessionExportResult{}, errors.New("offline") }, false) h.runner.wg.Wait() path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot if err := s.Close(); err != nil { t.Fatal(err) } reopened, err := pipeline.Open(path) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) r, err := newRunnerAt(ctx, reopened, root) if err != nil { t.Fatal(err) } defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }() r.exportSession = validGateExport r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { if def.Key != "two" { t.Errorf("replayed owner: %s", def.Key) } second.Add(1) body, err := os.ReadFile(def.Turn.Input) if err != nil || !strings.Contains(string(body), "computed output") { t.Errorf("lost computed handoff: %q %v", body, err) } return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil } job, _ := reopened.SessionExportJob(ctx, id, 1) if _, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil { t.Fatal(err) } if err := r.resumeExportGates(ctx); err != nil { t.Fatal(err) } r.wg.Wait() if err := r.resumeExportGates(ctx); err != nil { t.Fatal(err) } r.wg.Wait() d, _ := reopened.GetRun(ctx, id) if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" { t.Fatalf("restart replayed or lost successor: %+v", d) } }