22801a9b06
Six computed Flutter runs stopped at the export handoff gate because the rendered session.html cap was 2 MiB while their complete sessions (well inside the 8 MiB source read bound) rendered between 2.25 and 4.28 MiB once html escaping and the readable projection duplicated native text. Raise SessionExportFileLimit to 8 MiB, the server's existing per-artifact file bound, so a complete export is still listed and served. Normal export verification, versioned repair and the handoff gate keep using the single shared constant. The source read bound is unchanged, and an over-cap render is still visibly truncated and recorded partial. Add focused regressions for a complete export above the old cap, an over-new-cap file rejected as partial, repair above the old cap leaving original evidence untouched, the gate accepting the larger repair, and the constant staying inside the artifact file bound. Update the artifact, handoff-gate and operations docs and record the diagnosis in docs/notes/session-export-render-cap.md.
390 lines
14 KiB
Go
390 lines
14 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
type gateTestOptions struct {
|
|
missingOwner bool
|
|
waitForCancel bool
|
|
}
|
|
|
|
func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error), active bool, options ...gateTestOptions) (*runHandler, *pipeline.Store, int64, *atomic.Int32, *atomic.Int32) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
s, err := pipeline.Open(filepath.Join(t.TempDir(), "gate.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := Handler(ctx, s).(*runHandler)
|
|
t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() })
|
|
h.runner.artifactsRoot = t.TempDir()
|
|
h.runner.exportSession = export
|
|
if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := s.List(ctx)
|
|
ready := make(chan struct{})
|
|
var id int64
|
|
first, second := &atomic.Int32{}, &atomic.Int32{}
|
|
h.runner.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
|
<-ready
|
|
if def.Key == "one" {
|
|
first.Add(1)
|
|
metadata := []string{"v2", "exact-ref"}
|
|
if len(options) > 0 && options[0].missingOwner {
|
|
metadata = nil
|
|
}
|
|
if err := s.RunSession(ctx, id, 1, "ses_exact", active, metadata...); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
if len(options) > 0 && options[0].waitForCancel {
|
|
<-ctx.Done()
|
|
return stage.Result{}, ctx.Err()
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "computed output"}}, nil
|
|
}
|
|
second.Add(1)
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
|
}
|
|
d, err := h.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "gate", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id = d.ID
|
|
close(ready)
|
|
return h, s, id, first, second
|
|
}
|
|
|
|
func validGateExport(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
name := req.FileName
|
|
if name == "" {
|
|
name = stage.SessionHTMLFile
|
|
}
|
|
body := []byte("<!doctype html><html>exact session</html>")
|
|
if err := os.WriteFile(filepath.Join(req.ArtifactsDir, name), body, 0600); err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
return stage.SessionExportResult{Source: "native_export", Status: "complete", HTML: name, HTMLBytes: int64(len(body)), Messages: 2}, nil
|
|
}
|
|
|
|
func TestSessionExportGateOrdersExecutors(t *testing.T) {
|
|
entered, release := make(chan struct{}), make(chan struct{})
|
|
h, s, id, first, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if req.SessionID != "ses_exact" || req.Backend != "v2" || req.Reference != "exact-ref" {
|
|
t.Errorf("wrong owner: %+v", req)
|
|
}
|
|
close(entered)
|
|
<-release
|
|
return validGateExport(ctx, req)
|
|
}, false)
|
|
<-entered
|
|
if second.Load() != 0 {
|
|
t.Fatal("next executor ran before verified export")
|
|
}
|
|
close(release)
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
|
|
t.Fatalf("handoff failed: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("export offline")
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
ctx := context.Background()
|
|
d, _ := s.GetRun(ctx, id)
|
|
if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
|
|
t.Fatalf("lost computation/advanced: %+v", d)
|
|
}
|
|
h.runner.exportSession = validGateExport
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job == nil {
|
|
t.Fatal("missing durable recovery job")
|
|
}
|
|
if _, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := h.runner.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
if err := h.runner.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ = s.GetRun(ctx, id)
|
|
if first.Load() != 1 || second.Load() != 1 || len(d.Steps) != 2 || d.Status != "succeeded" {
|
|
t.Fatalf("export retry replayed execution: %+v (%d/%d)", d, first.Load(), second.Load())
|
|
}
|
|
}
|
|
|
|
// TestSessionExportGateAcceptsRepairAboveOldRenderCap keeps the handoff gate's
|
|
// versioned-repair verification on the same larger bound as normal export.
|
|
func TestSessionExportGateAcceptsRepairAboveOldRenderCap(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("export offline")
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
ctx := context.Background()
|
|
if err := s.ConfirmRecoveredSession(ctx, pipeline.SessionOwner{RunID: id, ID: "ses_exact"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
size := int64(2<<20) + 4096
|
|
h.runner.exportSession = func(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
f, err := os.Create(filepath.Join(req.ArtifactsDir, req.FileName))
|
|
if err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
if err := f.Truncate(size); err != nil {
|
|
f.Close()
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
return stage.SessionExportResult{}, err
|
|
}
|
|
return stage.SessionExportResult{Source: "native_export", HTML: req.FileName, HTMLBytes: size, Messages: 2}, nil
|
|
}
|
|
if _, err := h.runner.repairSessionExport(ctx, id, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The worker promotes a persisted complete repair before claiming a lease.
|
|
job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Second), time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if job != nil {
|
|
t.Fatalf("complete repair still claimed as a recovery job: %+v", job)
|
|
}
|
|
if err := h.runner.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(ctx, id)
|
|
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 {
|
|
t.Fatalf("large repair did not advance the gate: %+v (%d/%d)", d, first.Load(), second.Load())
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateCleanupUnconfirmedDoesNotExportOrAdvance(t *testing.T) {
|
|
var exports atomic.Int32
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
exports.Add(1)
|
|
return validGateExport(ctx, req)
|
|
}, true)
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
|
|
t.Fatalf("unconfirmed cleanup advanced/exported: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateCanceledCleanupUnconfirmedDoesNotAdvance(t *testing.T) {
|
|
var exports atomic.Int32
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
exports.Add(1)
|
|
return validGateExport(ctx, req)
|
|
}, true, gateTestOptions{waitForCancel: true})
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for {
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if len(d.Steps) > 0 && d.Steps[0].AgentActive {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("executor did not acquire session")
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if err := h.runner.cancel(id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if !d.CancelRequested || !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
|
|
t.Fatalf("canceled unconfirmed owner advanced: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateCanceledExecutionStillReadsExactOwnerAndExports(t *testing.T) {
|
|
var exports atomic.Int32
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
if ctx.Err() != nil || req.SessionID != "ses_exact" || req.Reference != "exact-ref" {
|
|
t.Errorf("lost bounded uncanceled exact owner: %+v %v", req, ctx.Err())
|
|
}
|
|
exports.Add(1)
|
|
return validGateExport(ctx, req)
|
|
}, false, gateTestOptions{waitForCancel: true})
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for {
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if len(d.Steps) > 0 && d.Steps[0].SessionID != "" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("executor did not record session")
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if err := h.runner.cancel(id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if d.Status != "canceled" || second.Load() != 0 || exports.Load() != 1 || d.Steps[0].SessionExport == nil {
|
|
t.Fatalf("canceled ownership/export was silently lost: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
|
|
var exports atomic.Int32
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
exports.Add(1)
|
|
return validGateExport(ctx, req)
|
|
}, false, gateTestOptions{missingOwner: true})
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
|
|
t.Fatalf("missing owner did not fail closed: %+v", d)
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateReservesWorkspaceWithoutExecutingAndCanCancel(t *testing.T) {
|
|
h, s, id, _, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("offline")
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
ctx := context.Background()
|
|
d, err := h.runner.getRun(id)
|
|
if err != nil || d.Executing {
|
|
t.Fatalf("parked export mislabeled executing: %+v %v", d, err)
|
|
}
|
|
pending, err := s.PendingSessionExportGate(ctx, id, 1)
|
|
if err != nil || !pending {
|
|
t.Fatalf("quiescent pending visit not exposed: %v %v", pending, err)
|
|
}
|
|
request := protocol.Message{PipelineID: d.PipelineID, Version: 1, RequestKey: "overlap", Workspace: d.Workspace, Input: map[string]any{"prompt": "another task"}}
|
|
if _, err := h.runner.start(request); err == nil {
|
|
t.Fatal("parked export released workspace")
|
|
}
|
|
if err := h.runner.cancel(id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pending, err = s.PendingSessionExportGate(ctx, id, 1)
|
|
if err != nil || pending {
|
|
t.Fatalf("canceled gate remains resumable: %v %v", pending, err)
|
|
}
|
|
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
request.RequestKey = "after-cancel"
|
|
if _, err := h.runner.start(request); err != nil {
|
|
t.Fatalf("cancel did not release workspace: %v", err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
if second.Load() != 0 {
|
|
t.Fatal("canceled gate advanced")
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
|
|
for _, mode := range []string{"missing", "empty", "partial", "wrong-size", "symlink"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
result, err := validGateExport(ctx, req)
|
|
path := filepath.Join(req.ArtifactsDir, result.HTML)
|
|
switch mode {
|
|
case "missing":
|
|
err = os.Remove(path)
|
|
case "empty":
|
|
result.Status = "empty"
|
|
case "partial":
|
|
result.Status = "partial"
|
|
case "wrong-size":
|
|
result.HTMLBytes++
|
|
case "symlink":
|
|
if err = os.Remove(path); err == nil {
|
|
err = os.Symlink("input.md", path)
|
|
}
|
|
}
|
|
return result, err
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
d, _ := s.GetRun(context.Background(), id)
|
|
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" {
|
|
t.Fatalf("invalid export advanced: %+v", d)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
return stage.SessionExportResult{}, errors.New("offline")
|
|
}, false)
|
|
h.runner.wg.Wait()
|
|
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reopened, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
r, err := newRunner(ctx, reopened)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
|
r.artifactsRoot = root
|
|
r.exportSession = validGateExport
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
if def.Key != "two" {
|
|
t.Errorf("replayed owner: %s", def.Key)
|
|
}
|
|
second.Add(1)
|
|
body, err := os.ReadFile(def.Turn.Input)
|
|
if err != nil || !strings.Contains(string(body), "computed output") {
|
|
t.Errorf("lost computed handoff: %q %v", body, err)
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
|
}
|
|
job, _ := reopened.SessionExportJob(ctx, id, 1)
|
|
if _, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if err := r.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
d, _ := reopened.GetRun(ctx, id)
|
|
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
|
|
t.Fatalf("restart replayed or lost successor: %+v", d)
|
|
}
|
|
}
|