runner: gate stage handoff on settled verified session export

This commit is contained in:
2026-10-20 20:54:51 +01:00
parent 3d35e8a863
commit f87e1ed488
18 changed files with 791 additions and 26 deletions
+1
View File
@@ -417,6 +417,7 @@ func waitTerminal(t *testing.T, r *runner, id int64) *protocol.RunDetail {
// deterministically against B's active session.
func TestOverlappingRunsDoNotFailQuiescence(t *testing.T) {
_, r, pipelineID := agentStore(t)
r.exportSession = validGateExport // Quiescence, not export transport, is under test.
a, b := t.TempDir(), t.TempDir()
type agentRun struct {
+3
View File
@@ -95,6 +95,7 @@ esac
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait() }()
r.exportSession = validGateExport // Export is a separate mandatory handoff gate.
workspace, err := filepath.EvalSymlinks(dir)
if err != nil {
t.Fatal(err)
@@ -455,6 +456,7 @@ esac
t.Fatal(err)
}
handler := Handler(ctx, store)
handler.(*runHandler).runner.exportSession = validGateExport
defer func() { cancel(); WaitRuns(handler) }()
items, err := store.List(ctx)
if err != nil {
@@ -566,6 +568,7 @@ esac
toolsBlock := func(tools string) string {
return "# stage worker\n\n```output\nResult.\n```\n\n```prompt\nDo the work.\n```\n\n```tools\n" + tools + "\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=1\ntimeout=30s\n```\n"
}
r.exportSession = validGateExport
runStage := func(name, tools string) (status, sessions, diagnostic string) {
t.Helper()
workspace := t.TempDir()
+88 -12
View File
@@ -384,11 +384,26 @@ func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
// checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run
// reserves an overlapping resolved workspace. Callers must hold r.mu.
func (r *runner) checkWorkspaceFree(workspace string) error {
return r.checkWorkspaceFreeExcept(workspace, 0)
}
func (r *runner) checkWorkspaceFreeExcept(workspace string, ownGate int64) error {
for id, entry := range r.active {
if workspacesOverlap(entry.workspace, workspace) {
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
}
}
check, cancel := context.WithTimeout(context.WithoutCancel(r.ctx), 5*time.Second)
defer cancel()
reservations, err := r.store.SessionExportReservations(check)
if err != nil {
return fmt.Errorf("read pending export workspace reservations: %w", err)
}
for id, reserved := range reservations {
if id != ownGate && workspacesOverlap(reserved, workspace) {
return fmt.Errorf("run %d is awaiting its session export; workspace %s remains reserved", id, reserved)
}
}
return nil
}
@@ -694,6 +709,11 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
} else if existing > 0 {
return r.store.GetRun(r.ctx, existing)
}
if blocked, err := r.store.SessionExportBlocked(r.ctx, original.ID); err != nil {
return nil, err
} else if blocked {
return nil, fmt.Errorf("run %d has a retained computation awaiting export-only recovery; model retry refused", original.ID)
}
if request.RetryVisit < 0 || (request.RetryVisit > 0 && request.RetryStage == "") {
return nil, fmt.Errorf("retry visit requires a positive --stage visit selection")
}
@@ -947,6 +967,7 @@ type runState struct {
// retryOverride is applied once, after the selected stage's memory bindings
// have been resolved. It is intentionally not part of run input or memory.
retryOverride stage.Values
priorBudgets map[string]stage.BudgetSnapshot
}
func newRunState(key string, input stage.Values) *runState {
@@ -963,7 +984,18 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
state.memory["input"] = memoryValue{output: original}
}
publicSeen := map[string]bool{}
state.priorBudgets = map[string]stage.BudgetSnapshot{}
for _, step := range detail.Steps {
// The runtime stamps this cumulative snapshot on every executed model
// visit. Reusing it preserves per-stage limits across export recovery.
var audit struct {
Limits struct {
Budget stage.BudgetSnapshot `json:"budget"`
} `json:"stage_limits"`
}
if json.Unmarshal([]byte(step.Transcript), &audit) == nil {
state.priorBudgets[step.Stage] = audit.Limits.Budget
}
state.visits[step.Stage]++
if step.Sequence >= state.nextSequence {
state.nextSequence = step.Sequence + 1
@@ -1715,6 +1747,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
modelConfig = spec.Config
if budgets[key] == nil {
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
if used, ok := state.priorBudgets[key]; ok {
budgets[key].RestoreUsage(used)
}
}
// The budget is shared across every visit to this stage key, so capture
// its cumulative usage before the visit and record only this visit's
@@ -1808,15 +1843,32 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
cancel()
}
// Export the stage's session into its own artifacts directory before the
// bounded scan, so the existing listing and serving paths find the files
// with no special casing. This is bounded, best-effort evidence: it never
// waits for session idleness and a failure never changes the stage's
// status, outcome, target or result validation.
// Only settled, exact recorded ownership can satisfy the handoff gate.
// A canceled execution must not cancel this ownership read silently.
var sessionExport *protocol.SessionExport
exportRequired := artifactsDir != "" && r.execute == nil
if artifactsDir != "" {
if owner, ownerErr := r.store.RunSessionOwner(ctx, id, sequence); ownerErr == nil && owner.ID != "" {
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
ownerCtx, ownerCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
owner, ownerErr := r.store.RepairSessionOwner(ownerCtx, id, sequence)
if ownerErr != nil && r.execute != nil {
// Only genuinely unowned fake executors are exempt. A recorded
// session with absent ownership metadata must still fail closed.
if recorded, readErr := r.store.RunSessionOwner(ownerCtx, id, sequence); readErr != nil || recorded.ID != "" {
exportRequired = true
if readErr == nil {
stageSession = recorded.ID
}
}
}
ownerCancel()
if ownerErr == nil && owner.ID != "" {
exportRequired = true // Owned test executors exercise the real gate too.
stageSession = owner.ID
if !errors.Is(err, stage.ErrStageCleanup) {
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
}
} else if exportRequired {
sessionExport = &protocol.SessionExport{Status: "failed", Source: "native_export", Diagnostic: "exact settled session ownership unavailable; handoff refused"}
}
}
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir, visitTurn.PublicDir)
@@ -1915,6 +1967,28 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
if panicked {
step.Target = protocol.FailureEnd
}
_, _, _, failedTerminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
if exportRequired && !completeSessionExport(sessionExport) && !(err != nil && failedTerminal && step.Target != protocol.SuccessEnd) {
// Preserve the real computation, files, outcome and target. In
// particular an export failure is not an agent failure/retry request.
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
saveErr := r.store.FinishStepExportBlocked(save, id, step)
cancel()
if saveErr != nil {
finish("failed", "error", saveErr.Error())
return
}
diagnostic := "Session export gate blocked; computed visit retained; export-only recovery pending"
if sessionExport != nil && sessionExport.Diagnostic != "" {
diagnostic += ": " + sessionExport.Diagnostic
}
if err != nil {
diagnostic = err.Error() + "; " + diagnostic
}
// No finishStage, memory publication, recordTurn or next executor.
r.finishRunState(id, name, workspace, "failed", "error", diagnostic, state.completed, sequence, key, stage.Values(step.Output))
return
}
// Generate the one-line verdict reason once, at completion. It is
// best-effort: a summariser failure or timeout returns NoSummary and must
// never fail the stage or the run, so every error is swallowed. A failed
@@ -1985,14 +2059,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
}
// exportStageSession writes the finished stage's session into its own artifacts
// directory and returns the evidence for the step. A non-OpenCode harness (pi)
// has no native export path, so it records that limitation and writes no files.
// Any export error is recorded as a sanitized diagnostic and never returned to
// the stage: the export is read-only, bounded and best-effort.
// directory and returns verified evidence. The caller enforces the handoff gate;
// an export failure never replaces the computation's result.
func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport {
evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend, Directory: owner.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339)}
defer classifySessionExport(evidence)
if owner.Backend != "v1" && owner.Backend != "v2" {
if owner.Backend != "v1" && owner.Backend != "v2" && !(owner.Backend == "pi" && owner.Reference != "") {
evidence.Source = "unavailable"
evidence.Diagnostic = "session export unavailable: harness has no native export path"
return evidence
@@ -2008,6 +2080,7 @@ func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionO
result, err := export(exportCtx, stage.SessionExportRequest{
SessionID: owner.ID,
Backend: owner.Backend,
Reference: owner.Reference,
Directory: owner.Directory,
ArtifactsDir: artifactsDir,
RunID: runID,
@@ -2029,6 +2102,9 @@ func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionO
evidence.Messages = result.Messages
evidence.Truncated = result.Truncated
evidence.Diagnostic = result.Diagnostic
if result.Status != "" && result.Status != "complete" && evidence.Diagnostic == "" {
evidence.Diagnostic = "native session export reported " + result.Status
}
verifySessionExportFile(evidence, artifactsDir)
return evidence
}
+99
View File
@@ -0,0 +1,99 @@
package server
import (
"context"
"os"
"path/filepath"
"strconv"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func completeSessionExport(e *protocol.SessionExport) bool {
return e != nil && e.Status == "complete" && e.SessionID != "" && e.Messages > 0 && e.HTML != "" && e.HTMLBytes > 0 && !e.Truncated && e.Diagnostic == ""
}
// resumeExportGates is also run when no job is due: a manual export repair or
// a crash after persisting repair evidence may have made a handoff eligible.
func (r *runner) resumeExportGates(ctx context.Context) error {
ready, err := r.store.ReadySessionExportGates(ctx)
if err != nil {
return err
}
for id, sequence := range ready {
r.mu.Lock()
if _, active := r.active[id]; active {
r.mu.Unlock()
continue
}
d, err := r.store.GetRun(ctx, id)
if err != nil {
r.mu.Unlock()
return err
}
if err := r.checkWorkspaceFreeExcept(d.Workspace, id); err != nil {
r.mu.Unlock()
continue
}
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
if err != nil {
r.mu.Unlock()
continue
}
job, err := r.store.SessionExportJob(ctx, id, sequence)
if err != nil || job == nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
r.mu.Unlock()
continue
}
var saved *protocol.RunStep
for i := range d.Steps {
if d.Steps[i].Sequence == sequence {
saved = &d.Steps[i]
break
}
}
verified := false
if saved != nil {
for _, e := range saved.SessionExportRepairs {
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
continue
}
// New repairs stay private in the original visit. Historical
// public repairs remain valid, but are never rewritten there.
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
if strings.HasPrefix(e.HTML, "public/") {
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
continue
}
if filepath.Clean(e.HTML) != e.HTML {
continue
}
root, openErr := os.OpenRoot(directory)
if openErr != nil {
continue
}
info, statErr := root.Lstat(e.HTML)
root.Close()
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
verified = true
break
}
}
}
if !verified {
r.mu.Unlock()
continue
}
if err := r.store.ResumeSessionExportGate(ctx, id, sequence); err != nil {
r.mu.Unlock()
continue
}
step := *saved
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
r.mu.Unlock()
}
return nil
}
+339
View File
@@ -0,0 +1,339 @@
package server
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
type gateTestOptions struct {
missingOwner bool
waitForCancel bool
}
func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error), active bool, options ...gateTestOptions) (*runHandler, *pipeline.Store, int64, *atomic.Int32, *atomic.Int32) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
s, err := pipeline.Open(filepath.Join(t.TempDir(), "gate.db"))
if err != nil {
t.Fatal(err)
}
h := Handler(ctx, s).(*runHandler)
t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() })
h.runner.artifactsRoot = t.TempDir()
h.runner.exportSession = export
if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
ready := make(chan struct{})
var id int64
first, second := &atomic.Int32{}, &atomic.Int32{}
h.runner.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
<-ready
if def.Key == "one" {
first.Add(1)
metadata := []string{"v2", "exact-ref"}
if len(options) > 0 && options[0].missingOwner {
metadata = nil
}
if err := s.RunSession(ctx, id, 1, "ses_exact", active, metadata...); err != nil {
return stage.Result{}, err
}
if len(options) > 0 && options[0].waitForCancel {
<-ctx.Done()
return stage.Result{}, ctx.Err()
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "computed output"}}, nil
}
second.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
d, err := h.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "gate", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
id = d.ID
close(ready)
return h, s, id, first, second
}
func validGateExport(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil {
return stage.SessionExportResult{}, err
}
name := req.FileName
if name == "" {
name = stage.SessionHTMLFile
}
body := []byte("<!doctype html><html>exact session</html>")
if err := os.WriteFile(filepath.Join(req.ArtifactsDir, name), body, 0600); err != nil {
return stage.SessionExportResult{}, err
}
return stage.SessionExportResult{Source: "native_export", Status: "complete", HTML: name, HTMLBytes: int64(len(body)), Messages: 2}, nil
}
func TestSessionExportGateOrdersExecutors(t *testing.T) {
entered, release := make(chan struct{}), make(chan struct{})
h, s, id, first, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if req.SessionID != "ses_exact" || req.Backend != "v2" || req.Reference != "exact-ref" {
t.Errorf("wrong owner: %+v", req)
}
close(entered)
<-release
return validGateExport(ctx, req)
}, false)
<-entered
if second.Load() != 0 {
t.Fatal("next executor ran before verified export")
}
close(release)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
t.Fatalf("handoff failed: %+v", d)
}
}
func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("export offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "failed" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
t.Fatalf("lost computation/advanced: %+v", d)
}
h.runner.exportSession = validGateExport
job, _ := s.SessionExportJob(ctx, id, 1)
if job == nil {
t.Fatal("missing durable recovery job")
}
if _, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
t.Fatal(err)
}
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ = s.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || len(d.Steps) != 2 || d.Status != "succeeded" {
t.Fatalf("export retry replayed execution: %+v (%d/%d)", d, first.Load(), second.Load())
}
}
func TestSessionExportGateCleanupUnconfirmedDoesNotExportOrAdvance(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, true)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
t.Fatalf("unconfirmed cleanup advanced/exported: %+v", d)
}
}
func TestSessionExportGateCanceledCleanupUnconfirmedDoesNotAdvance(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, true, gateTestOptions{waitForCancel: true})
deadline := time.Now().Add(3 * time.Second)
for {
d, _ := s.GetRun(context.Background(), id)
if len(d.Steps) > 0 && d.Steps[0].AgentActive {
break
}
if time.Now().After(deadline) {
t.Fatal("executor did not acquire session")
}
time.Sleep(time.Millisecond)
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if !d.CancelRequested || !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
t.Fatalf("canceled unconfirmed owner advanced: %+v", d)
}
}
func TestSessionExportGateCanceledExecutionStillReadsExactOwnerAndExports(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
if ctx.Err() != nil || req.SessionID != "ses_exact" || req.Reference != "exact-ref" {
t.Errorf("lost bounded uncanceled exact owner: %+v %v", req, ctx.Err())
}
exports.Add(1)
return validGateExport(ctx, req)
}, false, gateTestOptions{waitForCancel: true})
deadline := time.Now().Add(3 * time.Second)
for {
d, _ := s.GetRun(context.Background(), id)
if len(d.Steps) > 0 && d.Steps[0].SessionID != "" {
break
}
if time.Now().After(deadline) {
t.Fatal("executor did not record session")
}
time.Sleep(time.Millisecond)
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "canceled" || second.Load() != 0 || exports.Load() != 1 || d.Steps[0].SessionExport == nil {
t.Fatalf("canceled ownership/export was silently lost: %+v", d)
}
}
func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
var exports atomic.Int32
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
exports.Add(1)
return validGateExport(ctx, req)
}, false, gateTestOptions{missingOwner: true})
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
t.Fatalf("missing owner did not fail closed: %+v", d)
}
}
func TestSessionExportGateReservesWorkspaceWithoutExecutingAndCanCancel(t *testing.T) {
h, s, id, _, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
ctx := context.Background()
d, err := h.runner.getRun(id)
if err != nil || d.Executing {
t.Fatalf("parked export mislabeled executing: %+v %v", d, err)
}
pending, err := s.PendingSessionExportGate(ctx, id, 1)
if err != nil || !pending {
t.Fatalf("quiescent pending visit not exposed: %v %v", pending, err)
}
request := protocol.Message{PipelineID: d.PipelineID, Version: 1, RequestKey: "overlap", Workspace: d.Workspace, Input: map[string]any{"prompt": "another task"}}
if _, err := h.runner.start(request); err == nil {
t.Fatal("parked export released workspace")
}
if err := h.runner.cancel(id); err != nil {
t.Fatal(err)
}
pending, err = s.PendingSessionExportGate(ctx, id, 1)
if err != nil || pending {
t.Fatalf("canceled gate remains resumable: %v %v", pending, err)
}
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
request.RequestKey = "after-cancel"
if _, err := h.runner.start(request); err != nil {
t.Fatalf("cancel did not release workspace: %v", err)
}
h.runner.wg.Wait()
if second.Load() != 0 {
t.Fatal("canceled gate advanced")
}
}
func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
for _, mode := range []string{"missing", "empty", "partial", "wrong-size", "symlink"} {
t.Run(mode, func(t *testing.T) {
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
result, err := validGateExport(ctx, req)
path := filepath.Join(req.ArtifactsDir, result.HTML)
switch mode {
case "missing":
err = os.Remove(path)
case "empty":
result.Status = "empty"
case "partial":
result.Status = "partial"
case "wrong-size":
result.HTMLBytes++
case "symlink":
if err = os.Remove(path); err == nil {
err = os.Symlink("input.md", path)
}
}
return result, err
}, false)
h.runner.wg.Wait()
d, _ := s.GetRun(context.Background(), id)
if d.Status != "failed" || second.Load() != 0 || d.Output["markdown"] != "computed output" {
t.Fatalf("invalid export advanced: %+v", d)
}
})
}
}
func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) {
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("offline")
}, false)
h.runner.wg.Wait()
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
if err := s.Close(); err != nil {
t.Fatal(err)
}
reopened, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunner(ctx, reopened)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.artifactsRoot = root
r.exportSession = validGateExport
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
t.Errorf("replayed owner: %s", def.Key)
}
second.Add(1)
body, err := os.ReadFile(def.Turn.Input)
if err != nil || !strings.Contains(string(body), "computed output") {
t.Errorf("lost computed handoff: %q %v", body, err)
}
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
job, _ := reopened.SessionExportJob(ctx, id, 1)
if _, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
t.Fatal(err)
}
if err := r.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
r.wg.Wait()
if err := r.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
r.wg.Wait()
d, _ := reopened.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
t.Fatalf("restart replayed or lost successor: %+v", d)
}
}
+3
View File
@@ -64,6 +64,9 @@ func (r *runner) startSessionExportJobs() error {
break
}
}
if err := r.resumeExportGates(ctx); err != nil && ctx.Err() == nil {
log.Printf("session export handoff continuation failed: %v", err)
}
}
}()
return nil
+3 -2
View File
@@ -190,7 +190,8 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
if err != nil {
t.Fatal(err)
}
if job != nil && job.Status == "complete" {
settled, _ := s.GetRun(ctx, id)
if job != nil && job.Status == "complete" && settled.Status == "succeeded" {
break
}
if time.Now().After(deadline) {
@@ -203,7 +204,7 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
t.Fatal(err)
}
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != original.Status {
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "failed" {
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
}
}
+28 -5
View File
@@ -12,6 +12,7 @@ import (
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/opencode"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
@@ -25,8 +26,8 @@ const sessionExportBody = `{"info":{"id":"ses_run"},"messages":[` +
`{"type":"text","text":"done"},` +
`{"type":"tool","name":"shell","state":{"status":"completed","input":{"cmd":"ls"},"content":[{"type":"text","text":"output here"}]}}]}]}`
// sessionExportCLI points PIPELINES_OPENCODE at a fake `opencode export` that
// returns sessionExportBody, so the runner's real export path runs.
// sessionExportCLI isolates both the authenticated V2 API and legacy CLI.
// Neither export transport may fall through to the operator's real service.
func sessionExportCLI(t *testing.T) {
t.Helper()
path := filepath.Join(t.TempDir(), "opencode")
@@ -38,6 +39,28 @@ func sessionExportCLI(t *testing.T) {
t.Fatal(err)
}
t.Setenv("PIPELINES_OPENCODE", path)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
user, password, ok := r.BasicAuth()
if !ok || user != "opencode" || password != "fixture-password" {
w.WriteHeader(http.StatusUnauthorized)
return
}
if r.Method != "GET" || r.URL.Path != "/api/experimental/session/ses_run/export" || r.URL.Query().Get("sanitize") != "false" {
t.Errorf("unexpected export operation: %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":`+sessionExportBody+`}`)
}))
t.Cleanup(srv.Close)
passwordPath := filepath.Join(t.TempDir(), "password")
if err := os.WriteFile(passwordPath, []byte("fixture-password"), 0600); err != nil {
t.Fatal(err)
}
t.Setenv(opencode.EndpointEnvKey, srv.URL)
t.Setenv(opencode.EndpointPasswordFileEnvKey, passwordPath)
t.Setenv(opencode.EndpointUsernameEnvKey, "")
}
// sessionExportRunner runs one stage that records a session for the run, so the
@@ -145,7 +168,7 @@ func TestRunnerExportsTheHTMLSessionIntoArtifactsAndServesIt(t *testing.T) {
}
// TestRunnerSessionExportFailureTruncationAndUnavailableHarness proves the three
// non-success export outcomes are recorded as evidence and never fail the run.
// non-success exports block success without replacing the completed computation.
func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T) {
cases := []struct {
name string
@@ -188,8 +211,8 @@ func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" {
t.Fatalf("export outcome failed the run: %+v", detail.Status)
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "Session export gate blocked") || detail.Output["summary"] != "done" {
t.Fatalf("export gate did not preserve and block computation: %+v", detail)
}
step := detail.Steps[0]
if step.Status != "completed" || step.Outcome != "success" {
+1
View File
@@ -61,6 +61,7 @@ esac
t.Fatal(err)
}
handler := Handler(ctx, store)
handler.(*runHandler).runner.exportSession = validGateExport
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
+1
View File
@@ -116,6 +116,7 @@ func TestRunnerRecordsPerStageUsageAcrossMixedModels(t *testing.T) {
if err != nil {
t.Fatal(err)
}
r.exportSession = validGateExport
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mixed", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
if err != nil {
t.Fatal(err)