Files
pipelines/internal/server/session_export_exhausted_test.go
T
alan d35739cb49 Release session export gates whose recovery is exhausted or overdue
An exhausted export recovery job left its run parked forever in the
export wait, reserving the workspace and refusing later scheduled runs
(runs 221/226: OpenCode sessions above the 8 MiB export bound are
always truncated, so every retry stays partial).

A gate is now releasable when its job is exhausted: the visit keeps its
partial/unavailable evidence and diagnostic (still repairable), the run
continues to its next stage or terminal, and the workspace is freed.
Gates older than 15 minutes with a pending, lease-expired or missing job
are exhausted by a bounded wait. The worker reconciles on its first
tick after startup and every 30 seconds, so existing wedged runs recover
after a restart. Truncation diagnostics now name the size bound.
2026-11-01 17:24:56 +00:00

147 lines
5.4 KiB
Go

package server
import (
"context"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// truncatedGateExport mirrors the live failure: a native session larger than
// the export bound always renders a truncated, partial file.
func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
result, err := validGateExport(ctx, req)
result.Status, result.Truncated = "partial", true
return result, err
}
// exhaustExportJob spends the whole recovery budget of visit 1 on the runner.
func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) {
t.Helper()
ctx := context.Background()
for range pipeline.SessionExportJobAttempts {
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil {
t.Fatalf("missing recovery job: %v", err)
}
if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("recovery attempt not run: %v %v", worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") {
t.Fatalf("job not exhausted with the truncation reason: %+v", job)
}
}
func workspaceFree(r *runner, workspace string) error {
r.mu.Lock()
defer r.mu.Unlock()
return r.checkWorkspaceFree(workspace)
}
func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) {
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 {
t.Fatalf("partial export did not park the handoff: %+v", d)
}
if err := workspaceFree(h.runner, d.Workspace); err == nil {
t.Fatal("parked export did not reserve its workspace")
}
exhaustExportJob(t, h.runner, s, id)
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ = s.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 {
t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load())
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("gate survived: %v %v", blocked, err)
}
visit := d.Steps[0]
if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" {
t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery)
}
// The workspace is free for the next (for example scheduled) run.
if err := workspaceFree(h.runner, d.Workspace); err != nil {
t.Fatalf("workspace still reserved: %v", err)
}
// The operator can still repair the export later through the normal path.
h.runner.exportSession = validGateExport
repaired, err := h.runner.repairSessionExport(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
repairs := repaired.Steps[0].SessionExportRepairs
if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" {
t.Fatalf("later repair not recorded: %+v", repairs)
}
if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil {
t.Fatal(err)
}
if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" {
t.Fatalf("successful repair did not complete the job: %+v", job)
}
}
// A run parked before the fix, whose job was already exhausted, is released by
// the restarted server's startup reconciliation without any manual edit.
func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
h.runner.wg.Wait()
exhaustExportJob(t, h.runner, s, id)
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
if err := s.Close(); err != nil {
t.Fatal(err)
}
reopened, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
parked, _ := reopened.GetRun(context.Background(), id)
if parked.Status != "waiting" || parked.ExportPendingSequence != 1 {
t.Fatalf("fixture is not a parked export: %+v", parked)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunner(ctx, reopened)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.artifactsRoot = root
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
t.Errorf("replayed owner: %s", def.Key)
}
second.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
deadline := time.Now().Add(10 * time.Second)
for {
d, _ := reopened.GetRun(context.Background(), id)
if d.Status == "succeeded" {
break
}
if time.Now().After(deadline) {
t.Fatalf("startup did not release the exhausted gate: %+v", d)
}
time.Sleep(20 * time.Millisecond)
}
r.wg.Wait()
if first.Load() != 1 || second.Load() != 1 {
t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load())
}
if err := workspaceFree(r, parked.Workspace); err != nil {
t.Fatalf("workspace still reserved after startup release: %v", err)
}
}