d35739cb49
An exhausted export recovery job left its run parked forever in the export wait, reserving the workspace and refusing later scheduled runs (runs 221/226: OpenCode sessions above the 8 MiB export bound are always truncated, so every retry stays partial). A gate is now releasable when its job is exhausted: the visit keeps its partial/unavailable evidence and diagnostic (still repairable), the run continues to its next stage or terminal, and the workspace is freed. Gates older than 15 minutes with a pending, lease-expired or missing job are exhausted by a bounded wait. The worker reconciles on its first tick after startup and every 30 seconds, so existing wedged runs recover after a restart. Truncation diagnostics now name the size bound.
147 lines
5.4 KiB
Go
147 lines
5.4 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// truncatedGateExport mirrors the live failure: a native session larger than
|
|
// the export bound always renders a truncated, partial file.
|
|
func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
result, err := validGateExport(ctx, req)
|
|
result.Status, result.Truncated = "partial", true
|
|
return result, err
|
|
}
|
|
|
|
// exhaustExportJob spends the whole recovery budget of visit 1 on the runner.
|
|
func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
for range pipeline.SessionExportJobAttempts {
|
|
job, err := s.SessionExportJob(ctx, id, 1)
|
|
if err != nil || job == nil {
|
|
t.Fatalf("missing recovery job: %v", err)
|
|
}
|
|
if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
|
t.Fatalf("recovery attempt not run: %v %v", worked, err)
|
|
}
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") {
|
|
t.Fatalf("job not exhausted with the truncation reason: %+v", job)
|
|
}
|
|
}
|
|
|
|
func workspaceFree(r *runner, workspace string) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return r.checkWorkspaceFree(workspace)
|
|
}
|
|
|
|
func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
|
|
h.runner.wg.Wait()
|
|
ctx := context.Background()
|
|
d, _ := s.GetRun(ctx, id)
|
|
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 {
|
|
t.Fatalf("partial export did not park the handoff: %+v", d)
|
|
}
|
|
if err := workspaceFree(h.runner, d.Workspace); err == nil {
|
|
t.Fatal("parked export did not reserve its workspace")
|
|
}
|
|
exhaustExportJob(t, h.runner, s, id)
|
|
if err := h.runner.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ = s.GetRun(ctx, id)
|
|
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 {
|
|
t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load())
|
|
}
|
|
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
|
|
t.Fatalf("gate survived: %v %v", blocked, err)
|
|
}
|
|
visit := d.Steps[0]
|
|
if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" {
|
|
t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery)
|
|
}
|
|
// The workspace is free for the next (for example scheduled) run.
|
|
if err := workspaceFree(h.runner, d.Workspace); err != nil {
|
|
t.Fatalf("workspace still reserved: %v", err)
|
|
}
|
|
// The operator can still repair the export later through the normal path.
|
|
h.runner.exportSession = validGateExport
|
|
repaired, err := h.runner.repairSessionExport(ctx, id, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
repairs := repaired.Steps[0].SessionExportRepairs
|
|
if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" {
|
|
t.Fatalf("later repair not recorded: %+v", repairs)
|
|
}
|
|
if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" {
|
|
t.Fatalf("successful repair did not complete the job: %+v", job)
|
|
}
|
|
}
|
|
|
|
// A run parked before the fix, whose job was already exhausted, is released by
|
|
// the restarted server's startup reconciliation without any manual edit.
|
|
func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
|
|
h.runner.wg.Wait()
|
|
exhaustExportJob(t, h.runner, s, id)
|
|
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reopened, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
parked, _ := reopened.GetRun(context.Background(), id)
|
|
if parked.Status != "waiting" || parked.ExportPendingSequence != 1 {
|
|
t.Fatalf("fixture is not a parked export: %+v", parked)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
r, err := newRunner(ctx, reopened)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
|
r.artifactsRoot = root
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
if def.Key != "two" {
|
|
t.Errorf("replayed owner: %s", def.Key)
|
|
}
|
|
second.Add(1)
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
|
}
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for {
|
|
d, _ := reopened.GetRun(context.Background(), id)
|
|
if d.Status == "succeeded" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("startup did not release the exhausted gate: %+v", d)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
r.wg.Wait()
|
|
if first.Load() != 1 || second.Load() != 1 {
|
|
t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load())
|
|
}
|
|
if err := workspaceFree(r, parked.Workspace); err != nil {
|
|
t.Fatalf("workspace still reserved after startup release: %v", err)
|
|
}
|
|
}
|