Release session export gates whose recovery is exhausted or overdue
An exhausted export recovery job left its run parked forever in the export wait, reserving the workspace and refusing later scheduled runs (runs 221/226: OpenCode sessions above the 8 MiB export bound are always truncated, so every retry stays partial). A gate is now releasable when its job is exhausted: the visit keeps its partial/unavailable evidence and diagnostic (still repairable), the run continues to its next stage or terminal, and the workspace is freed. Gates older than 15 minutes with a pending, lease-expired or missing job are exhausted by a bounded wait. The worker reconciles on its first tick after startup and every 30 seconds, so existing wedged runs recover after a restart. Truncation diagnostics now name the size bound.
This commit is contained in:
@@ -0,0 +1,146 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// truncatedGateExport mirrors the live failure: a native session larger than
|
||||
// the export bound always renders a truncated, partial file.
|
||||
func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
result, err := validGateExport(ctx, req)
|
||||
result.Status, result.Truncated = "partial", true
|
||||
return result, err
|
||||
}
|
||||
|
||||
// exhaustExportJob spends the whole recovery budget of visit 1 on the runner.
|
||||
func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
for range pipeline.SessionExportJobAttempts {
|
||||
job, err := s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil || job == nil {
|
||||
t.Fatalf("missing recovery job: %v", err)
|
||||
}
|
||||
if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
||||
t.Fatalf("recovery attempt not run: %v %v", worked, err)
|
||||
}
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") {
|
||||
t.Fatalf("job not exhausted with the truncation reason: %+v", job)
|
||||
}
|
||||
}
|
||||
|
||||
func workspaceFree(r *runner, workspace string) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.checkWorkspaceFree(workspace)
|
||||
}
|
||||
|
||||
func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) {
|
||||
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
|
||||
h.runner.wg.Wait()
|
||||
ctx := context.Background()
|
||||
d, _ := s.GetRun(ctx, id)
|
||||
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 {
|
||||
t.Fatalf("partial export did not park the handoff: %+v", d)
|
||||
}
|
||||
if err := workspaceFree(h.runner, d.Workspace); err == nil {
|
||||
t.Fatal("parked export did not reserve its workspace")
|
||||
}
|
||||
exhaustExportJob(t, h.runner, s, id)
|
||||
if err := h.runner.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
d, _ = s.GetRun(ctx, id)
|
||||
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 {
|
||||
t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load())
|
||||
}
|
||||
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
|
||||
t.Fatalf("gate survived: %v %v", blocked, err)
|
||||
}
|
||||
visit := d.Steps[0]
|
||||
if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" {
|
||||
t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery)
|
||||
}
|
||||
// The workspace is free for the next (for example scheduled) run.
|
||||
if err := workspaceFree(h.runner, d.Workspace); err != nil {
|
||||
t.Fatalf("workspace still reserved: %v", err)
|
||||
}
|
||||
// The operator can still repair the export later through the normal path.
|
||||
h.runner.exportSession = validGateExport
|
||||
repaired, err := h.runner.repairSessionExport(ctx, id, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repairs := repaired.Steps[0].SessionExportRepairs
|
||||
if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" {
|
||||
t.Fatalf("later repair not recorded: %+v", repairs)
|
||||
}
|
||||
if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" {
|
||||
t.Fatalf("successful repair did not complete the job: %+v", job)
|
||||
}
|
||||
}
|
||||
|
||||
// A run parked before the fix, whose job was already exhausted, is released by
|
||||
// the restarted server's startup reconciliation without any manual edit.
|
||||
func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
|
||||
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
|
||||
h.runner.wg.Wait()
|
||||
exhaustExportJob(t, h.runner, s, id)
|
||||
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parked, _ := reopened.GetRun(context.Background(), id)
|
||||
if parked.Status != "waiting" || parked.ExportPendingSequence != 1 {
|
||||
t.Fatalf("fixture is not a parked export: %+v", parked)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r, err := newRunner(ctx, reopened)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
||||
r.artifactsRoot = root
|
||||
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
if def.Key != "two" {
|
||||
t.Errorf("replayed owner: %s", def.Key)
|
||||
}
|
||||
second.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
d, _ := reopened.GetRun(context.Background(), id)
|
||||
if d.Status == "succeeded" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("startup did not release the exhausted gate: %+v", d)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
r.wg.Wait()
|
||||
if first.Load() != 1 || second.Load() != 1 {
|
||||
t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load())
|
||||
}
|
||||
if err := workspaceFree(r, parked.Workspace); err != nil {
|
||||
t.Fatalf("workspace still reserved after startup release: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,13 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
@@ -37,13 +39,8 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
|
||||
if err != nil {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
job, err := r.store.SessionExportJob(ctx, id, sequence)
|
||||
if err != nil || job == nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
|
||||
if err != nil || job == nil {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
@@ -54,34 +51,18 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
break
|
||||
}
|
||||
}
|
||||
verified := false
|
||||
if saved != nil {
|
||||
for _, e := range saved.SessionExportRepairs {
|
||||
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
|
||||
continue
|
||||
}
|
||||
// New repairs stay private in the original visit. Historical
|
||||
// public repairs remain valid, but are never rewritten there.
|
||||
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
|
||||
if strings.HasPrefix(e.HTML, "public/") {
|
||||
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
|
||||
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
|
||||
continue
|
||||
}
|
||||
if filepath.Clean(e.HTML) != e.HTML {
|
||||
continue
|
||||
}
|
||||
root, openErr := os.OpenRoot(directory)
|
||||
if openErr != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := root.Lstat(e.HTML)
|
||||
root.Close()
|
||||
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
|
||||
verified = true
|
||||
break
|
||||
}
|
||||
// An exhausted recovery job is terminal: the visit continues with its
|
||||
// recorded partial/unavailable evidence (still repairable later)
|
||||
// rather than holding the run and its workspace forever.
|
||||
exhausted := job.Status == "exhausted"
|
||||
verified := exhausted && saved != nil
|
||||
if !exhausted {
|
||||
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
|
||||
if err != nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
verified = saved != nil && r.verifiedExportRepair(id, sequence, saved, owner)
|
||||
}
|
||||
if !verified {
|
||||
r.mu.Unlock()
|
||||
@@ -91,6 +72,9 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
if exhausted {
|
||||
log.Printf("run %d visit %d: session export recovery exhausted (%s); continuing with incomplete session evidence", id, sequence, job.Diagnostic)
|
||||
}
|
||||
step := *saved
|
||||
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
|
||||
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
|
||||
@@ -98,3 +82,34 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifiedExportRepair reports whether a complete repair of the exact owner is
|
||||
// present on disk with its recorded size.
|
||||
func (r *runner) verifiedExportRepair(id int64, sequence int, saved *protocol.RunStep, owner pipeline.SessionOwner) bool {
|
||||
for _, e := range saved.SessionExportRepairs {
|
||||
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
|
||||
continue
|
||||
}
|
||||
// New repairs stay private in the original visit. Historical
|
||||
// public repairs remain valid, but are never rewritten there.
|
||||
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
|
||||
if strings.HasPrefix(e.HTML, "public/") {
|
||||
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
|
||||
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
|
||||
continue
|
||||
}
|
||||
if filepath.Clean(e.HTML) != e.HTML {
|
||||
continue
|
||||
}
|
||||
root, openErr := os.OpenRoot(directory)
|
||||
if openErr != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := root.Lstat(e.HTML)
|
||||
root.Close()
|
||||
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -28,7 +29,11 @@ func (r *runner) startSessionExportJobs() error {
|
||||
defer cancel()
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
nextReconcile := time.Now().Add(30 * time.Second)
|
||||
// The first tick (one second after startup, once the handler finished
|
||||
// wiring the runner) reconciles too, so a gate whose job was already
|
||||
// exhausted, or overran its bounded wait, before a restart is released
|
||||
// at once instead of reserving its workspace.
|
||||
nextReconcile := time.Now()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -37,7 +42,7 @@ func (r *runner) startSessionExportJobs() error {
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
if time.Now().After(nextReconcile) {
|
||||
if !time.Now().Before(nextReconcile) {
|
||||
nextReconcile = time.Now().Add(30 * time.Second)
|
||||
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
|
||||
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
|
||||
@@ -46,6 +51,14 @@ func (r *runner) startSessionExportJobs() error {
|
||||
log.Printf("session export recovery reconciliation failed: %v", err)
|
||||
continue
|
||||
}
|
||||
if expired, err := r.store.ExpireSessionExportGates(ctx, time.Now(), pipeline.SessionExportGateMaxWait); err != nil {
|
||||
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
|
||||
return
|
||||
}
|
||||
log.Printf("session export wait expiry failed: %v", err)
|
||||
} else if expired > 0 {
|
||||
log.Printf("session export recovery stopped for %d visit(s) that exceeded the %s wait", expired, pipeline.SessionExportGateMaxWait)
|
||||
}
|
||||
}
|
||||
// Serial bounded batches; each readback has a 10-second budget.
|
||||
for range 8 {
|
||||
@@ -107,6 +120,11 @@ func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bo
|
||||
diagnostic = last.Diagnostic
|
||||
if diagnostic == "" && last.Status != "complete" {
|
||||
diagnostic = "Automatic native session export remains " + last.Status
|
||||
if last.Truncated {
|
||||
// The usual cause is inline attachment payloads (for
|
||||
// example images read by a tool) inside the native record.
|
||||
diagnostic += fmt.Sprintf(": the native session exceeds the %d MiB export bound, so a retry cannot complete it", stage.SessionExportSourceLimit>>20)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user