Files
pipelines/internal/server/session_export_gate.go
alan d35739cb49 Release session export gates whose recovery is exhausted or overdue
An exhausted export recovery job left its run parked forever in the
export wait, reserving the workspace and refusing later scheduled runs
(runs 221/226: OpenCode sessions above the 8 MiB export bound are
always truncated, so every retry stays partial).

A gate is now releasable when its job is exhausted: the visit keeps its
partial/unavailable evidence and diagnostic (still repairable), the run
continues to its next stage or terminal, and the workspace is freed.
Gates older than 15 minutes with a pending, lease-expired or missing job
are exhausted by a bounded wait. The worker reconciles on its first
tick after startup and every 30 seconds, so existing wedged runs recover
after a restart. Truncation diagnostics now name the size bound.
2026-11-01 17:24:56 +00:00

116 lines
3.8 KiB
Go

package server
import (
"context"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func completeSessionExport(e *protocol.SessionExport) bool {
return e != nil && e.Status == "complete" && e.SessionID != "" && e.Messages > 0 && e.HTML != "" && e.HTMLBytes > 0 && !e.Truncated && e.Diagnostic == ""
}
// resumeExportGates is also run when no job is due: a manual export repair or
// a crash after persisting repair evidence may have made a handoff eligible.
func (r *runner) resumeExportGates(ctx context.Context) error {
ready, err := r.store.ReadySessionExportGates(ctx)
if err != nil {
return err
}
for id, sequence := range ready {
r.mu.Lock()
if _, active := r.active[id]; active {
r.mu.Unlock()
continue
}
d, err := r.store.GetRun(ctx, id)
if err != nil {
r.mu.Unlock()
return err
}
if err := r.checkWorkspaceFreeExcept(d.Workspace, id); err != nil {
r.mu.Unlock()
continue
}
job, err := r.store.SessionExportJob(ctx, id, sequence)
if err != nil || job == nil {
r.mu.Unlock()
continue
}
var saved *protocol.RunStep
for i := range d.Steps {
if d.Steps[i].Sequence == sequence {
saved = &d.Steps[i]
break
}
}
// An exhausted recovery job is terminal: the visit continues with its
// recorded partial/unavailable evidence (still repairable later)
// rather than holding the run and its workspace forever.
exhausted := job.Status == "exhausted"
verified := exhausted && saved != nil
if !exhausted {
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
if err != nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
r.mu.Unlock()
continue
}
verified = saved != nil && r.verifiedExportRepair(id, sequence, saved, owner)
}
if !verified {
r.mu.Unlock()
continue
}
if err := r.store.ResumeSessionExportGate(ctx, id, sequence); err != nil {
r.mu.Unlock()
continue
}
if exhausted {
log.Printf("run %d visit %d: session export recovery exhausted (%s); continuing with incomplete session evidence", id, sequence, job.Diagnostic)
}
step := *saved
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
r.mu.Unlock()
}
return nil
}
// verifiedExportRepair reports whether a complete repair of the exact owner is
// present on disk with its recorded size.
func (r *runner) verifiedExportRepair(id int64, sequence int, saved *protocol.RunStep, owner pipeline.SessionOwner) bool {
for _, e := range saved.SessionExportRepairs {
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
continue
}
// New repairs stay private in the original visit. Historical
// public repairs remain valid, but are never rewritten there.
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
if strings.HasPrefix(e.HTML, "public/") {
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
continue
}
if filepath.Clean(e.HTML) != e.HTML {
continue
}
root, openErr := os.OpenRoot(directory)
if openErr != nil {
continue
}
info, statErr := root.Lstat(e.HTML)
root.Close()
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
return true
}
}
return false
}