d35739cb49
An exhausted export recovery job left its run parked forever in the export wait, reserving the workspace and refusing later scheduled runs (runs 221/226: OpenCode sessions above the 8 MiB export bound are always truncated, so every retry stays partial). A gate is now releasable when its job is exhausted: the visit keeps its partial/unavailable evidence and diagnostic (still repairable), the run continues to its next stage or terminal, and the workspace is freed. Gates older than 15 minutes with a pending, lease-expired or missing job are exhausted by a bounded wait. The worker reconciles on its first tick after startup and every 30 seconds, so existing wedged runs recover after a restart. Truncation diagnostics now name the size bound.
116 lines
3.8 KiB
Go
116 lines
3.8 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func completeSessionExport(e *protocol.SessionExport) bool {
|
|
return e != nil && e.Status == "complete" && e.SessionID != "" && e.Messages > 0 && e.HTML != "" && e.HTMLBytes > 0 && !e.Truncated && e.Diagnostic == ""
|
|
}
|
|
|
|
// resumeExportGates is also run when no job is due: a manual export repair or
|
|
// a crash after persisting repair evidence may have made a handoff eligible.
|
|
func (r *runner) resumeExportGates(ctx context.Context) error {
|
|
ready, err := r.store.ReadySessionExportGates(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for id, sequence := range ready {
|
|
r.mu.Lock()
|
|
if _, active := r.active[id]; active {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
d, err := r.store.GetRun(ctx, id)
|
|
if err != nil {
|
|
r.mu.Unlock()
|
|
return err
|
|
}
|
|
if err := r.checkWorkspaceFreeExcept(d.Workspace, id); err != nil {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
job, err := r.store.SessionExportJob(ctx, id, sequence)
|
|
if err != nil || job == nil {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
var saved *protocol.RunStep
|
|
for i := range d.Steps {
|
|
if d.Steps[i].Sequence == sequence {
|
|
saved = &d.Steps[i]
|
|
break
|
|
}
|
|
}
|
|
// An exhausted recovery job is terminal: the visit continues with its
|
|
// recorded partial/unavailable evidence (still repairable later)
|
|
// rather than holding the run and its workspace forever.
|
|
exhausted := job.Status == "exhausted"
|
|
verified := exhausted && saved != nil
|
|
if !exhausted {
|
|
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
|
|
if err != nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
verified = saved != nil && r.verifiedExportRepair(id, sequence, saved, owner)
|
|
}
|
|
if !verified {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
if err := r.store.ResumeSessionExportGate(ctx, id, sequence); err != nil {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
if exhausted {
|
|
log.Printf("run %d visit %d: session export recovery exhausted (%s); continuing with incomplete session evidence", id, sequence, job.Diagnostic)
|
|
}
|
|
step := *saved
|
|
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
|
|
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
|
|
r.mu.Unlock()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// verifiedExportRepair reports whether a complete repair of the exact owner is
|
|
// present on disk with its recorded size.
|
|
func (r *runner) verifiedExportRepair(id int64, sequence int, saved *protocol.RunStep, owner pipeline.SessionOwner) bool {
|
|
for _, e := range saved.SessionExportRepairs {
|
|
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
|
|
continue
|
|
}
|
|
// New repairs stay private in the original visit. Historical
|
|
// public repairs remain valid, but are never rewritten there.
|
|
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
|
|
if strings.HasPrefix(e.HTML, "public/") {
|
|
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
|
|
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
|
|
continue
|
|
}
|
|
if filepath.Clean(e.HTML) != e.HTML {
|
|
continue
|
|
}
|
|
root, openErr := os.OpenRoot(directory)
|
|
if openErr != nil {
|
|
continue
|
|
}
|
|
info, statErr := root.Lstat(e.HTML)
|
|
root.Close()
|
|
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|