Files
pipelines/internal/server/session_export_jobs.go
alan d35739cb49 Release session export gates whose recovery is exhausted or overdue
An exhausted export recovery job left its run parked forever in the
export wait, reserving the workspace and refusing later scheduled runs
(runs 221/226: OpenCode sessions above the 8 MiB export bound are
always truncated, so every retry stays partial).

A gate is now releasable when its job is exhausted: the visit keeps its
partial/unavailable evidence and diagnostic (still repairable), the run
continues to its next stage or terminal, and the workspace is freed.
Gates older than 15 minutes with a pending, lease-expired or missing job
are exhausted by a bounded wait. The worker reconciles on its first
tick after startup and every 30 seconds, so existing wedged runs recover
after a restart. Truncation diagnostics now name the size bound.
2026-11-01 17:24:56 +00:00

138 lines
4.6 KiB
Go

package server
import (
"context"
"fmt"
"log"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// startSessionExportJobs is the only runner startup integration. Current main's
// Store.Done stops the worker and in-flight readback even with Background callers.
// The interface avoids duplicating that newer store implementation/server lock.
func (r *runner) startSessionExportJobs() error {
if err := r.store.ReconcileSessionExportJobs(r.ctx, time.Now()); err != nil {
return err
}
var done <-chan struct{}
if lifecycle, ok := any(r.store).(interface{ Done() <-chan struct{} }); ok {
done = lifecycle.Done()
}
ctx, cancel := sessionExportWorkerContext(r.ctx, done)
r.schedulerWG.Add(1)
go func() {
defer r.schedulerWG.Done()
defer cancel()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
// The first tick (one second after startup, once the handler finished
// wiring the runner) reconciles too, so a gate whose job was already
// exhausted, or overran its bounded wait, before a restart is released
// at once instead of reserving its workspace.
nextReconcile := time.Now()
for {
select {
case <-ctx.Done():
return
case <-done:
return
case <-ticker.C:
}
if !time.Now().Before(nextReconcile) {
nextReconcile = time.Now().Add(30 * time.Second)
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export recovery reconciliation failed: %v", err)
continue
}
if expired, err := r.store.ExpireSessionExportGates(ctx, time.Now(), pipeline.SessionExportGateMaxWait); err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export wait expiry failed: %v", err)
} else if expired > 0 {
log.Printf("session export recovery stopped for %d visit(s) that exceeded the %s wait", expired, pipeline.SessionExportGateMaxWait)
}
}
// Serial bounded batches; each readback has a 10-second budget.
for range 8 {
if ctx.Err() != nil {
return
}
worked, err := r.recoverSessionExportJob(ctx, time.Now())
if err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export recovery failed: %v", err)
break
}
if !worked {
break
}
}
if err := r.resumeExportGates(ctx); err != nil && ctx.Err() == nil {
log.Printf("session export handoff continuation failed: %v", err)
}
}
}()
return nil
}
func sessionExportWorkerContext(parent context.Context, done <-chan struct{}) (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(parent)
go func() {
select {
case <-ctx.Done():
case <-done:
cancel()
}
}()
return ctx, cancel
}
func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bool, error) {
job, err := r.store.ClaimSessionExportJob(ctx, now, 45*time.Second)
if err != nil || job == nil {
return false, err
}
r.mu.Lock()
_, executing := r.active[job.RunID]
r.mu.Unlock()
if executing {
return true, r.store.DeferSessionExportJob(ctx, job, now)
}
owner := &pipeline.SessionOwner{ID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Reference: job.Reference, RunID: job.RunID}
detail, repairErr := r.repairSessionExportOwned(ctx, job.RunID, job.Sequence, owner)
diagnostic := ""
if repairErr != nil {
diagnostic = stage.RedactEvidence(repairErr.Error())
} else if detail != nil {
for _, step := range detail.Steps {
if step.Sequence == job.Sequence && len(step.SessionExportRepairs) > 0 {
last := step.SessionExportRepairs[len(step.SessionExportRepairs)-1]
diagnostic = last.Diagnostic
if diagnostic == "" && last.Status != "complete" {
diagnostic = "Automatic native session export remains " + last.Status
if last.Truncated {
// The usual cause is inline attachment payloads (for
// example images read by a tool) inside the native record.
diagnostic += fmt.Sprintf(": the native session exceeds the %d MiB export bound, so a retry cannot complete it", stage.SessionExportSourceLimit>>20)
}
}
}
}
}
// Save acknowledgement even on shutdown after an attempt. A process kill
// before this point leaves a recoverable, budgeted lease.
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
return true, r.store.FinishSessionExportJob(save, job, time.Now(), diagnostic)
}