Files
pipelines/internal/server/session_export_jobs.go
T

120 lines
3.6 KiB
Go

package server
import (
"context"
"log"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// startSessionExportJobs is the only runner startup integration. Current main's
// Store.Done stops the worker and in-flight readback even with Background callers.
// The interface avoids duplicating that newer store implementation/server lock.
func (r *runner) startSessionExportJobs() error {
if err := r.store.ReconcileSessionExportJobs(r.ctx, time.Now()); err != nil {
return err
}
var done <-chan struct{}
if lifecycle, ok := any(r.store).(interface{ Done() <-chan struct{} }); ok {
done = lifecycle.Done()
}
ctx, cancel := sessionExportWorkerContext(r.ctx, done)
r.schedulerWG.Add(1)
go func() {
defer r.schedulerWG.Done()
defer cancel()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
nextReconcile := time.Now().Add(30 * time.Second)
for {
select {
case <-ctx.Done():
return
case <-done:
return
case <-ticker.C:
}
if time.Now().After(nextReconcile) {
nextReconcile = time.Now().Add(30 * time.Second)
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export recovery reconciliation failed: %v", err)
continue
}
}
// Serial bounded batches; each readback has a 10-second budget.
for range 8 {
if ctx.Err() != nil {
return
}
worked, err := r.recoverSessionExportJob(ctx, time.Now())
if err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export recovery failed: %v", err)
break
}
if !worked {
break
}
}
if err := r.resumeExportGates(ctx); err != nil && ctx.Err() == nil {
log.Printf("session export handoff continuation failed: %v", err)
}
}
}()
return nil
}
func sessionExportWorkerContext(parent context.Context, done <-chan struct{}) (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(parent)
go func() {
select {
case <-ctx.Done():
case <-done:
cancel()
}
}()
return ctx, cancel
}
func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bool, error) {
job, err := r.store.ClaimSessionExportJob(ctx, now, 45*time.Second)
if err != nil || job == nil {
return false, err
}
r.mu.Lock()
_, executing := r.active[job.RunID]
r.mu.Unlock()
if executing {
return true, r.store.DeferSessionExportJob(ctx, job, now)
}
owner := &pipeline.SessionOwner{ID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Reference: job.Reference, RunID: job.RunID}
detail, repairErr := r.repairSessionExportOwned(ctx, job.RunID, job.Sequence, owner)
diagnostic := ""
if repairErr != nil {
diagnostic = stage.RedactEvidence(repairErr.Error())
} else if detail != nil {
for _, step := range detail.Steps {
if step.Sequence == job.Sequence && len(step.SessionExportRepairs) > 0 {
last := step.SessionExportRepairs[len(step.SessionExportRepairs)-1]
diagnostic = last.Diagnostic
if diagnostic == "" && last.Status != "complete" {
diagnostic = "Automatic native session export remains " + last.Status
}
}
}
}
// Save acknowledgement even on shutdown after an attempt. A process kill
// before this point leaves a recoverable, budgeted lease.
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
return true, r.store.FinishSessionExportJob(save, job, time.Now(), diagnostic)
}