120 lines
3.6 KiB
Go
120 lines
3.6 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// startSessionExportJobs is the only runner startup integration. Current main's
|
|
// Store.Done stops the worker and in-flight readback even with Background callers.
|
|
// The interface avoids duplicating that newer store implementation/server lock.
|
|
func (r *runner) startSessionExportJobs() error {
|
|
if err := r.store.ReconcileSessionExportJobs(r.ctx, time.Now()); err != nil {
|
|
return err
|
|
}
|
|
var done <-chan struct{}
|
|
if lifecycle, ok := any(r.store).(interface{ Done() <-chan struct{} }); ok {
|
|
done = lifecycle.Done()
|
|
}
|
|
ctx, cancel := sessionExportWorkerContext(r.ctx, done)
|
|
r.schedulerWG.Add(1)
|
|
go func() {
|
|
defer r.schedulerWG.Done()
|
|
defer cancel()
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
nextReconcile := time.Now().Add(30 * time.Second)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-done:
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
if time.Now().After(nextReconcile) {
|
|
nextReconcile = time.Now().Add(30 * time.Second)
|
|
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
|
|
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
|
|
return
|
|
}
|
|
log.Printf("session export recovery reconciliation failed: %v", err)
|
|
continue
|
|
}
|
|
}
|
|
// Serial bounded batches; each readback has a 10-second budget.
|
|
for range 8 {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
worked, err := r.recoverSessionExportJob(ctx, time.Now())
|
|
if err != nil {
|
|
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
|
|
return
|
|
}
|
|
log.Printf("session export recovery failed: %v", err)
|
|
break
|
|
}
|
|
if !worked {
|
|
break
|
|
}
|
|
}
|
|
if err := r.resumeExportGates(ctx); err != nil && ctx.Err() == nil {
|
|
log.Printf("session export handoff continuation failed: %v", err)
|
|
}
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
func sessionExportWorkerContext(parent context.Context, done <-chan struct{}) (context.Context, context.CancelFunc) {
|
|
ctx, cancel := context.WithCancel(parent)
|
|
go func() {
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-done:
|
|
cancel()
|
|
}
|
|
}()
|
|
return ctx, cancel
|
|
}
|
|
|
|
func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bool, error) {
|
|
job, err := r.store.ClaimSessionExportJob(ctx, now, 45*time.Second)
|
|
if err != nil || job == nil {
|
|
return false, err
|
|
}
|
|
r.mu.Lock()
|
|
_, executing := r.active[job.RunID]
|
|
r.mu.Unlock()
|
|
if executing {
|
|
return true, r.store.DeferSessionExportJob(ctx, job, now)
|
|
}
|
|
owner := &pipeline.SessionOwner{ID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Reference: job.Reference, RunID: job.RunID}
|
|
detail, repairErr := r.repairSessionExportOwned(ctx, job.RunID, job.Sequence, owner)
|
|
diagnostic := ""
|
|
if repairErr != nil {
|
|
diagnostic = stage.RedactEvidence(repairErr.Error())
|
|
} else if detail != nil {
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence == job.Sequence && len(step.SessionExportRepairs) > 0 {
|
|
last := step.SessionExportRepairs[len(step.SessionExportRepairs)-1]
|
|
diagnostic = last.Diagnostic
|
|
if diagnostic == "" && last.Status != "complete" {
|
|
diagnostic = "Automatic native session export remains " + last.Status
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Save acknowledgement even on shutdown after an attempt. A process kill
|
|
// before this point leaves a recoverable, budgeted lease.
|
|
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
defer cancel()
|
|
return true, r.store.FinishSessionExportJob(save, job, time.Now(), diagnostic)
|
|
}
|