Release session export gates whose recovery is exhausted or overdue

An exhausted export recovery job left its run parked forever in the
export wait, reserving the workspace and refusing later scheduled runs
(runs 221/226: OpenCode sessions above the 8 MiB export bound are
always truncated, so every retry stays partial).

A gate is now releasable when its job is exhausted: the visit keeps its
partial/unavailable evidence and diagnostic (still repairable), the run
continues to its next stage or terminal, and the workspace is freed.
Gates older than 15 minutes with a pending, lease-expired or missing job
are exhausted by a bounded wait. The worker reconciles on its first
tick after startup and every 30 seconds, so existing wedged runs recover
after a restart. Truncation diagnostics now name the size bound.
This commit is contained in:
2026-11-01 17:24:56 +00:00
parent c742bb17f0
commit d35739cb49
8 changed files with 405 additions and 46 deletions
+7
View File
@@ -1015,3 +1015,10 @@ run settles: up to three durable runtime attempts, with backoff and restart-safe
leases. `inspect RUN_ID --json` includes each visit's `session_export_recovery`
status/attempt count/diagnostic and preserved `session_export_repairs`. Exhausted
automatic recovery remains a warning; manual versioned repair is still available.
A run parked on an export handoff never waits beyond that: once its job is
exhausted (or the wait exceeds 15 minutes with the job pending or past its
lease), the visit keeps its partial/unavailable evidence and diagnostic, the run
continues to its next stage or terminal, and its workspace is released. The
server applies this at startup and every 30 seconds, so a run parked before an
upgrade recovers after the next restart. A native session larger than the 8 MiB
export bound (typically inline image payloads read by a tool) is always partial.
+2 -1
View File
@@ -47,7 +47,8 @@ only**, not credentials, projects, sessions, drafts, or history.
Model stages settle their exact native session and verify its export before
handoff. An export failure retains computed output and waits for export-only
recovery. **Retry export only** reads that session again; it does not rerun the
recovery; when that bounded recovery is exhausted the run continues with the
incomplete export recorded as a warning. **Retry export only** reads that session again; it does not rerun the
stage. A genuinely missing historical conversation cannot be reconstructed by
inventing messages.
+4 -3
View File
@@ -58,7 +58,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'')<>'complete')`, key, id)); err != nil {
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'') NOT IN ('complete','exhausted'))`, key, id)); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
@@ -396,8 +396,9 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
return err
}
// Terminal targets have no successor BeginStep to consume their gate.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status='complete')`, id); err != nil {
// Terminal targets have no successor BeginStep to consume their gate. An
// exhausted job is terminal too: its incomplete evidence stays recorded.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status IN ('complete','exhausted'))`, id); err != nil {
return err
}
return tx.Commit()
+97 -7
View File
@@ -1,6 +1,13 @@
package pipeline
import "context"
import (
"context"
"encoding/json"
"fmt"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func (s *Store) SessionExportBlocked(ctx context.Context, id int64) (bool, error) {
var blocked bool
@@ -47,8 +54,8 @@ func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, err
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
JOIN pipeline_runs r ON r.id=g.run_id
WHERE j.status='complete' AND r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`)
WHERE r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+releasableGateJob+` LIMIT 64`)
if err != nil {
return nil, err
}
@@ -65,10 +72,93 @@ AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_ac
return ready, rows.Err()
}
// ResumeSessionExportGate is a CAS, not a retry of the model invocation.
// ResumeSessionExportGate is a CAS, not a retry of the model invocation. A
// gate whose recovery job is exhausted is released exactly like a completed
// one: the visit keeps its partial/unavailable export evidence and the job's
// diagnostic, and the operator can still repair it later. When no export
// evidence was recorded at all, one unavailable record carries the diagnostic.
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`)
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence))
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND `+releasableGateJob+`)
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence)); err != nil {
return err
}
var job SessionExportJob
if err := tx.QueryRowContext(ctx, `SELECT session_id,backend,directory,status,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.SessionID, &job.Backend, &job.Directory, &job.Status, &job.Diagnostic); err != nil {
return err
}
if job.Status == "exhausted" {
var recorded bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM step_session_exports WHERE run_id=? AND sequence=?) OR EXISTS(SELECT 1 FROM session_export_repairs WHERE run_id=? AND sequence=?)`, id, sequence, id, sequence).Scan(&recorded); err != nil {
return err
}
if !recorded {
diagnostic := job.Diagnostic
if diagnostic == "" {
diagnostic = "Automatic session export recovery exhausted"
}
body, err := json.Marshal(protocol.SessionExport{Status: "unavailable", Source: "unavailable", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339), Diagnostic: diagnostic})
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,?,?)", id, sequence, string(body)); err != nil {
return err
}
}
}
return tx.Commit()
}
// releasableGateJob is a gate's terminal recovery state. A complete job must
// still match the exact original owner. An exhausted job is terminal whatever
// the reason (including changed ownership): waiting longer cannot help, so the
// handoff proceeds with the incomplete evidence instead of wedging the run and
// its workspace forever.
const releasableGateJob = `(j.status='exhausted' OR (j.status='complete' AND ` + exactExportJobOwner + `))`
// SessionExportGateMaxWait bounds how long a computed visit may wait for its
// export. The normal budget (three attempts with 45-second leases and up to a
// minute of backoff) finishes in a few minutes; this only catches a job that
// can never be claimed or acknowledged.
const SessionExportGateMaxWait = 15 * time.Minute
// ExpireSessionExportGates exhausts the recovery job of every gate older than
// maxWait whose job is still pending, or running past its lease, and gives a
// gate with no recovery job at all an exhausted one. A live lease is never
// stolen; it expires within its own bound and is exhausted on the next pass.
func (s *Store) ExpireSessionExportGates(ctx context.Context, now time.Time, maxWait time.Duration) (int64, error) {
cutoff := now.Add(-maxWait).UTC().Format(time.DateTime)
diagnostic := fmt.Sprintf("Session export wait exceeded %s; automatic recovery stopped", maxWait)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return 0, err
}
defer tx.Rollback()
result, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,
diagnostic=substr(?||CASE WHEN j.diagnostic<>'' THEN ' (last: '||j.diagnostic||')' ELSE '' END,1,2048)
WHERE (j.status='pending' OR (j.status='running' AND j.lease_until<=?))
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
WHERE g.run_id=j.run_id AND g.sequence=j.sequence AND s.finished_at IS NOT NULL AND s.finished_at<=?)`, diagnostic, now.UnixMilli(), cutoff)
if err != nil {
return 0, err
}
expired, _ := result.RowsAffected()
result, err = tx.ExecContext(ctx, `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic)
SELECT g.run_id,g.sequence,s.session_id,COALESCE(o.backend,''),r.workspace,COALESCE(o.reference,''),'exhausted',0,0,?
FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence JOIN pipeline_runs r ON r.id=g.run_id
LEFT JOIN run_session_owners o ON o.run_id=g.run_id AND o.sequence=g.sequence
WHERE s.finished_at IS NOT NULL AND s.finished_at<=?
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=g.run_id AND j.sequence=g.sequence)
ON CONFLICT(run_id,sequence) DO NOTHING`, "No automatic session export recovery exists for this visit; "+diagnostic, cutoff)
if err != nil {
return 0, err
}
created, _ := result.RowsAffected()
return expired + created, tx.Commit()
}
@@ -2,6 +2,7 @@ package pipeline
import (
"context"
"strings"
"testing"
"time"
@@ -73,3 +74,83 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
t.Fatal("successor started twice")
}
}
// A recovery job wedged in running past its lease (or pending and never
// claimable) is bounded by the gate's overall wait; a live lease is not stolen.
func TestSessionExportGateWaitIsBoundedForStuckJob(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "partial", SessionID: "ses_original", Backend: "v2", Truncated: true}}
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
t.Fatal(err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
start := time.UnixMilli(job.NextAt)
if claimed, err := s.ClaimSessionExportJob(ctx, start, time.Hour); err != nil || claimed == nil {
t.Fatalf("claim: %+v %v", claimed, err)
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(5*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
t.Fatalf("young gate expired: %d %v", n, err)
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(20*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
t.Fatalf("live lease stolen: %d %v", n, err)
}
if ready, _ := s.ReadySessionExportGates(ctx); ready[id] != 0 {
t.Fatal("running job released its gate")
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(2*time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
t.Fatalf("stuck job not bounded: %d %v", n, err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "wait exceeded") {
t.Fatalf("job not exhausted: %+v", job)
}
ready, err := s.ReadySessionExportGates(ctx)
if err != nil || ready[id] != 1 {
t.Fatalf("exhausted gate not ready: %v %v", ready, err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("terminal finish kept exhausted gate: %v %v", blocked, err)
}
d, _ := s.GetRun(ctx, id)
if d.Status != "succeeded" || d.Steps[0].SessionExport == nil || d.Steps[0].SessionExport.Status != "partial" || len(d.Steps[0].SessionExportRepairs) != 0 {
t.Fatalf("partial evidence not retained as-is: %+v", d)
}
}
// A gate whose visit never got a recovery job (no exact owner) is given an
// exhausted job after the bounded wait, and the release records why.
func TestSessionExportGateWithoutJobIsReleasedAfterWait(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
if _, err := s.db.Exec(`DELETE FROM run_session_owners WHERE run_id=?`, id); err != nil {
t.Fatal(err)
}
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}}
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
t.Fatal(err)
}
if job, _ := s.SessionExportJob(ctx, id, 1); job != nil {
t.Fatalf("fixture unexpectedly has a job: %+v", job)
}
if n, err := s.ExpireSessionExportGates(ctx, time.Now().Add(time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
t.Fatalf("jobless gate not bounded: %d %v", n, err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
t.Fatalf("successor refused after release: %v", err)
}
d, _ := s.GetRun(ctx, id)
repairs := d.Steps[0].SessionExportRepairs
if len(repairs) != 1 || repairs[0].Status != "unavailable" || !strings.Contains(repairs[0].Diagnostic, "No automatic session export recovery") {
t.Fatalf("release did not record unavailable evidence: %+v", repairs)
}
}
@@ -0,0 +1,146 @@
package server
import (
"context"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// truncatedGateExport mirrors the live failure: a native session larger than
// the export bound always renders a truncated, partial file.
func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
result, err := validGateExport(ctx, req)
result.Status, result.Truncated = "partial", true
return result, err
}
// exhaustExportJob spends the whole recovery budget of visit 1 on the runner.
func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) {
t.Helper()
ctx := context.Background()
for range pipeline.SessionExportJobAttempts {
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil {
t.Fatalf("missing recovery job: %v", err)
}
if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("recovery attempt not run: %v %v", worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") {
t.Fatalf("job not exhausted with the truncation reason: %+v", job)
}
}
func workspaceFree(r *runner, workspace string) error {
r.mu.Lock()
defer r.mu.Unlock()
return r.checkWorkspaceFree(workspace)
}
func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) {
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 {
t.Fatalf("partial export did not park the handoff: %+v", d)
}
if err := workspaceFree(h.runner, d.Workspace); err == nil {
t.Fatal("parked export did not reserve its workspace")
}
exhaustExportJob(t, h.runner, s, id)
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ = s.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 {
t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load())
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("gate survived: %v %v", blocked, err)
}
visit := d.Steps[0]
if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" {
t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery)
}
// The workspace is free for the next (for example scheduled) run.
if err := workspaceFree(h.runner, d.Workspace); err != nil {
t.Fatalf("workspace still reserved: %v", err)
}
// The operator can still repair the export later through the normal path.
h.runner.exportSession = validGateExport
repaired, err := h.runner.repairSessionExport(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
repairs := repaired.Steps[0].SessionExportRepairs
if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" {
t.Fatalf("later repair not recorded: %+v", repairs)
}
if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil {
t.Fatal(err)
}
if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" {
t.Fatalf("successful repair did not complete the job: %+v", job)
}
}
// A run parked before the fix, whose job was already exhausted, is released by
// the restarted server's startup reconciliation without any manual edit.
func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
h.runner.wg.Wait()
exhaustExportJob(t, h.runner, s, id)
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
if err := s.Close(); err != nil {
t.Fatal(err)
}
reopened, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
parked, _ := reopened.GetRun(context.Background(), id)
if parked.Status != "waiting" || parked.ExportPendingSequence != 1 {
t.Fatalf("fixture is not a parked export: %+v", parked)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunner(ctx, reopened)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.artifactsRoot = root
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
t.Errorf("replayed owner: %s", def.Key)
}
second.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
deadline := time.Now().Add(10 * time.Second)
for {
d, _ := reopened.GetRun(context.Background(), id)
if d.Status == "succeeded" {
break
}
if time.Now().After(deadline) {
t.Fatalf("startup did not release the exhausted gate: %+v", d)
}
time.Sleep(20 * time.Millisecond)
}
r.wg.Wait()
if first.Load() != 1 || second.Load() != 1 {
t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load())
}
if err := workspaceFree(r, parked.Workspace); err != nil {
t.Fatalf("workspace still reserved after startup release: %v", err)
}
}
+48 -33
View File
@@ -2,11 +2,13 @@ package server
import (
"context"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
@@ -37,13 +39,8 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
r.mu.Unlock()
continue
}
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
if err != nil {
r.mu.Unlock()
continue
}
job, err := r.store.SessionExportJob(ctx, id, sequence)
if err != nil || job == nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
if err != nil || job == nil {
r.mu.Unlock()
continue
}
@@ -54,34 +51,18 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
break
}
}
verified := false
if saved != nil {
for _, e := range saved.SessionExportRepairs {
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
continue
}
// New repairs stay private in the original visit. Historical
// public repairs remain valid, but are never rewritten there.
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
if strings.HasPrefix(e.HTML, "public/") {
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
continue
}
if filepath.Clean(e.HTML) != e.HTML {
continue
}
root, openErr := os.OpenRoot(directory)
if openErr != nil {
continue
}
info, statErr := root.Lstat(e.HTML)
root.Close()
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
verified = true
break
}
// An exhausted recovery job is terminal: the visit continues with its
// recorded partial/unavailable evidence (still repairable later)
// rather than holding the run and its workspace forever.
exhausted := job.Status == "exhausted"
verified := exhausted && saved != nil
if !exhausted {
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
if err != nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
r.mu.Unlock()
continue
}
verified = saved != nil && r.verifiedExportRepair(id, sequence, saved, owner)
}
if !verified {
r.mu.Unlock()
@@ -91,6 +72,9 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
r.mu.Unlock()
continue
}
if exhausted {
log.Printf("run %d visit %d: session export recovery exhausted (%s); continuing with incomplete session evidence", id, sequence, job.Diagnostic)
}
step := *saved
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
@@ -98,3 +82,34 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
}
return nil
}
// verifiedExportRepair reports whether a complete repair of the exact owner is
// present on disk with its recorded size.
func (r *runner) verifiedExportRepair(id int64, sequence int, saved *protocol.RunStep, owner pipeline.SessionOwner) bool {
for _, e := range saved.SessionExportRepairs {
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
continue
}
// New repairs stay private in the original visit. Historical
// public repairs remain valid, but are never rewritten there.
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
if strings.HasPrefix(e.HTML, "public/") {
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
continue
}
if filepath.Clean(e.HTML) != e.HTML {
continue
}
root, openErr := os.OpenRoot(directory)
if openErr != nil {
continue
}
info, statErr := root.Lstat(e.HTML)
root.Close()
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
return true
}
}
return false
}
+20 -2
View File
@@ -2,6 +2,7 @@ package server
import (
"context"
"fmt"
"log"
"strings"
"time"
@@ -28,7 +29,11 @@ func (r *runner) startSessionExportJobs() error {
defer cancel()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
nextReconcile := time.Now().Add(30 * time.Second)
// The first tick (one second after startup, once the handler finished
// wiring the runner) reconciles too, so a gate whose job was already
// exhausted, or overran its bounded wait, before a restart is released
// at once instead of reserving its workspace.
nextReconcile := time.Now()
for {
select {
case <-ctx.Done():
@@ -37,7 +42,7 @@ func (r *runner) startSessionExportJobs() error {
return
case <-ticker.C:
}
if time.Now().After(nextReconcile) {
if !time.Now().Before(nextReconcile) {
nextReconcile = time.Now().Add(30 * time.Second)
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
@@ -46,6 +51,14 @@ func (r *runner) startSessionExportJobs() error {
log.Printf("session export recovery reconciliation failed: %v", err)
continue
}
if expired, err := r.store.ExpireSessionExportGates(ctx, time.Now(), pipeline.SessionExportGateMaxWait); err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export wait expiry failed: %v", err)
} else if expired > 0 {
log.Printf("session export recovery stopped for %d visit(s) that exceeded the %s wait", expired, pipeline.SessionExportGateMaxWait)
}
}
// Serial bounded batches; each readback has a 10-second budget.
for range 8 {
@@ -107,6 +120,11 @@ func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bo
diagnostic = last.Diagnostic
if diagnostic == "" && last.Status != "complete" {
diagnostic = "Automatic native session export remains " + last.Status
if last.Truncated {
// The usual cause is inline attachment payloads (for
// example images read by a tool) inside the native record.
diagnostic += fmt.Sprintf(": the native session exceeds the %d MiB export bound, so a retry cannot complete it", stage.SessionExportSourceLimit>>20)
}
}
}
}