Merge: runs no longer wedge on an exhausted session export

This commit is contained in:
2026-11-01 17:27:05 +00:00
8 changed files with 405 additions and 46 deletions
+7
View File
@@ -1015,3 +1015,10 @@ run settles: up to three durable runtime attempts, with backoff and restart-safe
leases. `inspect RUN_ID --json` includes each visit's `session_export_recovery`
status/attempt count/diagnostic and preserved `session_export_repairs`. Exhausted
automatic recovery remains a warning; manual versioned repair is still available.
A run parked on an export handoff never waits beyond that: once its job is
exhausted (or the wait exceeds 15 minutes with the job pending or past its
lease), the visit keeps its partial/unavailable evidence and diagnostic, the run
continues to its next stage or terminal, and its workspace is released. The
server applies this at startup and every 30 seconds, so a run parked before an
upgrade recovers after the next restart. A native session larger than the 8 MiB
export bound (typically inline image payloads read by a tool) is always partial.
+2 -1
View File
@@ -47,7 +47,8 @@ only**, not credentials, projects, sessions, drafts, or history.
Model stages settle their exact native session and verify its export before
handoff. An export failure retains computed output and waits for export-only
recovery. **Retry export only** reads that session again; it does not rerun the
recovery; when that bounded recovery is exhausted the run continues with the
incomplete export recorded as a warning. **Retry export only** reads that session again; it does not rerun the
stage. A genuinely missing historical conversation cannot be reconstructed by
inventing messages.
+4 -3
View File
@@ -58,7 +58,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'')<>'complete')`, key, id)); err != nil {
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'') NOT IN ('complete','exhausted'))`, key, id)); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
@@ -396,8 +396,9 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
return err
}
// Terminal targets have no successor BeginStep to consume their gate.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status='complete')`, id); err != nil {
// Terminal targets have no successor BeginStep to consume their gate. An
// exhausted job is terminal too: its incomplete evidence stays recorded.
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status IN ('complete','exhausted'))`, id); err != nil {
return err
}
return tx.Commit()
+97 -7
View File
@@ -1,6 +1,13 @@
package pipeline
import "context"
import (
"context"
"encoding/json"
"fmt"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func (s *Store) SessionExportBlocked(ctx context.Context, id int64) (bool, error) {
var blocked bool
@@ -47,8 +54,8 @@ func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, err
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
JOIN pipeline_runs r ON r.id=g.run_id
WHERE j.status='complete' AND r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`)
WHERE r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+releasableGateJob+` LIMIT 64`)
if err != nil {
return nil, err
}
@@ -65,10 +72,93 @@ AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_ac
return ready, rows.Err()
}
// ResumeSessionExportGate is a CAS, not a retry of the model invocation.
// ResumeSessionExportGate is a CAS, not a retry of the model invocation. A
// gate whose recovery job is exhausted is released exactly like a completed
// one: the visit keeps its partial/unavailable export evidence and the job's
// diagnostic, and the operator can still repair it later. When no export
// evidence was recorded at all, one unavailable record carries the diagnostic.
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`)
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence))
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND `+releasableGateJob+`)
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence)); err != nil {
return err
}
var job SessionExportJob
if err := tx.QueryRowContext(ctx, `SELECT session_id,backend,directory,status,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.SessionID, &job.Backend, &job.Directory, &job.Status, &job.Diagnostic); err != nil {
return err
}
if job.Status == "exhausted" {
var recorded bool
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM step_session_exports WHERE run_id=? AND sequence=?) OR EXISTS(SELECT 1 FROM session_export_repairs WHERE run_id=? AND sequence=?)`, id, sequence, id, sequence).Scan(&recorded); err != nil {
return err
}
if !recorded {
diagnostic := job.Diagnostic
if diagnostic == "" {
diagnostic = "Automatic session export recovery exhausted"
}
body, err := json.Marshal(protocol.SessionExport{Status: "unavailable", Source: "unavailable", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339), Diagnostic: diagnostic})
if err != nil {
return err
}
if _, err := tx.ExecContext(ctx, "INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,?,?)", id, sequence, string(body)); err != nil {
return err
}
}
}
return tx.Commit()
}
// releasableGateJob is a gate's terminal recovery state. A complete job must
// still match the exact original owner. An exhausted job is terminal whatever
// the reason (including changed ownership): waiting longer cannot help, so the
// handoff proceeds with the incomplete evidence instead of wedging the run and
// its workspace forever.
const releasableGateJob = `(j.status='exhausted' OR (j.status='complete' AND ` + exactExportJobOwner + `))`
// SessionExportGateMaxWait bounds how long a computed visit may wait for its
// export. The normal budget (three attempts with 45-second leases and up to a
// minute of backoff) finishes in a few minutes; this only catches a job that
// can never be claimed or acknowledged.
const SessionExportGateMaxWait = 15 * time.Minute
// ExpireSessionExportGates exhausts the recovery job of every gate older than
// maxWait whose job is still pending, or running past its lease, and gives a
// gate with no recovery job at all an exhausted one. A live lease is never
// stolen; it expires within its own bound and is exhausted on the next pass.
func (s *Store) ExpireSessionExportGates(ctx context.Context, now time.Time, maxWait time.Duration) (int64, error) {
cutoff := now.Add(-maxWait).UTC().Format(time.DateTime)
diagnostic := fmt.Sprintf("Session export wait exceeded %s; automatic recovery stopped", maxWait)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return 0, err
}
defer tx.Rollback()
result, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,
diagnostic=substr(?||CASE WHEN j.diagnostic<>'' THEN ' (last: '||j.diagnostic||')' ELSE '' END,1,2048)
WHERE (j.status='pending' OR (j.status='running' AND j.lease_until<=?))
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
WHERE g.run_id=j.run_id AND g.sequence=j.sequence AND s.finished_at IS NOT NULL AND s.finished_at<=?)`, diagnostic, now.UnixMilli(), cutoff)
if err != nil {
return 0, err
}
expired, _ := result.RowsAffected()
result, err = tx.ExecContext(ctx, `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic)
SELECT g.run_id,g.sequence,s.session_id,COALESCE(o.backend,''),r.workspace,COALESCE(o.reference,''),'exhausted',0,0,?
FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence JOIN pipeline_runs r ON r.id=g.run_id
LEFT JOIN run_session_owners o ON o.run_id=g.run_id AND o.sequence=g.sequence
WHERE s.finished_at IS NOT NULL AND s.finished_at<=?
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=g.run_id AND j.sequence=g.sequence)
ON CONFLICT(run_id,sequence) DO NOTHING`, "No automatic session export recovery exists for this visit; "+diagnostic, cutoff)
if err != nil {
return 0, err
}
created, _ := result.RowsAffected()
return expired + created, tx.Commit()
}
@@ -2,6 +2,7 @@ package pipeline
import (
"context"
"strings"
"testing"
"time"
@@ -73,3 +74,83 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
t.Fatal("successor started twice")
}
}
// A recovery job wedged in running past its lease (or pending and never
// claimable) is bounded by the gate's overall wait; a live lease is not stolen.
func TestSessionExportGateWaitIsBoundedForStuckJob(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "partial", SessionID: "ses_original", Backend: "v2", Truncated: true}}
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
t.Fatal(err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
start := time.UnixMilli(job.NextAt)
if claimed, err := s.ClaimSessionExportJob(ctx, start, time.Hour); err != nil || claimed == nil {
t.Fatalf("claim: %+v %v", claimed, err)
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(5*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
t.Fatalf("young gate expired: %d %v", n, err)
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(20*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
t.Fatalf("live lease stolen: %d %v", n, err)
}
if ready, _ := s.ReadySessionExportGates(ctx); ready[id] != 0 {
t.Fatal("running job released its gate")
}
if n, err := s.ExpireSessionExportGates(ctx, start.Add(2*time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
t.Fatalf("stuck job not bounded: %d %v", n, err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "wait exceeded") {
t.Fatalf("job not exhausted: %+v", job)
}
ready, err := s.ReadySessionExportGates(ctx)
if err != nil || ready[id] != 1 {
t.Fatalf("exhausted gate not ready: %v %v", ready, err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("terminal finish kept exhausted gate: %v %v", blocked, err)
}
d, _ := s.GetRun(ctx, id)
if d.Status != "succeeded" || d.Steps[0].SessionExport == nil || d.Steps[0].SessionExport.Status != "partial" || len(d.Steps[0].SessionExportRepairs) != 0 {
t.Fatalf("partial evidence not retained as-is: %+v", d)
}
}
// A gate whose visit never got a recovery job (no exact owner) is given an
// exhausted job after the bounded wait, and the release records why.
func TestSessionExportGateWithoutJobIsReleasedAfterWait(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
if _, err := s.db.Exec(`DELETE FROM run_session_owners WHERE run_id=?`, id); err != nil {
t.Fatal(err)
}
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}}
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
t.Fatal(err)
}
if job, _ := s.SessionExportJob(ctx, id, 1); job != nil {
t.Fatalf("fixture unexpectedly has a job: %+v", job)
}
if n, err := s.ExpireSessionExportGates(ctx, time.Now().Add(time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
t.Fatalf("jobless gate not bounded: %d %v", n, err)
}
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
t.Fatalf("successor refused after release: %v", err)
}
d, _ := s.GetRun(ctx, id)
repairs := d.Steps[0].SessionExportRepairs
if len(repairs) != 1 || repairs[0].Status != "unavailable" || !strings.Contains(repairs[0].Diagnostic, "No automatic session export recovery") {
t.Fatalf("release did not record unavailable evidence: %+v", repairs)
}
}
@@ -0,0 +1,146 @@
package server
import (
"context"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// truncatedGateExport mirrors the live failure: a native session larger than
// the export bound always renders a truncated, partial file.
func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
result, err := validGateExport(ctx, req)
result.Status, result.Truncated = "partial", true
return result, err
}
// exhaustExportJob spends the whole recovery budget of visit 1 on the runner.
func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) {
t.Helper()
ctx := context.Background()
for range pipeline.SessionExportJobAttempts {
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil {
t.Fatalf("missing recovery job: %v", err)
}
if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("recovery attempt not run: %v %v", worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") {
t.Fatalf("job not exhausted with the truncation reason: %+v", job)
}
}
func workspaceFree(r *runner, workspace string) error {
r.mu.Lock()
defer r.mu.Unlock()
return r.checkWorkspaceFree(workspace)
}
func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) {
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 {
t.Fatalf("partial export did not park the handoff: %+v", d)
}
if err := workspaceFree(h.runner, d.Workspace); err == nil {
t.Fatal("parked export did not reserve its workspace")
}
exhaustExportJob(t, h.runner, s, id)
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ = s.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 {
t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load())
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("gate survived: %v %v", blocked, err)
}
visit := d.Steps[0]
if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" {
t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery)
}
// The workspace is free for the next (for example scheduled) run.
if err := workspaceFree(h.runner, d.Workspace); err != nil {
t.Fatalf("workspace still reserved: %v", err)
}
// The operator can still repair the export later through the normal path.
h.runner.exportSession = validGateExport
repaired, err := h.runner.repairSessionExport(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
repairs := repaired.Steps[0].SessionExportRepairs
if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" {
t.Fatalf("later repair not recorded: %+v", repairs)
}
if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil {
t.Fatal(err)
}
if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" {
t.Fatalf("successful repair did not complete the job: %+v", job)
}
}
// A run parked before the fix, whose job was already exhausted, is released by
// the restarted server's startup reconciliation without any manual edit.
func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
h.runner.wg.Wait()
exhaustExportJob(t, h.runner, s, id)
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
if err := s.Close(); err != nil {
t.Fatal(err)
}
reopened, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
parked, _ := reopened.GetRun(context.Background(), id)
if parked.Status != "waiting" || parked.ExportPendingSequence != 1 {
t.Fatalf("fixture is not a parked export: %+v", parked)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunner(ctx, reopened)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.artifactsRoot = root
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
t.Errorf("replayed owner: %s", def.Key)
}
second.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
deadline := time.Now().Add(10 * time.Second)
for {
d, _ := reopened.GetRun(context.Background(), id)
if d.Status == "succeeded" {
break
}
if time.Now().After(deadline) {
t.Fatalf("startup did not release the exhausted gate: %+v", d)
}
time.Sleep(20 * time.Millisecond)
}
r.wg.Wait()
if first.Load() != 1 || second.Load() != 1 {
t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load())
}
if err := workspaceFree(r, parked.Workspace); err != nil {
t.Fatalf("workspace still reserved after startup release: %v", err)
}
}
+48 -33
View File
@@ -2,11 +2,13 @@ package server
import (
"context"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
@@ -37,13 +39,8 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
r.mu.Unlock()
continue
}
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
if err != nil {
r.mu.Unlock()
continue
}
job, err := r.store.SessionExportJob(ctx, id, sequence)
if err != nil || job == nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
if err != nil || job == nil {
r.mu.Unlock()
continue
}
@@ -54,34 +51,18 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
break
}
}
verified := false
if saved != nil {
for _, e := range saved.SessionExportRepairs {
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
continue
}
// New repairs stay private in the original visit. Historical
// public repairs remain valid, but are never rewritten there.
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
if strings.HasPrefix(e.HTML, "public/") {
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
continue
}
if filepath.Clean(e.HTML) != e.HTML {
continue
}
root, openErr := os.OpenRoot(directory)
if openErr != nil {
continue
}
info, statErr := root.Lstat(e.HTML)
root.Close()
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
verified = true
break
}
// An exhausted recovery job is terminal: the visit continues with its
// recorded partial/unavailable evidence (still repairable later)
// rather than holding the run and its workspace forever.
exhausted := job.Status == "exhausted"
verified := exhausted && saved != nil
if !exhausted {
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
if err != nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
r.mu.Unlock()
continue
}
verified = saved != nil && r.verifiedExportRepair(id, sequence, saved, owner)
}
if !verified {
r.mu.Unlock()
@@ -91,6 +72,9 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
r.mu.Unlock()
continue
}
if exhausted {
log.Printf("run %d visit %d: session export recovery exhausted (%s); continuing with incomplete session evidence", id, sequence, job.Diagnostic)
}
step := *saved
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
@@ -98,3 +82,34 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
}
return nil
}
// verifiedExportRepair reports whether a complete repair of the exact owner is
// present on disk with its recorded size.
func (r *runner) verifiedExportRepair(id int64, sequence int, saved *protocol.RunStep, owner pipeline.SessionOwner) bool {
for _, e := range saved.SessionExportRepairs {
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
continue
}
// New repairs stay private in the original visit. Historical
// public repairs remain valid, but are never rewritten there.
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
if strings.HasPrefix(e.HTML, "public/") {
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
continue
}
if filepath.Clean(e.HTML) != e.HTML {
continue
}
root, openErr := os.OpenRoot(directory)
if openErr != nil {
continue
}
info, statErr := root.Lstat(e.HTML)
root.Close()
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
return true
}
}
return false
}
+20 -2
View File
@@ -2,6 +2,7 @@ package server
import (
"context"
"fmt"
"log"
"strings"
"time"
@@ -28,7 +29,11 @@ func (r *runner) startSessionExportJobs() error {
defer cancel()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
nextReconcile := time.Now().Add(30 * time.Second)
// The first tick (one second after startup, once the handler finished
// wiring the runner) reconciles too, so a gate whose job was already
// exhausted, or overran its bounded wait, before a restart is released
// at once instead of reserving its workspace.
nextReconcile := time.Now()
for {
select {
case <-ctx.Done():
@@ -37,7 +42,7 @@ func (r *runner) startSessionExportJobs() error {
return
case <-ticker.C:
}
if time.Now().After(nextReconcile) {
if !time.Now().Before(nextReconcile) {
nextReconcile = time.Now().Add(30 * time.Second)
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
@@ -46,6 +51,14 @@ func (r *runner) startSessionExportJobs() error {
log.Printf("session export recovery reconciliation failed: %v", err)
continue
}
if expired, err := r.store.ExpireSessionExportGates(ctx, time.Now(), pipeline.SessionExportGateMaxWait); err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export wait expiry failed: %v", err)
} else if expired > 0 {
log.Printf("session export recovery stopped for %d visit(s) that exceeded the %s wait", expired, pipeline.SessionExportGateMaxWait)
}
}
// Serial bounded batches; each readback has a 10-second budget.
for range 8 {
@@ -107,6 +120,11 @@ func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bo
diagnostic = last.Diagnostic
if diagnostic == "" && last.Status != "complete" {
diagnostic = "Automatic native session export remains " + last.Status
if last.Truncated {
// The usual cause is inline attachment payloads (for
// example images read by a tool) inside the native record.
diagnostic += fmt.Sprintf(": the native session exceeds the %d MiB export bound, so a retry cannot complete it", stage.SessionExportSourceLimit>>20)
}
}
}
}