diff --git a/docs/cli.md b/docs/cli.md index 4242d51d..9e1a1112 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1015,3 +1015,10 @@ run settles: up to three durable runtime attempts, with backoff and restart-safe leases. `inspect RUN_ID --json` includes each visit's `session_export_recovery` status/attempt count/diagnostic and preserved `session_export_repairs`. Exhausted automatic recovery remains a warning; manual versioned repair is still available. +A run parked on an export handoff never waits beyond that: once its job is +exhausted (or the wait exceeds 15 minutes with the job pending or past its +lease), the visit keeps its partial/unavailable evidence and diagnostic, the run +continues to its next stage or terminal, and its workspace is released. The +server applies this at startup and every 30 seconds, so a run parked before an +upgrade recovers after the next restart. A native session larger than the 8 MiB +export bound (typically inline image payloads read by a tool) is always partial. diff --git a/docs/first-version-testing.md b/docs/first-version-testing.md index 408f286f..8cd2b3a4 100644 --- a/docs/first-version-testing.md +++ b/docs/first-version-testing.md @@ -47,7 +47,8 @@ only**, not credentials, projects, sessions, drafts, or history. Model stages settle their exact native session and verify its export before handoff. An export failure retains computed output and waits for export-only -recovery. **Retry export only** reads that session again; it does not rerun the +recovery; when that bounded recovery is exhausted the run continues with the +incomplete export recorded as a warning. **Retry export only** reads that session again; it does not rerun the stage. A genuinely missing historical conversation cannot be reconstructed by inventing messages. diff --git a/internal/pipeline/runs_mutate.go b/internal/pipeline/runs_mutate.go index ee598c02..e3176632 100644 --- a/internal/pipeline/runs_mutate.go +++ b/internal/pipeline/runs_mutate.go @@ -58,7 +58,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin } defer tx.Rollback() if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0 -AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'')<>'complete')`, key, id)); err != nil { +AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'') NOT IN ('complete','exhausted'))`, key, id)); err != nil { return err } if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil { @@ -396,8 +396,9 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil { return err } - // Terminal targets have no successor BeginStep to consume their gate. - if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status='complete')`, id); err != nil { + // Terminal targets have no successor BeginStep to consume their gate. An + // exhausted job is terminal too: its incomplete evidence stays recorded. + if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status IN ('complete','exhausted'))`, id); err != nil { return err } return tx.Commit() diff --git a/internal/pipeline/session_export_gates.go b/internal/pipeline/session_export_gates.go index 14c73d7b..18229935 100644 --- a/internal/pipeline/session_export_gates.go +++ b/internal/pipeline/session_export_gates.go @@ -1,6 +1,13 @@ package pipeline -import "context" +import ( + "context" + "encoding/json" + "fmt" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) func (s *Store) SessionExportBlocked(ctx context.Context, id int64) (bool, error) { var blocked bool @@ -47,8 +54,8 @@ func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, err rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence JOIN pipeline_runs r ON r.id=g.run_id -WHERE j.status='complete' AND r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0 -AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`) +WHERE r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0 +AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+releasableGateJob+` LIMIT 64`) if err != nil { return nil, err } @@ -65,10 +72,93 @@ AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_ac return ready, rows.Err() } -// ResumeSessionExportGate is a CAS, not a retry of the model invocation. +// ResumeSessionExportGate is a CAS, not a retry of the model invocation. A +// gate whose recovery job is exhausted is released exactly like a completed +// one: the visit keeps its partial/unavailable export evidence and the job's +// diagnostic, and the operator can still repair it later. When no export +// evidence was recorded at all, one unavailable record carries the diagnostic. func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error { - return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0 -AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`) -AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence)) +AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND `+releasableGateJob+`) +AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence)); err != nil { + return err + } + var job SessionExportJob + if err := tx.QueryRowContext(ctx, `SELECT session_id,backend,directory,status,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.SessionID, &job.Backend, &job.Directory, &job.Status, &job.Diagnostic); err != nil { + return err + } + if job.Status == "exhausted" { + var recorded bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM step_session_exports WHERE run_id=? AND sequence=?) OR EXISTS(SELECT 1 FROM session_export_repairs WHERE run_id=? AND sequence=?)`, id, sequence, id, sequence).Scan(&recorded); err != nil { + return err + } + if !recorded { + diagnostic := job.Diagnostic + if diagnostic == "" { + diagnostic = "Automatic session export recovery exhausted" + } + body, err := json.Marshal(protocol.SessionExport{Status: "unavailable", Source: "unavailable", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339), Diagnostic: diagnostic}) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,?,?)", id, sequence, string(body)); err != nil { + return err + } + } + } + return tx.Commit() +} + +// releasableGateJob is a gate's terminal recovery state. A complete job must +// still match the exact original owner. An exhausted job is terminal whatever +// the reason (including changed ownership): waiting longer cannot help, so the +// handoff proceeds with the incomplete evidence instead of wedging the run and +// its workspace forever. +const releasableGateJob = `(j.status='exhausted' OR (j.status='complete' AND ` + exactExportJobOwner + `))` + +// SessionExportGateMaxWait bounds how long a computed visit may wait for its +// export. The normal budget (three attempts with 45-second leases and up to a +// minute of backoff) finishes in a few minutes; this only catches a job that +// can never be claimed or acknowledged. +const SessionExportGateMaxWait = 15 * time.Minute + +// ExpireSessionExportGates exhausts the recovery job of every gate older than +// maxWait whose job is still pending, or running past its lease, and gives a +// gate with no recovery job at all an exhausted one. A live lease is never +// stolen; it expires within its own bound and is exhausted on the next pass. +func (s *Store) ExpireSessionExportGates(ctx context.Context, now time.Time, maxWait time.Duration) (int64, error) { + cutoff := now.Add(-maxWait).UTC().Format(time.DateTime) + diagnostic := fmt.Sprintf("Session export wait exceeded %s; automatic recovery stopped", maxWait) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0, +diagnostic=substr(?||CASE WHEN j.diagnostic<>'' THEN ' (last: '||j.diagnostic||')' ELSE '' END,1,2048) +WHERE (j.status='pending' OR (j.status='running' AND j.lease_until<=?)) +AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence +WHERE g.run_id=j.run_id AND g.sequence=j.sequence AND s.finished_at IS NOT NULL AND s.finished_at<=?)`, diagnostic, now.UnixMilli(), cutoff) + if err != nil { + return 0, err + } + expired, _ := result.RowsAffected() + result, err = tx.ExecContext(ctx, `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic) +SELECT g.run_id,g.sequence,s.session_id,COALESCE(o.backend,''),r.workspace,COALESCE(o.reference,''),'exhausted',0,0,? +FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence JOIN pipeline_runs r ON r.id=g.run_id +LEFT JOIN run_session_owners o ON o.run_id=g.run_id AND o.sequence=g.sequence +WHERE s.finished_at IS NOT NULL AND s.finished_at<=? +AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=g.run_id AND j.sequence=g.sequence) +ON CONFLICT(run_id,sequence) DO NOTHING`, "No automatic session export recovery exists for this visit; "+diagnostic, cutoff) + if err != nil { + return 0, err + } + created, _ := result.RowsAffected() + return expired + created, tx.Commit() } diff --git a/internal/pipeline/session_export_gates_test.go b/internal/pipeline/session_export_gates_test.go index d2fad153..58b41a61 100644 --- a/internal/pipeline/session_export_gates_test.go +++ b/internal/pipeline/session_export_gates_test.go @@ -2,6 +2,7 @@ package pipeline import ( "context" + "strings" "testing" "time" @@ -73,3 +74,83 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) { t.Fatal("successor started twice") } } + +// A recovery job wedged in running past its lease (or pending and never +// claimable) is bounded by the gate's overall wait; a live lease is not stolen. +func TestSessionExportGateWaitIsBoundedForStuckJob(t *testing.T) { + s, id, _ := exportJobFixture(t) + ctx := context.Background() + step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "partial", SessionID: "ses_original", Backend: "v2", Truncated: true}} + if err := s.FinishStepExportBlocked(ctx, id, step); err != nil { + t.Fatal(err) + } + job, _ := s.SessionExportJob(ctx, id, 1) + start := time.UnixMilli(job.NextAt) + if claimed, err := s.ClaimSessionExportJob(ctx, start, time.Hour); err != nil || claimed == nil { + t.Fatalf("claim: %+v %v", claimed, err) + } + if n, err := s.ExpireSessionExportGates(ctx, start.Add(5*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 { + t.Fatalf("young gate expired: %d %v", n, err) + } + if n, err := s.ExpireSessionExportGates(ctx, start.Add(20*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 { + t.Fatalf("live lease stolen: %d %v", n, err) + } + if ready, _ := s.ReadySessionExportGates(ctx); ready[id] != 0 { + t.Fatal("running job released its gate") + } + if n, err := s.ExpireSessionExportGates(ctx, start.Add(2*time.Hour), SessionExportGateMaxWait); err != nil || n != 1 { + t.Fatalf("stuck job not bounded: %d %v", n, err) + } + job, _ = s.SessionExportJob(ctx, id, 1) + if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "wait exceeded") { + t.Fatalf("job not exhausted: %+v", job) + } + ready, err := s.ReadySessionExportGates(ctx) + if err != nil || ready[id] != 1 { + t.Fatalf("exhausted gate not ready: %v %v", ready, err) + } + if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil { + t.Fatal(err) + } + if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { + t.Fatal(err) + } + if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked { + t.Fatalf("terminal finish kept exhausted gate: %v %v", blocked, err) + } + d, _ := s.GetRun(ctx, id) + if d.Status != "succeeded" || d.Steps[0].SessionExport == nil || d.Steps[0].SessionExport.Status != "partial" || len(d.Steps[0].SessionExportRepairs) != 0 { + t.Fatalf("partial evidence not retained as-is: %+v", d) + } +} + +// A gate whose visit never got a recovery job (no exact owner) is given an +// exhausted job after the bounded wait, and the release records why. +func TestSessionExportGateWithoutJobIsReleasedAfterWait(t *testing.T) { + s, id, _ := exportJobFixture(t) + ctx := context.Background() + if _, err := s.db.Exec(`DELETE FROM run_session_owners WHERE run_id=?`, id); err != nil { + t.Fatal(err) + } + step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}} + if err := s.FinishStepExportBlocked(ctx, id, step); err != nil { + t.Fatal(err) + } + if job, _ := s.SessionExportJob(ctx, id, 1); job != nil { + t.Fatalf("fixture unexpectedly has a job: %+v", job) + } + if n, err := s.ExpireSessionExportGates(ctx, time.Now().Add(time.Hour), SessionExportGateMaxWait); err != nil || n != 1 { + t.Fatalf("jobless gate not bounded: %d %v", n, err) + } + if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil { + t.Fatal(err) + } + if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil { + t.Fatalf("successor refused after release: %v", err) + } + d, _ := s.GetRun(ctx, id) + repairs := d.Steps[0].SessionExportRepairs + if len(repairs) != 1 || repairs[0].Status != "unavailable" || !strings.Contains(repairs[0].Diagnostic, "No automatic session export recovery") { + t.Fatalf("release did not record unavailable evidence: %+v", repairs) + } +} diff --git a/internal/server/session_export_exhausted_test.go b/internal/server/session_export_exhausted_test.go new file mode 100644 index 00000000..c401081c --- /dev/null +++ b/internal/server/session_export_exhausted_test.go @@ -0,0 +1,146 @@ +package server + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// truncatedGateExport mirrors the live failure: a native session larger than +// the export bound always renders a truncated, partial file. +func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) { + result, err := validGateExport(ctx, req) + result.Status, result.Truncated = "partial", true + return result, err +} + +// exhaustExportJob spends the whole recovery budget of visit 1 on the runner. +func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) { + t.Helper() + ctx := context.Background() + for range pipeline.SessionExportJobAttempts { + job, err := s.SessionExportJob(ctx, id, 1) + if err != nil || job == nil { + t.Fatalf("missing recovery job: %v", err) + } + if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked { + t.Fatalf("recovery attempt not run: %v %v", worked, err) + } + } + job, _ := s.SessionExportJob(ctx, id, 1) + if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") { + t.Fatalf("job not exhausted with the truncation reason: %+v", job) + } +} + +func workspaceFree(r *runner, workspace string) error { + r.mu.Lock() + defer r.mu.Unlock() + return r.checkWorkspaceFree(workspace) +} + +func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) { + h, s, id, first, second := gateRunner(t, truncatedGateExport, false) + h.runner.wg.Wait() + ctx := context.Background() + d, _ := s.GetRun(ctx, id) + if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 { + t.Fatalf("partial export did not park the handoff: %+v", d) + } + if err := workspaceFree(h.runner, d.Workspace); err == nil { + t.Fatal("parked export did not reserve its workspace") + } + exhaustExportJob(t, h.runner, s, id) + if err := h.runner.resumeExportGates(ctx); err != nil { + t.Fatal(err) + } + h.runner.wg.Wait() + d, _ = s.GetRun(ctx, id) + if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 { + t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load()) + } + if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked { + t.Fatalf("gate survived: %v %v", blocked, err) + } + visit := d.Steps[0] + if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" { + t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery) + } + // The workspace is free for the next (for example scheduled) run. + if err := workspaceFree(h.runner, d.Workspace); err != nil { + t.Fatalf("workspace still reserved: %v", err) + } + // The operator can still repair the export later through the normal path. + h.runner.exportSession = validGateExport + repaired, err := h.runner.repairSessionExport(ctx, id, 1) + if err != nil { + t.Fatal(err) + } + repairs := repaired.Steps[0].SessionExportRepairs + if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" { + t.Fatalf("later repair not recorded: %+v", repairs) + } + if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil { + t.Fatal(err) + } + if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" { + t.Fatalf("successful repair did not complete the job: %+v", job) + } +} + +// A run parked before the fix, whose job was already exhausted, is released by +// the restarted server's startup reconciliation without any manual edit. +func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) { + h, s, id, first, second := gateRunner(t, truncatedGateExport, false) + h.runner.wg.Wait() + exhaustExportJob(t, h.runner, s, id) + path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := pipeline.Open(path) + if err != nil { + t.Fatal(err) + } + parked, _ := reopened.GetRun(context.Background(), id) + if parked.Status != "waiting" || parked.ExportPendingSequence != 1 { + t.Fatalf("fixture is not a parked export: %+v", parked) + } + ctx, cancel := context.WithCancel(context.Background()) + r, err := newRunner(ctx, reopened) + if err != nil { + t.Fatal(err) + } + defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }() + r.artifactsRoot = root + r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { + if def.Key != "two" { + t.Errorf("replayed owner: %s", def.Key) + } + second.Add(1) + return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil + } + deadline := time.Now().Add(10 * time.Second) + for { + d, _ := reopened.GetRun(context.Background(), id) + if d.Status == "succeeded" { + break + } + if time.Now().After(deadline) { + t.Fatalf("startup did not release the exhausted gate: %+v", d) + } + time.Sleep(20 * time.Millisecond) + } + r.wg.Wait() + if first.Load() != 1 || second.Load() != 1 { + t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load()) + } + if err := workspaceFree(r, parked.Workspace); err != nil { + t.Fatalf("workspace still reserved after startup release: %v", err) + } +} diff --git a/internal/server/session_export_gate.go b/internal/server/session_export_gate.go index fe7b6746..6f09f238 100644 --- a/internal/server/session_export_gate.go +++ b/internal/server/session_export_gate.go @@ -2,11 +2,13 @@ package server import ( "context" + "log" "os" "path/filepath" "strconv" "strings" + "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) @@ -37,13 +39,8 @@ func (r *runner) resumeExportGates(ctx context.Context) error { r.mu.Unlock() continue } - owner, err := r.store.RepairSessionOwner(ctx, id, sequence) - if err != nil { - r.mu.Unlock() - continue - } job, err := r.store.SessionExportJob(ctx, id, sequence) - if err != nil || job == nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference { + if err != nil || job == nil { r.mu.Unlock() continue } @@ -54,34 +51,18 @@ func (r *runner) resumeExportGates(ctx context.Context) error { break } } - verified := false - if saved != nil { - for _, e := range saved.SessionExportRepairs { - if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory { - continue - } - // New repairs stay private in the original visit. Historical - // public repairs remain valid, but are never rewritten there. - directory := r.turnPaths(id, sequence, saved.Stage).TurnDir - if strings.HasPrefix(e.HTML, "public/") { - directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10)) - } else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") { - continue - } - if filepath.Clean(e.HTML) != e.HTML { - continue - } - root, openErr := os.OpenRoot(directory) - if openErr != nil { - continue - } - info, statErr := root.Lstat(e.HTML) - root.Close() - if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit { - verified = true - break - } + // An exhausted recovery job is terminal: the visit continues with its + // recorded partial/unavailable evidence (still repairable later) + // rather than holding the run and its workspace forever. + exhausted := job.Status == "exhausted" + verified := exhausted && saved != nil + if !exhausted { + owner, err := r.store.RepairSessionOwner(ctx, id, sequence) + if err != nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference { + r.mu.Unlock() + continue } + verified = saved != nil && r.verifiedExportRepair(id, sequence, saved, owner) } if !verified { r.mu.Unlock() @@ -91,6 +72,9 @@ func (r *runner) resumeExportGates(ctx context.Context) error { r.mu.Unlock() continue } + if exhausted { + log.Printf("run %d visit %d: session export recovery exhausted (%s); continuing with incomplete session evidence", id, sequence, job.Diagnostic) + } step := *saved r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true) r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) }) @@ -98,3 +82,34 @@ func (r *runner) resumeExportGates(ctx context.Context) error { } return nil } + +// verifiedExportRepair reports whether a complete repair of the exact owner is +// present on disk with its recorded size. +func (r *runner) verifiedExportRepair(id int64, sequence int, saved *protocol.RunStep, owner pipeline.SessionOwner) bool { + for _, e := range saved.SessionExportRepairs { + if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory { + continue + } + // New repairs stay private in the original visit. Historical + // public repairs remain valid, but are never rewritten there. + directory := r.turnPaths(id, sequence, saved.Stage).TurnDir + if strings.HasPrefix(e.HTML, "public/") { + directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10)) + } else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") { + continue + } + if filepath.Clean(e.HTML) != e.HTML { + continue + } + root, openErr := os.OpenRoot(directory) + if openErr != nil { + continue + } + info, statErr := root.Lstat(e.HTML) + root.Close() + if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit { + return true + } + } + return false +} diff --git a/internal/server/session_export_jobs.go b/internal/server/session_export_jobs.go index 9cb69f0b..1ac69a4a 100644 --- a/internal/server/session_export_jobs.go +++ b/internal/server/session_export_jobs.go @@ -2,6 +2,7 @@ package server import ( "context" + "fmt" "log" "strings" "time" @@ -28,7 +29,11 @@ func (r *runner) startSessionExportJobs() error { defer cancel() ticker := time.NewTicker(time.Second) defer ticker.Stop() - nextReconcile := time.Now().Add(30 * time.Second) + // The first tick (one second after startup, once the handler finished + // wiring the runner) reconciles too, so a gate whose job was already + // exhausted, or overran its bounded wait, before a restart is released + // at once instead of reserving its workspace. + nextReconcile := time.Now() for { select { case <-ctx.Done(): @@ -37,7 +42,7 @@ func (r *runner) startSessionExportJobs() error { return case <-ticker.C: } - if time.Now().After(nextReconcile) { + if !time.Now().Before(nextReconcile) { nextReconcile = time.Now().Add(30 * time.Second) if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil { if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") { @@ -46,6 +51,14 @@ func (r *runner) startSessionExportJobs() error { log.Printf("session export recovery reconciliation failed: %v", err) continue } + if expired, err := r.store.ExpireSessionExportGates(ctx, time.Now(), pipeline.SessionExportGateMaxWait); err != nil { + if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") { + return + } + log.Printf("session export wait expiry failed: %v", err) + } else if expired > 0 { + log.Printf("session export recovery stopped for %d visit(s) that exceeded the %s wait", expired, pipeline.SessionExportGateMaxWait) + } } // Serial bounded batches; each readback has a 10-second budget. for range 8 { @@ -107,6 +120,11 @@ func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bo diagnostic = last.Diagnostic if diagnostic == "" && last.Status != "complete" { diagnostic = "Automatic native session export remains " + last.Status + if last.Truncated { + // The usual cause is inline attachment payloads (for + // example images read by a tool) inside the native record. + diagnostic += fmt.Sprintf(": the native session exceeds the %d MiB export bound, so a retry cannot complete it", stage.SessionExportSourceLimit>>20) + } } } }