Release session export gates whose recovery is exhausted or overdue
An exhausted export recovery job left its run parked forever in the export wait, reserving the workspace and refusing later scheduled runs (runs 221/226: OpenCode sessions above the 8 MiB export bound are always truncated, so every retry stays partial). A gate is now releasable when its job is exhausted: the visit keeps its partial/unavailable evidence and diagnostic (still repairable), the run continues to its next stage or terminal, and the workspace is freed. Gates older than 15 minutes with a pending, lease-expired or missing job are exhausted by a bounded wait. The worker reconciles on its first tick after startup and every 30 seconds, so existing wedged runs recover after a restart. Truncation diagnostics now name the size bound.
This commit is contained in:
@@ -1015,3 +1015,10 @@ run settles: up to three durable runtime attempts, with backoff and restart-safe
|
||||
leases. `inspect RUN_ID --json` includes each visit's `session_export_recovery`
|
||||
status/attempt count/diagnostic and preserved `session_export_repairs`. Exhausted
|
||||
automatic recovery remains a warning; manual versioned repair is still available.
|
||||
A run parked on an export handoff never waits beyond that: once its job is
|
||||
exhausted (or the wait exceeds 15 minutes with the job pending or past its
|
||||
lease), the visit keeps its partial/unavailable evidence and diagnostic, the run
|
||||
continues to its next stage or terminal, and its workspace is released. The
|
||||
server applies this at startup and every 30 seconds, so a run parked before an
|
||||
upgrade recovers after the next restart. A native session larger than the 8 MiB
|
||||
export bound (typically inline image payloads read by a tool) is always partial.
|
||||
|
||||
@@ -47,7 +47,8 @@ only**, not credentials, projects, sessions, drafts, or history.
|
||||
|
||||
Model stages settle their exact native session and verify its export before
|
||||
handoff. An export failure retains computed output and waits for export-only
|
||||
recovery. **Retry export only** reads that session again; it does not rerun the
|
||||
recovery; when that bounded recovery is exhausted the run continues with the
|
||||
incomplete export recorded as a warning. **Retry export only** reads that session again; it does not rerun the
|
||||
stage. A genuinely missing historical conversation cannot be reconstructed by
|
||||
inventing messages.
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0
|
||||
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'')<>'complete')`, key, id)); err != nil {
|
||||
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'') NOT IN ('complete','exhausted'))`, key, id)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
|
||||
@@ -396,8 +396,9 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a
|
||||
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
// Terminal targets have no successor BeginStep to consume their gate.
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status='complete')`, id); err != nil {
|
||||
// Terminal targets have no successor BeginStep to consume their gate. An
|
||||
// exhausted job is terminal too: its incomplete evidence stays recorded.
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status IN ('complete','exhausted'))`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
package pipeline
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func (s *Store) SessionExportBlocked(ctx context.Context, id int64) (bool, error) {
|
||||
var blocked bool
|
||||
@@ -47,8 +54,8 @@ func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, err
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
|
||||
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
|
||||
JOIN pipeline_runs r ON r.id=g.run_id
|
||||
WHERE j.status='complete' AND r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`)
|
||||
WHERE r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+releasableGateJob+` LIMIT 64`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -65,10 +72,93 @@ AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_ac
|
||||
return ready, rows.Err()
|
||||
}
|
||||
|
||||
// ResumeSessionExportGate is a CAS, not a retry of the model invocation.
|
||||
// ResumeSessionExportGate is a CAS, not a retry of the model invocation. A
|
||||
// gate whose recovery job is exhausted is released exactly like a completed
|
||||
// one: the visit keeps its partial/unavailable export evidence and the job's
|
||||
// diagnostic, and the operator can still repair it later. When no export
|
||||
// evidence was recorded at all, one unavailable record carries the diagnostic.
|
||||
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
|
||||
WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
|
||||
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`)
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence))
|
||||
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND `+releasableGateJob+`)
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence)); err != nil {
|
||||
return err
|
||||
}
|
||||
var job SessionExportJob
|
||||
if err := tx.QueryRowContext(ctx, `SELECT session_id,backend,directory,status,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.SessionID, &job.Backend, &job.Directory, &job.Status, &job.Diagnostic); err != nil {
|
||||
return err
|
||||
}
|
||||
if job.Status == "exhausted" {
|
||||
var recorded bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM step_session_exports WHERE run_id=? AND sequence=?) OR EXISTS(SELECT 1 FROM session_export_repairs WHERE run_id=? AND sequence=?)`, id, sequence, id, sequence).Scan(&recorded); err != nil {
|
||||
return err
|
||||
}
|
||||
if !recorded {
|
||||
diagnostic := job.Diagnostic
|
||||
if diagnostic == "" {
|
||||
diagnostic = "Automatic session export recovery exhausted"
|
||||
}
|
||||
body, err := json.Marshal(protocol.SessionExport{Status: "unavailable", Source: "unavailable", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339), Diagnostic: diagnostic})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "INSERT INTO session_export_repairs(run_id,sequence,export_json) VALUES(?,?,?)", id, sequence, string(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// releasableGateJob is a gate's terminal recovery state. A complete job must
|
||||
// still match the exact original owner. An exhausted job is terminal whatever
|
||||
// the reason (including changed ownership): waiting longer cannot help, so the
|
||||
// handoff proceeds with the incomplete evidence instead of wedging the run and
|
||||
// its workspace forever.
|
||||
const releasableGateJob = `(j.status='exhausted' OR (j.status='complete' AND ` + exactExportJobOwner + `))`
|
||||
|
||||
// SessionExportGateMaxWait bounds how long a computed visit may wait for its
|
||||
// export. The normal budget (three attempts with 45-second leases and up to a
|
||||
// minute of backoff) finishes in a few minutes; this only catches a job that
|
||||
// can never be claimed or acknowledged.
|
||||
const SessionExportGateMaxWait = 15 * time.Minute
|
||||
|
||||
// ExpireSessionExportGates exhausts the recovery job of every gate older than
|
||||
// maxWait whose job is still pending, or running past its lease, and gives a
|
||||
// gate with no recovery job at all an exhausted one. A live lease is never
|
||||
// stolen; it expires within its own bound and is exhausted on the next pass.
|
||||
func (s *Store) ExpireSessionExportGates(ctx context.Context, now time.Time, maxWait time.Duration) (int64, error) {
|
||||
cutoff := now.Add(-maxWait).UTC().Format(time.DateTime)
|
||||
diagnostic := fmt.Sprintf("Session export wait exceeded %s; automatic recovery stopped", maxWait)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
result, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,
|
||||
diagnostic=substr(?||CASE WHEN j.diagnostic<>'' THEN ' (last: '||j.diagnostic||')' ELSE '' END,1,2048)
|
||||
WHERE (j.status='pending' OR (j.status='running' AND j.lease_until<=?))
|
||||
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
|
||||
WHERE g.run_id=j.run_id AND g.sequence=j.sequence AND s.finished_at IS NOT NULL AND s.finished_at<=?)`, diagnostic, now.UnixMilli(), cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
expired, _ := result.RowsAffected()
|
||||
result, err = tx.ExecContext(ctx, `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic)
|
||||
SELECT g.run_id,g.sequence,s.session_id,COALESCE(o.backend,''),r.workspace,COALESCE(o.reference,''),'exhausted',0,0,?
|
||||
FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence JOIN pipeline_runs r ON r.id=g.run_id
|
||||
LEFT JOIN run_session_owners o ON o.run_id=g.run_id AND o.sequence=g.sequence
|
||||
WHERE s.finished_at IS NOT NULL AND s.finished_at<=?
|
||||
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=g.run_id AND j.sequence=g.sequence)
|
||||
ON CONFLICT(run_id,sequence) DO NOTHING`, "No automatic session export recovery exists for this visit; "+diagnostic, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
created, _ := result.RowsAffected()
|
||||
return expired + created, tx.Commit()
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -73,3 +74,83 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
|
||||
t.Fatal("successor started twice")
|
||||
}
|
||||
}
|
||||
|
||||
// A recovery job wedged in running past its lease (or pending and never
|
||||
// claimable) is bounded by the gate's overall wait; a live lease is not stolen.
|
||||
func TestSessionExportGateWaitIsBoundedForStuckJob(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "partial", SessionID: "ses_original", Backend: "v2", Truncated: true}}
|
||||
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
start := time.UnixMilli(job.NextAt)
|
||||
if claimed, err := s.ClaimSessionExportJob(ctx, start, time.Hour); err != nil || claimed == nil {
|
||||
t.Fatalf("claim: %+v %v", claimed, err)
|
||||
}
|
||||
if n, err := s.ExpireSessionExportGates(ctx, start.Add(5*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
|
||||
t.Fatalf("young gate expired: %d %v", n, err)
|
||||
}
|
||||
if n, err := s.ExpireSessionExportGates(ctx, start.Add(20*time.Minute), SessionExportGateMaxWait); err != nil || n != 0 {
|
||||
t.Fatalf("live lease stolen: %d %v", n, err)
|
||||
}
|
||||
if ready, _ := s.ReadySessionExportGates(ctx); ready[id] != 0 {
|
||||
t.Fatal("running job released its gate")
|
||||
}
|
||||
if n, err := s.ExpireSessionExportGates(ctx, start.Add(2*time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
|
||||
t.Fatalf("stuck job not bounded: %d %v", n, err)
|
||||
}
|
||||
job, _ = s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "wait exceeded") {
|
||||
t.Fatalf("job not exhausted: %+v", job)
|
||||
}
|
||||
ready, err := s.ReadySessionExportGates(ctx)
|
||||
if err != nil || ready[id] != 1 {
|
||||
t.Fatalf("exhausted gate not ready: %v %v", ready, err)
|
||||
}
|
||||
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
|
||||
t.Fatalf("terminal finish kept exhausted gate: %v %v", blocked, err)
|
||||
}
|
||||
d, _ := s.GetRun(ctx, id)
|
||||
if d.Status != "succeeded" || d.Steps[0].SessionExport == nil || d.Steps[0].SessionExport.Status != "partial" || len(d.Steps[0].SessionExportRepairs) != 0 {
|
||||
t.Fatalf("partial evidence not retained as-is: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// A gate whose visit never got a recovery job (no exact owner) is given an
|
||||
// exhausted job after the bounded wait, and the release records why.
|
||||
func TestSessionExportGateWithoutJobIsReleasedAfterWait(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
if _, err := s.db.Exec(`DELETE FROM run_session_owners WHERE run_id=?`, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}}
|
||||
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job, _ := s.SessionExportJob(ctx, id, 1); job != nil {
|
||||
t.Fatalf("fixture unexpectedly has a job: %+v", job)
|
||||
}
|
||||
if n, err := s.ExpireSessionExportGates(ctx, time.Now().Add(time.Hour), SessionExportGateMaxWait); err != nil || n != 1 {
|
||||
t.Fatalf("jobless gate not bounded: %d %v", n, err)
|
||||
}
|
||||
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
|
||||
t.Fatalf("successor refused after release: %v", err)
|
||||
}
|
||||
d, _ := s.GetRun(ctx, id)
|
||||
repairs := d.Steps[0].SessionExportRepairs
|
||||
if len(repairs) != 1 || repairs[0].Status != "unavailable" || !strings.Contains(repairs[0].Diagnostic, "No automatic session export recovery") {
|
||||
t.Fatalf("release did not record unavailable evidence: %+v", repairs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// truncatedGateExport mirrors the live failure: a native session larger than
|
||||
// the export bound always renders a truncated, partial file.
|
||||
func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
result, err := validGateExport(ctx, req)
|
||||
result.Status, result.Truncated = "partial", true
|
||||
return result, err
|
||||
}
|
||||
|
||||
// exhaustExportJob spends the whole recovery budget of visit 1 on the runner.
|
||||
func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
for range pipeline.SessionExportJobAttempts {
|
||||
job, err := s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil || job == nil {
|
||||
t.Fatalf("missing recovery job: %v", err)
|
||||
}
|
||||
if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
||||
t.Fatalf("recovery attempt not run: %v %v", worked, err)
|
||||
}
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") {
|
||||
t.Fatalf("job not exhausted with the truncation reason: %+v", job)
|
||||
}
|
||||
}
|
||||
|
||||
func workspaceFree(r *runner, workspace string) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.checkWorkspaceFree(workspace)
|
||||
}
|
||||
|
||||
func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) {
|
||||
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
|
||||
h.runner.wg.Wait()
|
||||
ctx := context.Background()
|
||||
d, _ := s.GetRun(ctx, id)
|
||||
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 {
|
||||
t.Fatalf("partial export did not park the handoff: %+v", d)
|
||||
}
|
||||
if err := workspaceFree(h.runner, d.Workspace); err == nil {
|
||||
t.Fatal("parked export did not reserve its workspace")
|
||||
}
|
||||
exhaustExportJob(t, h.runner, s, id)
|
||||
if err := h.runner.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
d, _ = s.GetRun(ctx, id)
|
||||
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 {
|
||||
t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load())
|
||||
}
|
||||
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
|
||||
t.Fatalf("gate survived: %v %v", blocked, err)
|
||||
}
|
||||
visit := d.Steps[0]
|
||||
if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" {
|
||||
t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery)
|
||||
}
|
||||
// The workspace is free for the next (for example scheduled) run.
|
||||
if err := workspaceFree(h.runner, d.Workspace); err != nil {
|
||||
t.Fatalf("workspace still reserved: %v", err)
|
||||
}
|
||||
// The operator can still repair the export later through the normal path.
|
||||
h.runner.exportSession = validGateExport
|
||||
repaired, err := h.runner.repairSessionExport(ctx, id, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repairs := repaired.Steps[0].SessionExportRepairs
|
||||
if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" {
|
||||
t.Fatalf("later repair not recorded: %+v", repairs)
|
||||
}
|
||||
if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" {
|
||||
t.Fatalf("successful repair did not complete the job: %+v", job)
|
||||
}
|
||||
}
|
||||
|
||||
// A run parked before the fix, whose job was already exhausted, is released by
|
||||
// the restarted server's startup reconciliation without any manual edit.
|
||||
func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
|
||||
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
|
||||
h.runner.wg.Wait()
|
||||
exhaustExportJob(t, h.runner, s, id)
|
||||
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parked, _ := reopened.GetRun(context.Background(), id)
|
||||
if parked.Status != "waiting" || parked.ExportPendingSequence != 1 {
|
||||
t.Fatalf("fixture is not a parked export: %+v", parked)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r, err := newRunner(ctx, reopened)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
||||
r.artifactsRoot = root
|
||||
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
if def.Key != "two" {
|
||||
t.Errorf("replayed owner: %s", def.Key)
|
||||
}
|
||||
second.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
d, _ := reopened.GetRun(context.Background(), id)
|
||||
if d.Status == "succeeded" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("startup did not release the exhausted gate: %+v", d)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
r.wg.Wait()
|
||||
if first.Load() != 1 || second.Load() != 1 {
|
||||
t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load())
|
||||
}
|
||||
if err := workspaceFree(r, parked.Workspace); err != nil {
|
||||
t.Fatalf("workspace still reserved after startup release: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,13 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
@@ -37,13 +39,8 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
|
||||
if err != nil {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
job, err := r.store.SessionExportJob(ctx, id, sequence)
|
||||
if err != nil || job == nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
|
||||
if err != nil || job == nil {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
@@ -54,34 +51,18 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
break
|
||||
}
|
||||
}
|
||||
verified := false
|
||||
if saved != nil {
|
||||
for _, e := range saved.SessionExportRepairs {
|
||||
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
|
||||
continue
|
||||
}
|
||||
// New repairs stay private in the original visit. Historical
|
||||
// public repairs remain valid, but are never rewritten there.
|
||||
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
|
||||
if strings.HasPrefix(e.HTML, "public/") {
|
||||
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
|
||||
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
|
||||
continue
|
||||
}
|
||||
if filepath.Clean(e.HTML) != e.HTML {
|
||||
continue
|
||||
}
|
||||
root, openErr := os.OpenRoot(directory)
|
||||
if openErr != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := root.Lstat(e.HTML)
|
||||
root.Close()
|
||||
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
|
||||
verified = true
|
||||
break
|
||||
}
|
||||
// An exhausted recovery job is terminal: the visit continues with its
|
||||
// recorded partial/unavailable evidence (still repairable later)
|
||||
// rather than holding the run and its workspace forever.
|
||||
exhausted := job.Status == "exhausted"
|
||||
verified := exhausted && saved != nil
|
||||
if !exhausted {
|
||||
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
|
||||
if err != nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
verified = saved != nil && r.verifiedExportRepair(id, sequence, saved, owner)
|
||||
}
|
||||
if !verified {
|
||||
r.mu.Unlock()
|
||||
@@ -91,6 +72,9 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
if exhausted {
|
||||
log.Printf("run %d visit %d: session export recovery exhausted (%s); continuing with incomplete session evidence", id, sequence, job.Diagnostic)
|
||||
}
|
||||
step := *saved
|
||||
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
|
||||
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
|
||||
@@ -98,3 +82,34 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifiedExportRepair reports whether a complete repair of the exact owner is
|
||||
// present on disk with its recorded size.
|
||||
func (r *runner) verifiedExportRepair(id int64, sequence int, saved *protocol.RunStep, owner pipeline.SessionOwner) bool {
|
||||
for _, e := range saved.SessionExportRepairs {
|
||||
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
|
||||
continue
|
||||
}
|
||||
// New repairs stay private in the original visit. Historical
|
||||
// public repairs remain valid, but are never rewritten there.
|
||||
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
|
||||
if strings.HasPrefix(e.HTML, "public/") {
|
||||
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
|
||||
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
|
||||
continue
|
||||
}
|
||||
if filepath.Clean(e.HTML) != e.HTML {
|
||||
continue
|
||||
}
|
||||
root, openErr := os.OpenRoot(directory)
|
||||
if openErr != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := root.Lstat(e.HTML)
|
||||
root.Close()
|
||||
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -28,7 +29,11 @@ func (r *runner) startSessionExportJobs() error {
|
||||
defer cancel()
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
nextReconcile := time.Now().Add(30 * time.Second)
|
||||
// The first tick (one second after startup, once the handler finished
|
||||
// wiring the runner) reconciles too, so a gate whose job was already
|
||||
// exhausted, or overran its bounded wait, before a restart is released
|
||||
// at once instead of reserving its workspace.
|
||||
nextReconcile := time.Now()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -37,7 +42,7 @@ func (r *runner) startSessionExportJobs() error {
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
if time.Now().After(nextReconcile) {
|
||||
if !time.Now().Before(nextReconcile) {
|
||||
nextReconcile = time.Now().Add(30 * time.Second)
|
||||
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
|
||||
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
|
||||
@@ -46,6 +51,14 @@ func (r *runner) startSessionExportJobs() error {
|
||||
log.Printf("session export recovery reconciliation failed: %v", err)
|
||||
continue
|
||||
}
|
||||
if expired, err := r.store.ExpireSessionExportGates(ctx, time.Now(), pipeline.SessionExportGateMaxWait); err != nil {
|
||||
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
|
||||
return
|
||||
}
|
||||
log.Printf("session export wait expiry failed: %v", err)
|
||||
} else if expired > 0 {
|
||||
log.Printf("session export recovery stopped for %d visit(s) that exceeded the %s wait", expired, pipeline.SessionExportGateMaxWait)
|
||||
}
|
||||
}
|
||||
// Serial bounded batches; each readback has a 10-second budget.
|
||||
for range 8 {
|
||||
@@ -107,6 +120,11 @@ func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bo
|
||||
diagnostic = last.Diagnostic
|
||||
if diagnostic == "" && last.Status != "complete" {
|
||||
diagnostic = "Automatic native session export remains " + last.Status
|
||||
if last.Truncated {
|
||||
// The usual cause is inline attachment payloads (for
|
||||
// example images read by a tool) inside the native record.
|
||||
diagnostic += fmt.Sprintf(": the native session exceeds the %d MiB export bound, so a retry cannot complete it", stage.SessionExportSourceLimit>>20)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user