diff --git a/docs/cli.md b/docs/cli.md index 82646c43..972d6d43 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -898,3 +898,9 @@ command is `repair_session_export` with `run_id` and `export_sequence`; the repl is a run detail containing each visit's `session_export_repairs`. CLI JSON emits the repair record with explicit completeness `status`, and incomplete repair exits nonzero. Recovery still depends on upstream session availability. + +Incomplete native stage-session exports also recover automatically after the +run settles: up to three durable runtime attempts, with backoff and restart-safe +leases. `inspect RUN_ID --json` includes each visit's `session_export_recovery` +status/attempt count/diagnostic and preserved `session_export_repairs`. Exhausted +automatic recovery remains a warning; manual versioned repair is still available. diff --git a/docs/notes/audit-deliverables.md b/docs/notes/audit-deliverables.md index 6ce801cc..4c73c8f6 100644 --- a/docs/notes/audit-deliverables.md +++ b/docs/notes/audit-deliverables.md @@ -43,6 +43,42 @@ There are at most 20 manual repair attempts per visit. Repairs cannot guarantee recovery if the upstream session or original directory is gone. The web pane explains original failure and lists repair attempts/links separately. +## Durable automatic recovery + +Incomplete native exports now enqueue a `session_export_jobs` row in the same +transaction as `FinishStep`. The snapshot records the exact original session ID, +backend, directory and harness reference. Runtime startup also reconciles missing +jobs from persisted ownership/steps, including interrupted initial capture; it +never guesses missing ownership or edits original evidence. Reconciliation is +bounded to 256 newly discovered jobs per sweep (startup, then every 30 seconds). + +A runtime-owned worker claims jobs only for settled runs/steps, with no active +agent in that run or using that session, and with `cleanup_unconfirmed=0` on +current main. The runner's in-memory cleanup gate also defers execution without +spending a retry. Each automatic attempt uses the existing versioned, read-only +repair path and verifies ownership against the durable snapshot before readback. +No stage prompt, pipeline definition, stage execution or run outcome changes. + +The first attempt is due after 5 seconds; further attempts wait 20 and 60 seconds. +There are at most 3 automatic claims, each with the existing 10-second readback +budget. A 45-second persisted lease recovers interrupted workers after restart; +attempt counts/backoff are not reset, including process kills. Saved complete +repair evidence can acknowledge a job after a crash without exporting again. +Expired final-attempt leases become exhausted, not an infinite retry loop. +Wrong-session readbacks never qualify as complete evidence. Failed/empty/partial +repairs and exhausted-job diagnostics remain visible in run inspection and the +web visit pane (`session_export_recovery`). Originals remain unchanged. + +Worker shutdown follows runner context cancellation and current main's +`Store.Done()` (including cancellation of in-flight readback). The small startup +helper detects that lifecycle interface without modifying this branch's older +store/server-lock implementation. Its compatibility fallback exits on a closed +database; current main's lifecycle/cleanup implementations must be retained when +integrating. Upstream loss, missing original ownership/directory, inventory +limits or sessions exceeding export caps can still leave exhausted evidence; +manual versioned repair remains available. Browser captures for recovery status +are not captured under the no-live-runs audit constraint. + No production service, live run or historical data was modified for this audit. Unit/fixture tests cover reports, failure gates, retry inheritance, idempotency, migration, transient/permanent export readback, empty-export diagnostics and diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index 50594e15..bf642dda 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -186,6 +186,13 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ' return nil, err } for i := range r.Steps { + var recovery protocol.SessionExportRecovery + jobErr := tx.QueryRowContext(ctx, "SELECT status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?", id, r.Steps[i].Sequence).Scan(&recovery.Status, &recovery.Attempts, &recovery.NextAt, &recovery.Diagnostic) + if jobErr == nil { + r.Steps[i].SessionExportRecovery = &recovery + } else if jobErr != sql.ErrNoRows { + return nil, jobErr + } repairs, err := tx.QueryContext(ctx, "SELECT export_json FROM session_export_repairs WHERE run_id=? AND sequence=? ORDER BY id", id, r.Steps[i].Sequence) if err != nil { return nil, err diff --git a/internal/pipeline/runs_mutate.go b/internal/pipeline/runs_mutate.go index 0dfb69bb..608c6c62 100644 --- a/internal/pipeline/runs_mutate.go +++ b/internal/pipeline/runs_mutate.go @@ -145,6 +145,9 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome, return err } } + if err := enqueueStepExportJob(ctx, tx, id, step.Sequence); err != nil { + return err + } // The reported usage the stage already measured for budget accounting is // persisted as step-scoped evidence in the same transaction, mirroring the // other step evidence. It is written only when the stage actually reported diff --git a/internal/pipeline/runs_schema.go b/internal/pipeline/runs_schema.go index b3591738..1a1553c2 100644 --- a/internal/pipeline/runs_schema.go +++ b/internal/pipeline/runs_schema.go @@ -79,7 +79,10 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints ( if err := s.dropResourceSchema(); err != nil { return err } - return s.migrateRunModels() + if err := s.migrateRunModels(); err != nil { + return err + } + return s.initSessionExportJobs() } // dropResourceSchema removes the retired resource pool, ownership, wait queue diff --git a/internal/pipeline/session_export_jobs.go b/internal/pipeline/session_export_jobs.go new file mode 100644 index 00000000..a83f8594 --- /dev/null +++ b/internal/pipeline/session_export_jobs.go @@ -0,0 +1,208 @@ +package pipeline + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "fmt" + "time" +) + +const SessionExportJobAttempts = 3 + +// SessionExportJob is an immutable ownership snapshot plus durable recovery +// state. Attempts count recovery operations, not the export CLI's inner reads. +type SessionExportJob struct { + RunID int64 `json:"run_id"` + Sequence int `json:"sequence"` + SessionID string `json:"session_id"` + Backend string `json:"backend"` + Directory string `json:"directory"` + Reference string `json:"-"` + Status string `json:"status"` + Attempts int `json:"attempts"` + NextAt int64 `json:"next_attempt_at_ms"` + Diagnostic string `json:"diagnostic,omitempty"` + Token string `json:"-"` +} + +func (s *Store) initSessionExportJobs() error { + _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_jobs ( +run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, +session_id TEXT NOT NULL, backend TEXT NOT NULL, directory TEXT NOT NULL, reference TEXT NOT NULL, +status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','complete','exhausted')), attempts INTEGER NOT NULL DEFAULT 0 CHECK(attempts BETWEEN 0 AND 3), +next_at INTEGER NOT NULL, lease_until INTEGER NOT NULL DEFAULT 0, token TEXT NOT NULL DEFAULT '', diagnostic TEXT NOT NULL DEFAULT '', +PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE); +CREATE INDEX IF NOT EXISTS session_export_jobs_due ON session_export_jobs(status,next_at,lease_until);`) + return err +} + +// Missing or incomplete original evidence is recoverable only with an exact +// native ownership record. No backend guesses, prompt parsing or stage reruns. +const enqueueExportJobs = `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at) +SELECT s.run_id,s.sequence,s.session_id,o.backend,r.workspace,o.reference,? +FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id +JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence +LEFT JOIN step_session_exports x ON x.run_id=s.run_id AND x.sequence=s.sequence +WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND o.backend IN ('v1','v2') +AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=s.run_id AND j.sequence=s.sequence) +AND NOT (COALESCE(json_extract(x.export_json,'$.status'),'complete')='complete' +AND COALESCE(json_extract(x.export_json,'$.session_id'),'')=s.session_id +AND COALESCE(json_extract(x.export_json,'$.backend'),'')=o.backend +AND COALESCE(json_extract(x.export_json,'$.messages'),0)>0 +AND COALESCE(json_extract(x.export_json,'$.html'),'')<>'' +AND COALESCE(json_extract(x.export_json,'$.html_bytes'),0)>0 +AND COALESCE(json_extract(x.export_json,'$.truncated'),0)=0 +AND COALESCE(json_extract(x.export_json,'$.diagnostic'),'')='')` + +func enqueueStepExportJob(ctx context.Context, tx *sql.Tx, id int64, sequence int) error { + _, err := tx.ExecContext(ctx, enqueueExportJobs+` AND s.run_id=? AND s.sequence=? ON CONFLICT(run_id,sequence) DO NOTHING`, time.Now().Add(5*time.Second).UnixMilli(), id, sequence) + return err +} + +// ReconcileSessionExportJobs recovers interrupted initial capture and historical +// missing jobs from source-of-truth step ownership. Existing schedules, exhausted +// budgets and completed jobs are never reset, even across repeated restarts. +func (s *Store) ReconcileSessionExportJobs(ctx context.Context, now time.Time) error { + _, err := s.db.ExecContext(ctx, enqueueExportJobs+` LIMIT 256 ON CONFLICT(run_id,sequence) DO NOTHING`, now.Add(5*time.Second).UnixMilli()) + return err +} + +const exactExportJobOwner = `EXISTS(SELECT 1 FROM run_steps s JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence JOIN pipeline_runs r ON r.id=s.run_id WHERE s.run_id=j.run_id AND s.sequence=j.sequence AND s.session_id=j.session_id AND o.backend=j.backend AND o.reference=j.reference AND r.workspace=j.directory)` + +// A complete repair must match the job's identity and have nonempty verified +// file evidence. This also closes the crash window after repair persistence but +// before acknowledgement of its job lease, without writing another export. +const completeExportJobEvidence = `EXISTS(SELECT 1 FROM session_export_repairs p WHERE p.run_id=j.run_id AND p.sequence=j.sequence +AND json_extract(p.export_json,'$.status')='complete' +AND json_extract(p.export_json,'$.session_id')=j.session_id +AND json_extract(p.export_json,'$.backend')=j.backend +AND json_extract(p.export_json,'$.directory')=j.directory +AND json_extract(p.export_json,'$.messages')>0 AND json_extract(p.export_json,'$.html_bytes')>0 +AND COALESCE(json_extract(p.export_json,'$.html'),'')<>'' +AND COALESCE(json_extract(p.export_json,'$.truncated'),0)=0 +AND COALESCE(json_extract(p.export_json,'$.diagnostic'),'')='')` + +// cleanupGuard supports the older audit branch without editing its store or +// cancellation schema. On current main the persisted flag is mandatory in the +// claim predicate. The legacy schema has no cleanup-unconfirmed state. +func (s *Store) exportJobCleanupGuard(ctx context.Context) (string, error) { + rows, err := s.db.QueryContext(ctx, "PRAGMA table_info(pipeline_runs)") + if err != nil { + return "", err + } + defer rows.Close() + for rows.Next() { + var cid, notnull, pk int + var name, kind string + var def sql.NullString + if err := rows.Scan(&cid, &name, &kind, ¬null, &def, &pk); err != nil { + return "", err + } + if name == "cleanup_unconfirmed" { + return " AND r.cleanup_unconfirmed=0", nil + } + } + return "", rows.Err() +} + +func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease time.Duration) (*SessionExportJob, error) { + if lease <= 0 { + return nil, fmt.Errorf("session export recovery lease must be positive") + } + guard, err := s.exportJobCleanupGuard(ctx) + if err != nil { + return nil, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + // A persisted successful repair wins, including after a process restart. + if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE status<>'complete' AND `+completeExportJobEvidence); err != nil { + return nil, err + } + if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Original session ownership changed or is unavailable; automatic recovery refused' WHERE status IN ('pending','running') AND NOT `+exactExportJobOwner); err != nil { + return nil, err + } + if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Recovery budget exhausted; last worker stopped before acknowledgement' WHERE (status='pending' OR (status='running' AND lease_until<=?)) AND attempts>=?`, now.UnixMilli(), SessionExportJobAttempts); err != nil { + return nil, err + } + var job SessionExportJob + err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic +FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence +WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts0) +ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic) + if err == sql.ErrNoRows { + return nil, tx.Commit() + } + if err != nil { + return nil, err + } + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return nil, err + } + job.Token = hex.EncodeToString(nonce[:]) + if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='running',attempts=attempts+1,token=?,lease_until=? WHERE run_id=? AND sequence=?`, job.Token, now.Add(lease).UnixMilli(), job.RunID, job.Sequence); err != nil { + return nil, err + } + job.Attempts++ + job.Status = "running" + if err := tx.Commit(); err != nil { + return nil, err + } + return &job, nil +} + +// DeferSessionExportJob returns a lease without spending a retry when the +// runner's in-memory cleanup gate is still held. No native operation ran. +func (s *Store) DeferSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time) error { + return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status='pending',attempts=attempts-1,token='',lease_until=0,next_at=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, now.Add(5*time.Second).UnixMilli(), job.RunID, job.Sequence, job.Token)) +} + +func (s *Store) FinishSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time, diagnostic string) error { + // Never trust an optimistic worker return as proof of a full export. + result, err := s.db.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE run_id=? AND sequence=? AND status='running' AND token=? AND `+completeExportJobEvidence, job.RunID, job.Sequence, job.Token) + if err != nil { + return err + } + if n, _ := result.RowsAffected(); n == 1 { + return nil + } + status := "pending" + delay := 20 * time.Second + if job.Attempts == 2 { + delay = 60 * time.Second + } + if job.Attempts >= SessionExportJobAttempts { + status = "exhausted" + } + nextAt := now.Add(delay).UnixMilli() + if status == "exhausted" { + nextAt = 0 + } + if diagnostic == "" { + diagnostic = "Native session evidence remains incomplete; upstream recovery is not guaranteed" + } + if len(diagnostic) > 2048 { + diagnostic = diagnostic[:2048] + } + return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status=?,token='',lease_until=0,next_at=?,diagnostic=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, status, nextAt, diagnostic, job.RunID, job.Sequence, job.Token)) +} + +func (s *Store) SessionExportJob(ctx context.Context, id int64, sequence int) (*SessionExportJob, error) { + var job SessionExportJob + err := s.db.QueryRowContext(ctx, `SELECT run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("session export recovery: %w", err) + } + return &job, nil +} diff --git a/internal/pipeline/session_export_jobs_test.go b/internal/pipeline/session_export_jobs_test.go new file mode 100644 index 00000000..680c04c3 --- /dev/null +++ b/internal/pipeline/session_export_jobs_test.go @@ -0,0 +1,311 @@ +package pipeline + +import ( + "context" + "path/filepath" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func exportJobFixture(t *testing.T) (*Store, int64, string) { + t.Helper() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "jobs.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if _, err := s.Submit(ctx, "fixture", Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + items, _ := s.List(ctx) + definition, _ := s.Get(ctx, items[0].ID, 1) + id, _, err := s.CreateRun(ctx, definition, "jobs", "/original/directory", false, nil, map[string]any{"prompt": "fixture"}) + if err != nil { + t.Fatal(err) + } + if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil { + t.Fatal(err) + } + if err := s.RunSession(ctx, id, 1, "ses_original", false, "v2", "original-reference"); err != nil { + t.Fatal(err) + } + return s, id, path +} + +func finishExportJobStep(t *testing.T, s *Store, id int64) { + t.Helper() + if err := s.FinishStep(context.Background(), id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Output: map[string]any{}, SessionExport: &protocol.SessionExport{Status: "empty", Messages: 0, SessionID: "ses_original", Backend: "v2"}}); err != nil { + t.Fatal(err) + } +} + +func TestSessionExportJobEnqueueIsAtomicAndWaitsForSettlement(t *testing.T) { + s, id, _ := exportJobFixture(t) + ctx := context.Background() + if _, err := s.db.Exec(`CREATE TRIGGER fail_usage BEFORE INSERT ON step_usage BEGIN SELECT RAISE(ABORT,'fixture usage failure'); END`); err != nil { + t.Fatal(err) + } + err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Output: map[string]any{}, Usage: &protocol.StageUsage{}}) + if err == nil { + t.Fatal("fixture did not fail") + } + job, err := s.SessionExportJob(ctx, id, 1) + if err != nil || job != nil { + t.Fatalf("job escaped rolled-back step: %+v %v", job, err) + } + if _, err := s.db.Exec("DROP TRIGGER fail_usage"); err != nil { + t.Fatal(err) + } + finishExportJobStep(t, s, id) + job, err = s.SessionExportJob(ctx, id, 1) + if err != nil || job == nil || job.SessionID != "ses_original" || job.Directory != "/original/directory" || job.Reference != "original-reference" { + t.Fatalf("snapshot missing: %+v %v", job, err) + } + if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil { + t.Fatalf("unsettled run claimed: %+v %v", claimed, err) + } + if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { + t.Fatal(err) + } + if claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt-1), time.Minute); err != nil || claimed != nil { + t.Fatalf("backoff ignored: %+v %v", claimed, err) + } + claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute) + if err != nil || claimed == nil || claimed.Attempts != 1 { + t.Fatalf("settled due job not claimed: %+v %v", claimed, err) + } + now := time.UnixMilli(job.NextAt) + if err := s.FinishSessionExportJob(ctx, claimed, now, "empty"); err != nil { + t.Fatal(err) + } + job, _ = s.SessionExportJob(ctx, id, 1) + if job.Attempts != 1 || job.Status != "pending" || job.NextAt != now.Add(20*time.Second).UnixMilli() { + t.Fatalf("first backoff wrong: %+v", job) + } +} + +func TestSessionExportJobBackoffRestartAndExhaustion(t *testing.T) { + s, id, path := exportJobFixture(t) + ctx := context.Background() + finishExportJobStep(t, s, id) + if err := s.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil { + t.Fatal(err) + } + job, _ := s.SessionExportJob(ctx, id, 1) + now := time.UnixMilli(job.NextAt) + claimed, err := s.ClaimSessionExportJob(ctx, now, time.Minute) + if err != nil || claimed == nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err := s.ReconcileSessionExportJobs(ctx, now); err != nil { + t.Fatal(err) + } + if other, err := s.ClaimSessionExportJob(ctx, now.Add(59*time.Second), time.Minute); err != nil || other != nil { + t.Fatalf("live lease stolen: %+v %v", other, err) + } + recovered, err := s.ClaimSessionExportJob(ctx, now.Add(time.Minute), time.Minute) + if err != nil || recovered == nil || recovered.Attempts != 2 || recovered.Token == claimed.Token { + t.Fatalf("restart did not recover lease: %+v %v", recovered, err) + } + if err := s.FinishSessionExportJob(ctx, claimed, now, "stale worker"); err == nil { + t.Fatal("stale lease overwrote recovery") + } + if err := s.FinishSessionExportJob(ctx, recovered, now, "upstream unavailable"); err != nil { + t.Fatal(err) + } + job, _ = s.SessionExportJob(ctx, id, 1) + if job.Status != "pending" || job.NextAt != now.Add(60*time.Second).UnixMilli() { + t.Fatalf("second backoff wrong: %+v", job) + } + last, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute) + if err != nil || last == nil || last.Attempts != 3 { + t.Fatalf("third attempt missing: %+v %v", last, err) + } + if err := s.FinishSessionExportJob(ctx, last, now.Add(time.Minute), "session gone"); err != nil { + t.Fatal(err) + } + if err := s.ReconcileSessionExportJobs(ctx, now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + job, _ = s.SessionExportJob(ctx, id, 1) + if job.Status != "exhausted" || job.Attempts != 3 || job.Diagnostic != "session gone" { + t.Fatalf("budget reset: %+v", job) + } + if next, err := s.ClaimSessionExportJob(ctx, now.Add(24*time.Hour), time.Minute); err != nil || next != nil { + t.Fatalf("infinite retry: %+v %v", next, err) + } +} + +func TestSessionExportJobGuardsCleanupAgentsAndExactOwnership(t *testing.T) { + s, id, _ := exportJobFixture(t) + ctx := context.Background() + finishExportJobStep(t, s, id) + if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("ALTER TABLE pipeline_runs ADD COLUMN cleanup_unconfirmed INTEGER NOT NULL DEFAULT 0"); err != nil { + t.Fatal(err) + } + for _, queries := range [][2]string{ + {"UPDATE pipeline_runs SET cleanup_unconfirmed=1", "UPDATE pipeline_runs SET cleanup_unconfirmed=0"}, + {"UPDATE run_steps SET agent_active=1", "UPDATE run_steps SET agent_active=0"}, + } { + if _, err := s.db.Exec(queries[0]); err != nil { + t.Fatal(err) + } + if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil { + t.Fatalf("unsafe job claimed: %+v %v", job, err) + } + if _, err := s.db.Exec(queries[1]); err != nil { + t.Fatal(err) + } + } + if _, err := s.db.Exec("UPDATE run_session_owners SET backend='v1'"); err != nil { + t.Fatal(err) + } + if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil { + t.Fatalf("ownership drift exported: %+v %v", job, err) + } + job, _ := s.SessionExportJob(ctx, id, 1) + if job.Status != "exhausted" || job.Attempts != 0 || job.Backend != "v2" { + t.Fatalf("snapshot overwritten: %+v", job) + } +} + +func TestSessionExportJobRecoversPersistedSuccessWithoutAnotherRead(t *testing.T) { + s, id, _ := exportJobFixture(t) + ctx := context.Background() + finishExportJobStep(t, s, id) + if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil { + t.Fatal(err) + } + job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute) + if err != nil || job == nil { + t.Fatal(err) + } + evidence := &protocol.SessionExport{Status: "complete", SessionID: "ses_wrong", Backend: job.Backend, Directory: job.Directory, HTML: "public/repaired.html", HTMLBytes: 100, Messages: 2} + if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil { + t.Fatal(err) + } + if err := s.FinishSessionExportJob(ctx, job, time.Now(), "wrong identity"); err != nil { + t.Fatal(err) + } + state, _ := s.SessionExportJob(ctx, id, 1) + if state.Status == "complete" { + t.Fatal("wrong-session evidence marked clean") + } + evidence.SessionID = job.SessionID + if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil { + t.Fatal(err) + } + if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(24*time.Hour), time.Minute); err != nil || claimed != nil { + t.Fatalf("successful persisted repair repeated: %+v %v", claimed, err) + } + state, _ = s.SessionExportJob(ctx, id, 1) + if state.Status != "complete" || state.Attempts != 1 { + t.Fatalf("acknowledgement not recovered: %+v", state) + } +} + +func TestSessionExportJobMigrationReconcilesMissingJobsWithoutReset(t *testing.T) { + s, id, path := exportJobFixture(t) + ctx := context.Background() + finishExportJobStep(t, s, id) + original, err := s.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("DROP TABLE session_export_jobs"); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err := s.ReconcileSessionExportJobs(ctx, time.Now()); err != nil { + t.Fatal(err) + } + job, err := s.SessionExportJob(ctx, id, 1) + if err != nil || job == nil || job.SessionID != "ses_original" { + t.Fatalf("historical incomplete export not reconciled: %+v %v", job, err) + } + if err := s.ReconcileSessionExportJobs(ctx, time.Now().Add(time.Hour)); err != nil { + t.Fatal(err) + } + again, _ := s.SessionExportJob(ctx, id, 1) + if again.NextAt != job.NextAt || again.Attempts != job.Attempts { + t.Fatal("restart reset retry schedule") + } + current, err := s.GetRun(ctx, id) + if err != nil || current.Steps[0].SessionExport.Status != original.Steps[0].SessionExport.Status || current.Steps[0].SessionExport.Messages != 0 { + t.Fatal("migration changed original evidence") + } +} + +func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) { + for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "wrong-session"} { + t.Run(status, func(t *testing.T) { + s, id, _ := exportJobFixture(t) + ctx := context.Background() + evidence := &protocol.SessionExport{Status: status, SessionID: "ses_original", Backend: "v2", Messages: 2, HTML: "session.html", HTMLBytes: 100} + if status == "missing" { + evidence = nil + } + if status == "wrong-session" { + evidence.Status = "complete" + evidence.SessionID = "ses_wrong" + } + if status == "unsupported" { + if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi'"); err != nil { + t.Fatal(err) + } + } + if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", SessionExport: evidence, Output: map[string]any{}}); err != nil { + t.Fatal(err) + } + job, err := s.SessionExportJob(ctx, id, 1) + if err != nil { + t.Fatal(err) + } + want := status != "complete" && status != "unsupported" + if (job != nil) != want { + t.Fatalf("%s: %+v", status, job) + } + }) + } +} + +func TestSessionExportJobFinalCrashedLeaseDoesNotResetBudget(t *testing.T) { + s, id, _ := exportJobFixture(t) + ctx := context.Background() + finishExportJobStep(t, s, id) + if err := s.FinishRun(ctx, id, "failed", "error", "", nil); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("UPDATE session_export_jobs SET status='running',attempts=3,lease_until=0,token='lost'"); err != nil { + t.Fatal(err) + } + if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil { + t.Fatalf("final crash retried: %+v %v", job, err) + } + job, _ := s.SessionExportJob(ctx, id, 1) + if job.Status != "exhausted" || job.Attempts != 3 || job.NextAt != 0 { + t.Fatalf("final lease not exhausted: %+v", job) + } +} diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 0c7799ab..9c34ff12 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -125,6 +125,14 @@ type SessionExport struct { Diagnostic string `json:"diagnostic,omitempty"` } +// SessionExportRecovery describes the runtime's bounded durable repair job. +type SessionExportRecovery struct { + Status string `json:"status"` + Attempts int `json:"attempts"` + NextAt int64 `json:"next_attempt_at_ms"` + Diagnostic string `json:"diagnostic,omitempty"` +} + type RunSummary struct { NotifyMode string `json:"notify_mode,omitempty"` ID int64 `json:"id"` @@ -168,25 +176,26 @@ func (r RunSummary) Label() string { } type RunStep struct { - Control *ControlEvidence `json:"control,omitempty"` - FileChanges *StageFileChanges `json:"file_changes,omitempty"` - Verdict *StageVerdict `json:"verdict,omitempty"` - SessionExport *SessionExport `json:"session_export,omitempty"` - SessionExportRepairs []*SessionExport `json:"session_export_repairs,omitempty"` - Usage *StageUsage `json:"usage,omitempty"` - Notifications []NotificationReceipt `json:"notifications,omitempty"` - Artifacts []StageArtifact `json:"artifacts,omitempty"` - Sequence int `json:"sequence"` - Stage string `json:"stage"` - Status string `json:"status"` - Outcome string `json:"outcome"` - Target string `json:"target"` - Input map[string]any `json:"input"` - Output map[string]any `json:"output"` - Diagnostic string `json:"diagnostic"` - SessionID string `json:"session_id"` - AgentActive bool `json:"agent_active"` - Transcript string `json:"transcript"` + Control *ControlEvidence `json:"control,omitempty"` + FileChanges *StageFileChanges `json:"file_changes,omitempty"` + Verdict *StageVerdict `json:"verdict,omitempty"` + SessionExport *SessionExport `json:"session_export,omitempty"` + SessionExportRepairs []*SessionExport `json:"session_export_repairs,omitempty"` + SessionExportRecovery *SessionExportRecovery `json:"session_export_recovery,omitempty"` + Usage *StageUsage `json:"usage,omitempty"` + Notifications []NotificationReceipt `json:"notifications,omitempty"` + Artifacts []StageArtifact `json:"artifacts,omitempty"` + Sequence int `json:"sequence"` + Stage string `json:"stage"` + Status string `json:"status"` + Outcome string `json:"outcome"` + Target string `json:"target"` + Input map[string]any `json:"input"` + Output map[string]any `json:"output"` + Diagnostic string `json:"diagnostic"` + SessionID string `json:"session_id"` + AgentActive bool `json:"agent_active"` + Transcript string `json:"transcript"` // Provider and Model are the effective provider/model this visit actually ran // with, recorded for every model stage (including failures). They are empty // for non-model stages and historical steps. diff --git a/internal/server/runner.go b/internal/server/runner.go index fe0e2217..212cae9d 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -118,6 +118,9 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) { } } r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession} + if err := r.startSessionExportJobs(); err != nil { + return nil, err + } r.schedulerWG.Add(1) go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }() return r, nil diff --git a/internal/server/session_export_jobs.go b/internal/server/session_export_jobs.go new file mode 100644 index 00000000..15be1d5a --- /dev/null +++ b/internal/server/session_export_jobs.go @@ -0,0 +1,116 @@ +package server + +import ( + "context" + "log" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// startSessionExportJobs is the only runner startup integration. Current main's +// Store.Done stops the worker and in-flight readback even with Background callers. +// The interface avoids duplicating that newer store implementation/server lock. +func (r *runner) startSessionExportJobs() error { + if err := r.store.ReconcileSessionExportJobs(r.ctx, time.Now()); err != nil { + return err + } + var done <-chan struct{} + if lifecycle, ok := any(r.store).(interface{ Done() <-chan struct{} }); ok { + done = lifecycle.Done() + } + ctx, cancel := sessionExportWorkerContext(r.ctx, done) + r.schedulerWG.Add(1) + go func() { + defer r.schedulerWG.Done() + defer cancel() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + nextReconcile := time.Now().Add(30 * time.Second) + for { + select { + case <-ctx.Done(): + return + case <-done: + return + case <-ticker.C: + } + if time.Now().After(nextReconcile) { + nextReconcile = time.Now().Add(30 * time.Second) + if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil { + if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") { + return + } + log.Printf("session export recovery reconciliation failed: %v", err) + continue + } + } + // Serial bounded batches; each readback has a 10-second budget. + for range 8 { + if ctx.Err() != nil { + return + } + worked, err := r.recoverSessionExportJob(ctx, time.Now()) + if err != nil { + if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") { + return + } + log.Printf("session export recovery failed: %v", err) + break + } + if !worked { + break + } + } + } + }() + return nil +} + +func sessionExportWorkerContext(parent context.Context, done <-chan struct{}) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(parent) + go func() { + select { + case <-ctx.Done(): + case <-done: + cancel() + } + }() + return ctx, cancel +} + +func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bool, error) { + job, err := r.store.ClaimSessionExportJob(ctx, now, 45*time.Second) + if err != nil || job == nil { + return false, err + } + r.mu.Lock() + _, executing := r.active[job.RunID] + r.mu.Unlock() + if executing { + return true, r.store.DeferSessionExportJob(ctx, job, now) + } + owner := &pipeline.SessionOwner{ID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Reference: job.Reference, RunID: job.RunID} + detail, repairErr := r.repairSessionExportOwned(ctx, job.RunID, job.Sequence, owner) + diagnostic := "" + if repairErr != nil { + diagnostic = stage.RedactEvidence(repairErr.Error()) + } else if detail != nil { + for _, step := range detail.Steps { + if step.Sequence == job.Sequence && len(step.SessionExportRepairs) > 0 { + last := step.SessionExportRepairs[len(step.SessionExportRepairs)-1] + diagnostic = last.Diagnostic + if diagnostic == "" && last.Status != "complete" { + diagnostic = "Automatic native session export remains " + last.Status + } + } + } + } + // Save acknowledgement even on shutdown after an attempt. A process kill + // before this point leaves a recoverable, budgeted lease. + save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + return true, r.store.FinishSessionExportJob(save, job, time.Now(), diagnostic) +} diff --git a/internal/server/session_export_jobs_test.go b/internal/server/session_export_jobs_test.go new file mode 100644 index 00000000..09421faf --- /dev/null +++ b/internal/server/session_export_jobs_test.go @@ -0,0 +1,223 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) { + t.Helper() + h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) { + return stage.SessionExportResult{}, errors.New("initial readback unavailable") + }) + // Synthetic executor has no agent process; model confirmed runtime cleanup. + if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil { + t.Fatal(err) + } + h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + t.Error("export recovery reran a stage") + return stage.Result{}, errors.New("forbidden rerun") + } + return h, s, id +} + +func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) { + h, s, id := recoveryRunner(t) + ctx := context.Background() + original, _ := s.GetRun(ctx, id) + before, _ := json.Marshal(original.Steps[0].SessionExport) + owner, _ := s.RepairSessionOwner(ctx, id, 1) + var calls atomic.Int32 + h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) { + if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory { + t.Errorf("ownership changed: %+v", request) + } + if calls.Add(1) == 1 { + return stage.SessionExportResult{}, errors.New("temporary upstream read failure") + } + body := []byte("

recorded messages

") + if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil { + return stage.SessionExportResult{}, err + } + return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil + } + for i := 0; i < 2; i++ { + job, _ := s.SessionExportJob(ctx, id, 1) + worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)) + if err != nil || !worked { + t.Fatalf("attempt %d: %v %v", i, worked, err) + } + } + job, _ := s.SessionExportJob(ctx, id, 1) + if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 { + t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load()) + } + detail, _ := s.GetRun(ctx, id) + after, _ := json.Marshal(detail.Steps[0].SessionExport) + if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 { + t.Fatalf("original evidence/execution changed: %+v", detail) + } + if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" { + t.Fatal("recovery state not visible") + } +} + +func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) { + h, s, id := recoveryRunner(t) + ctx := context.Background() + dir := t.TempDir() + calls := filepath.Join(dir, "calls") + bin := filepath.Join(dir, "opencode") + script := fmt.Sprintf("#!/bin/sh\necho read >> %q\necho '{\"info\":{\"id\":\"ses_wrong\"},\"messages\":[{\"info\":{\"role\":\"assistant\"},\"parts\":[{\"type\":\"text\",\"text\":\"wrong\"}]}]}'\n", calls) + if err := os.WriteFile(bin, []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PIPELINES_OPENCODE", bin) + h.runner.exportSession = stage.ExportSession + for range 3 { + job, _ := s.SessionExportJob(ctx, id, 1) + if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked { + t.Fatalf("job failed: %v %v", worked, err) + } + } + job, _ := s.SessionExportJob(ctx, id, 1) + if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") { + t.Fatalf("wrong session looks clean: %+v", job) + } + if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked { + t.Fatalf("unbounded recovery: %v %v", worked, err) + } + body, _ := os.ReadFile(calls) + if strings.Count(string(body), "read") != 3 { + t.Fatalf("wrong-session read was retried internally: %s", body) + } + detail, _ := s.GetRun(ctx, id) + if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 { + t.Fatalf("lost failed attempts: %+v", detail.Steps) + } + for _, repair := range detail.Steps[0].SessionExportRepairs { + if repair.Status != "failed" || repair.HTML != "" { + t.Fatalf("wrong session marked clean: %+v", repair) + } + } +} + +func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) { + h, s, id := recoveryRunner(t) + ctx := context.Background() + h.runner.mu.Lock() + h.runner.active[id] = activeRun{} + h.runner.mu.Unlock() + defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }() + job, _ := s.SessionExportJob(ctx, id, 1) + if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked { + t.Fatalf("deferral failed: %v %v", worked, err) + } + job, _ = s.SessionExportJob(ctx, id, 1) + if job.Status != "pending" || job.Attempts != 0 { + t.Fatalf("cleanup spent attempt: %+v", job) + } +} + +func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) { + for _, closeStore := range []bool{false, true} { + ctx, cancel := context.WithCancel(context.Background()) + s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db")) + if err != nil { + t.Fatal(err) + } + r, err := newRunner(ctx, s) + if err != nil { + t.Fatal(err) + } + if closeStore { + s.Close() + } else { + cancel() + } + done := make(chan struct{}) + go func() { r.schedulerWG.Wait(); close(done) }() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("export worker did not stop") + } + cancel() + s.Close() + } +} + +func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) { + h, s, id := recoveryRunner(t) + root := h.runner.artifactsRoot + path := filepath.Join(s.Directory(), "session.db") + original, _ := s.GetRun(context.Background(), id) + before, _ := json.Marshal(original.Steps[0].SessionExport) + if err := s.Close(); err != nil { + t.Fatal(err) + } + s, err := pipeline.Open(path) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sessionExportCLI(t) // read-only native CLI fixture, never a live model/run + r, err := newRunner(ctx, s) + if err != nil { + t.Fatal(err) + } + defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }() + r.artifactsRoot = root + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + t.Error("restart recovery reran a stage") + return stage.Result{}, errors.New("forbidden rerun") + } + deadline := time.Now().Add(9 * time.Second) + for { + job, err := s.SessionExportJob(ctx, id, 1) + if err != nil { + t.Fatal(err) + } + if job != nil && job.Status == "complete" { + break + } + if time.Now().After(deadline) { + t.Fatalf("persisted job did not auto-recover: %+v", job) + } + time.Sleep(25 * time.Millisecond) + } + detail, err := s.GetRun(ctx, id) + if err != nil { + t.Fatal(err) + } + after, _ := json.Marshal(detail.Steps[0].SessionExport) + if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != original.Status { + t.Fatalf("recovery overwrote history or reran stage: %+v", detail) + } +} + +func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) { + done := make(chan struct{}) + ctx, cancel := sessionExportWorkerContext(context.Background(), done) + defer cancel() + exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout) + defer stop() + close(done) + select { + case <-exportCtx.Done(): + case <-time.After(time.Second): + t.Fatal("Store.Done did not cancel in-flight readback context") + } +} diff --git a/internal/server/session_export_repair.go b/internal/server/session_export_repair.go index 2a86e1e8..8f30f91b 100644 --- a/internal/server/session_export_repair.go +++ b/internal/server/session_export_repair.go @@ -10,6 +10,7 @@ import ( "strconv" "time" + "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) @@ -17,6 +18,10 @@ import ( // repairSessionExport is runtime-owned and read-only to OpenCode. It never // resumes a session, runs a stage, changes a definition, or overwrites evidence. func (r *runner) repairSessionExport(ctx context.Context, id int64, sequence int) (*protocol.RunDetail, error) { + return r.repairSessionExportOwned(ctx, id, sequence, nil) +} + +func (r *runner) repairSessionExportOwned(ctx context.Context, id int64, sequence int, expected *pipeline.SessionOwner) (*protocol.RunDetail, error) { r.exportRepairMu.Lock() defer r.exportRepairMu.Unlock() if id <= 0 || sequence <= 0 { @@ -52,6 +57,9 @@ func (r *runner) repairSessionExport(ctx context.Context, id int64, sequence int if err != nil { return nil, fmt.Errorf("exact original session ownership metadata unavailable: %w", err) } + if expected != nil && (owner.ID != expected.ID || owner.Backend != expected.Backend || owner.Directory != expected.Directory || owner.Reference != expected.Reference) { + return nil, fmt.Errorf("original session ownership differs from durable recovery snapshot; export refused") + } active, err := r.store.ActiveSessionOwners(ctx) if err != nil { return nil, err diff --git a/web/PARITY.md b/web/PARITY.md index 89711e8f..10c5d04c 100644 --- a/web/PARITY.md +++ b/web/PARITY.md @@ -31,6 +31,11 @@ Audit additions (2026-10-20): **partial** — explicit run Report format selecto empty/failed/partial explanations. Unit-tested; browser captures missing under the audit's no-live-runs constraint. Existing captured states below are unchanged. +Automatic session export recovery additionally shows pending/running/complete/ +exhausted job status and its bounded attempt count in the visit pane. Backend +recovery is fixture-tested; this new status line remains **partial**, without a +browser capture (no live runs were authorized). + ## Transport and connection | # | Feature (Android counterpart) | Protocol | Status | | --- | --- | --- | --- | diff --git a/web/src/views/runDetail.js b/web/src/views/runDetail.js index 80b0399f..4282407e 100644 --- a/web/src/views/runDetail.js +++ b/web/src/views/runDetail.js @@ -1294,6 +1294,8 @@ function artifactAction(kind, label, disabled, onclick) { function renderSessionExport(step) { const file = sessionExportArtifact(step); const wrap = h("div", { class: "session-export" }); + const recovery = step?.session_export_recovery; + if (recovery) wrap.append(h("div", { class: "muted", text: `Automatic export recovery: ${recovery.status} · ${recovery.attempts}/3 attempts. ${recovery.diagnostic || ""}` })); for (const repair of step?.session_export_repairs || []) { const row = h("div", { class: "session-export-repair" }); row.append(h("span", { text: `Repair: ${repair.status || "partial"} · ${repair.messages || 0} messages. ${repair.diagnostic || ""}` }));