Recover incomplete session exports through durable runtime jobs

This commit is contained in:
2026-10-20 16:28:14 +01:00
parent 6841cdd2a4
commit a049b552f6
14 changed files with 960 additions and 20 deletions
+6
View File
@@ -898,3 +898,9 @@ command is `repair_session_export` with `run_id` and `export_sequence`; the repl
is a run detail containing each visit's `session_export_repairs`. CLI JSON emits
the repair record with explicit completeness `status`, and incomplete repair
exits nonzero. Recovery still depends on upstream session availability.
Incomplete native stage-session exports also recover automatically after the
run settles: up to three durable runtime attempts, with backoff and restart-safe
leases. `inspect RUN_ID --json` includes each visit's `session_export_recovery`
status/attempt count/diagnostic and preserved `session_export_repairs`. Exhausted
automatic recovery remains a warning; manual versioned repair is still available.
+36
View File
@@ -43,6 +43,42 @@ There are at most 20 manual repair attempts per visit. Repairs cannot guarantee
recovery if the upstream session or original directory is gone. The web pane
explains original failure and lists repair attempts/links separately.
## Durable automatic recovery
Incomplete native exports now enqueue a `session_export_jobs` row in the same
transaction as `FinishStep`. The snapshot records the exact original session ID,
backend, directory and harness reference. Runtime startup also reconciles missing
jobs from persisted ownership/steps, including interrupted initial capture; it
never guesses missing ownership or edits original evidence. Reconciliation is
bounded to 256 newly discovered jobs per sweep (startup, then every 30 seconds).
A runtime-owned worker claims jobs only for settled runs/steps, with no active
agent in that run or using that session, and with `cleanup_unconfirmed=0` on
current main. The runner's in-memory cleanup gate also defers execution without
spending a retry. Each automatic attempt uses the existing versioned, read-only
repair path and verifies ownership against the durable snapshot before readback.
No stage prompt, pipeline definition, stage execution or run outcome changes.
The first attempt is due after 5 seconds; further attempts wait 20 and 60 seconds.
There are at most 3 automatic claims, each with the existing 10-second readback
budget. A 45-second persisted lease recovers interrupted workers after restart;
attempt counts/backoff are not reset, including process kills. Saved complete
repair evidence can acknowledge a job after a crash without exporting again.
Expired final-attempt leases become exhausted, not an infinite retry loop.
Wrong-session readbacks never qualify as complete evidence. Failed/empty/partial
repairs and exhausted-job diagnostics remain visible in run inspection and the
web visit pane (`session_export_recovery`). Originals remain unchanged.
Worker shutdown follows runner context cancellation and current main's
`Store.Done()` (including cancellation of in-flight readback). The small startup
helper detects that lifecycle interface without modifying this branch's older
store/server-lock implementation. Its compatibility fallback exits on a closed
database; current main's lifecycle/cleanup implementations must be retained when
integrating. Upstream loss, missing original ownership/directory, inventory
limits or sessions exceeding export caps can still leave exhausted evidence;
manual versioned repair remains available. Browser captures for recovery status
are not captured under the no-live-runs audit constraint.
No production service, live run or historical data was modified for this audit.
Unit/fixture tests cover reports, failure gates, retry inheritance, idempotency,
migration, transient/permanent export readback, empty-export diagnostics and
+7
View File
@@ -186,6 +186,13 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
return nil, err
}
for i := range r.Steps {
var recovery protocol.SessionExportRecovery
jobErr := tx.QueryRowContext(ctx, "SELECT status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?", id, r.Steps[i].Sequence).Scan(&recovery.Status, &recovery.Attempts, &recovery.NextAt, &recovery.Diagnostic)
if jobErr == nil {
r.Steps[i].SessionExportRecovery = &recovery
} else if jobErr != sql.ErrNoRows {
return nil, jobErr
}
repairs, err := tx.QueryContext(ctx, "SELECT export_json FROM session_export_repairs WHERE run_id=? AND sequence=? ORDER BY id", id, r.Steps[i].Sequence)
if err != nil {
return nil, err
+3
View File
@@ -145,6 +145,9 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome,
return err
}
}
if err := enqueueStepExportJob(ctx, tx, id, step.Sequence); err != nil {
return err
}
// The reported usage the stage already measured for budget accounting is
// persisted as step-scoped evidence in the same transaction, mirroring the
// other step evidence. It is written only when the stage actually reported
+4 -1
View File
@@ -79,7 +79,10 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints (
if err := s.dropResourceSchema(); err != nil {
return err
}
return s.migrateRunModels()
if err := s.migrateRunModels(); err != nil {
return err
}
return s.initSessionExportJobs()
}
// dropResourceSchema removes the retired resource pool, ownership, wait queue
+208
View File
@@ -0,0 +1,208 @@
package pipeline
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"fmt"
"time"
)
const SessionExportJobAttempts = 3
// SessionExportJob is an immutable ownership snapshot plus durable recovery
// state. Attempts count recovery operations, not the export CLI's inner reads.
type SessionExportJob struct {
RunID int64 `json:"run_id"`
Sequence int `json:"sequence"`
SessionID string `json:"session_id"`
Backend string `json:"backend"`
Directory string `json:"directory"`
Reference string `json:"-"`
Status string `json:"status"`
Attempts int `json:"attempts"`
NextAt int64 `json:"next_attempt_at_ms"`
Diagnostic string `json:"diagnostic,omitempty"`
Token string `json:"-"`
}
func (s *Store) initSessionExportJobs() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_jobs (
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL,
session_id TEXT NOT NULL, backend TEXT NOT NULL, directory TEXT NOT NULL, reference TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','complete','exhausted')), attempts INTEGER NOT NULL DEFAULT 0 CHECK(attempts BETWEEN 0 AND 3),
next_at INTEGER NOT NULL, lease_until INTEGER NOT NULL DEFAULT 0, token TEXT NOT NULL DEFAULT '', diagnostic TEXT NOT NULL DEFAULT '',
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE);
CREATE INDEX IF NOT EXISTS session_export_jobs_due ON session_export_jobs(status,next_at,lease_until);`)
return err
}
// Missing or incomplete original evidence is recoverable only with an exact
// native ownership record. No backend guesses, prompt parsing or stage reruns.
const enqueueExportJobs = `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at)
SELECT s.run_id,s.sequence,s.session_id,o.backend,r.workspace,o.reference,?
FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id
JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence
LEFT JOIN step_session_exports x ON x.run_id=s.run_id AND x.sequence=s.sequence
WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND o.backend IN ('v1','v2')
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=s.run_id AND j.sequence=s.sequence)
AND NOT (COALESCE(json_extract(x.export_json,'$.status'),'complete')='complete'
AND COALESCE(json_extract(x.export_json,'$.session_id'),'')=s.session_id
AND COALESCE(json_extract(x.export_json,'$.backend'),'')=o.backend
AND COALESCE(json_extract(x.export_json,'$.messages'),0)>0
AND COALESCE(json_extract(x.export_json,'$.html'),'')<>''
AND COALESCE(json_extract(x.export_json,'$.html_bytes'),0)>0
AND COALESCE(json_extract(x.export_json,'$.truncated'),0)=0
AND COALESCE(json_extract(x.export_json,'$.diagnostic'),'')='')`
func enqueueStepExportJob(ctx context.Context, tx *sql.Tx, id int64, sequence int) error {
_, err := tx.ExecContext(ctx, enqueueExportJobs+` AND s.run_id=? AND s.sequence=? ON CONFLICT(run_id,sequence) DO NOTHING`, time.Now().Add(5*time.Second).UnixMilli(), id, sequence)
return err
}
// ReconcileSessionExportJobs recovers interrupted initial capture and historical
// missing jobs from source-of-truth step ownership. Existing schedules, exhausted
// budgets and completed jobs are never reset, even across repeated restarts.
func (s *Store) ReconcileSessionExportJobs(ctx context.Context, now time.Time) error {
_, err := s.db.ExecContext(ctx, enqueueExportJobs+` LIMIT 256 ON CONFLICT(run_id,sequence) DO NOTHING`, now.Add(5*time.Second).UnixMilli())
return err
}
const exactExportJobOwner = `EXISTS(SELECT 1 FROM run_steps s JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence JOIN pipeline_runs r ON r.id=s.run_id WHERE s.run_id=j.run_id AND s.sequence=j.sequence AND s.session_id=j.session_id AND o.backend=j.backend AND o.reference=j.reference AND r.workspace=j.directory)`
// A complete repair must match the job's identity and have nonempty verified
// file evidence. This also closes the crash window after repair persistence but
// before acknowledgement of its job lease, without writing another export.
const completeExportJobEvidence = `EXISTS(SELECT 1 FROM session_export_repairs p WHERE p.run_id=j.run_id AND p.sequence=j.sequence
AND json_extract(p.export_json,'$.status')='complete'
AND json_extract(p.export_json,'$.session_id')=j.session_id
AND json_extract(p.export_json,'$.backend')=j.backend
AND json_extract(p.export_json,'$.directory')=j.directory
AND json_extract(p.export_json,'$.messages')>0 AND json_extract(p.export_json,'$.html_bytes')>0
AND COALESCE(json_extract(p.export_json,'$.html'),'')<>''
AND COALESCE(json_extract(p.export_json,'$.truncated'),0)=0
AND COALESCE(json_extract(p.export_json,'$.diagnostic'),'')='')`
// cleanupGuard supports the older audit branch without editing its store or
// cancellation schema. On current main the persisted flag is mandatory in the
// claim predicate. The legacy schema has no cleanup-unconfirmed state.
func (s *Store) exportJobCleanupGuard(ctx context.Context) (string, error) {
rows, err := s.db.QueryContext(ctx, "PRAGMA table_info(pipeline_runs)")
if err != nil {
return "", err
}
defer rows.Close()
for rows.Next() {
var cid, notnull, pk int
var name, kind string
var def sql.NullString
if err := rows.Scan(&cid, &name, &kind, &notnull, &def, &pk); err != nil {
return "", err
}
if name == "cleanup_unconfirmed" {
return " AND r.cleanup_unconfirmed=0", nil
}
}
return "", rows.Err()
}
func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease time.Duration) (*SessionExportJob, error) {
if lease <= 0 {
return nil, fmt.Errorf("session export recovery lease must be positive")
}
guard, err := s.exportJobCleanupGuard(ctx)
if err != nil {
return nil, err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return nil, err
}
defer tx.Rollback()
// A persisted successful repair wins, including after a process restart.
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE status<>'complete' AND `+completeExportJobEvidence); err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Original session ownership changed or is unavailable; automatic recovery refused' WHERE status IN ('pending','running') AND NOT `+exactExportJobOwner); err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Recovery budget exhausted; last worker stopped before acknowledgement' WHERE (status='pending' OR (status='running' AND lease_until<=?)) AND attempts>=?`, now.UnixMilli(), SessionExportJobAttempts); err != nil {
return nil, err
}
var job SessionExportJob
err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic
FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence
WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts<?
AND r.status IN ('succeeded','failed','canceled','interrupted') AND s.status NOT IN ('running','waiting')`+guard+`
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=j.run_id OR a.session_id=j.session_id) AND a.agent_active<>0)
ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
if err == sql.ErrNoRows {
return nil, tx.Commit()
}
if err != nil {
return nil, err
}
var nonce [16]byte
if _, err := rand.Read(nonce[:]); err != nil {
return nil, err
}
job.Token = hex.EncodeToString(nonce[:])
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='running',attempts=attempts+1,token=?,lease_until=? WHERE run_id=? AND sequence=?`, job.Token, now.Add(lease).UnixMilli(), job.RunID, job.Sequence); err != nil {
return nil, err
}
job.Attempts++
job.Status = "running"
if err := tx.Commit(); err != nil {
return nil, err
}
return &job, nil
}
// DeferSessionExportJob returns a lease without spending a retry when the
// runner's in-memory cleanup gate is still held. No native operation ran.
func (s *Store) DeferSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time) error {
return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status='pending',attempts=attempts-1,token='',lease_until=0,next_at=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, now.Add(5*time.Second).UnixMilli(), job.RunID, job.Sequence, job.Token))
}
func (s *Store) FinishSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time, diagnostic string) error {
// Never trust an optimistic worker return as proof of a full export.
result, err := s.db.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE run_id=? AND sequence=? AND status='running' AND token=? AND `+completeExportJobEvidence, job.RunID, job.Sequence, job.Token)
if err != nil {
return err
}
if n, _ := result.RowsAffected(); n == 1 {
return nil
}
status := "pending"
delay := 20 * time.Second
if job.Attempts == 2 {
delay = 60 * time.Second
}
if job.Attempts >= SessionExportJobAttempts {
status = "exhausted"
}
nextAt := now.Add(delay).UnixMilli()
if status == "exhausted" {
nextAt = 0
}
if diagnostic == "" {
diagnostic = "Native session evidence remains incomplete; upstream recovery is not guaranteed"
}
if len(diagnostic) > 2048 {
diagnostic = diagnostic[:2048]
}
return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status=?,token='',lease_until=0,next_at=?,diagnostic=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, status, nextAt, diagnostic, job.RunID, job.Sequence, job.Token))
}
func (s *Store) SessionExportJob(ctx context.Context, id int64, sequence int) (*SessionExportJob, error) {
var job SessionExportJob
err := s.db.QueryRowContext(ctx, `SELECT run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("session export recovery: %w", err)
}
return &job, nil
}
@@ -0,0 +1,311 @@
package pipeline
import (
"context"
"path/filepath"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func exportJobFixture(t *testing.T) (*Store, int64, string) {
t.Helper()
ctx := context.Background()
path := filepath.Join(t.TempDir(), "jobs.db")
s, err := Open(path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
if _, err := s.Submit(ctx, "fixture", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := s.List(ctx)
definition, _ := s.Get(ctx, items[0].ID, 1)
id, _, err := s.CreateRun(ctx, definition, "jobs", "/original/directory", false, nil, map[string]any{"prompt": "fixture"})
if err != nil {
t.Fatal(err)
}
if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
if err := s.RunSession(ctx, id, 1, "ses_original", false, "v2", "original-reference"); err != nil {
t.Fatal(err)
}
return s, id, path
}
func finishExportJobStep(t *testing.T, s *Store, id int64) {
t.Helper()
if err := s.FinishStep(context.Background(), id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Output: map[string]any{}, SessionExport: &protocol.SessionExport{Status: "empty", Messages: 0, SessionID: "ses_original", Backend: "v2"}}); err != nil {
t.Fatal(err)
}
}
func TestSessionExportJobEnqueueIsAtomicAndWaitsForSettlement(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
if _, err := s.db.Exec(`CREATE TRIGGER fail_usage BEFORE INSERT ON step_usage BEGIN SELECT RAISE(ABORT,'fixture usage failure'); END`); err != nil {
t.Fatal(err)
}
err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Output: map[string]any{}, Usage: &protocol.StageUsage{}})
if err == nil {
t.Fatal("fixture did not fail")
}
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job != nil {
t.Fatalf("job escaped rolled-back step: %+v %v", job, err)
}
if _, err := s.db.Exec("DROP TRIGGER fail_usage"); err != nil {
t.Fatal(err)
}
finishExportJobStep(t, s, id)
job, err = s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil || job.SessionID != "ses_original" || job.Directory != "/original/directory" || job.Reference != "original-reference" {
t.Fatalf("snapshot missing: %+v %v", job, err)
}
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
t.Fatalf("unsettled run claimed: %+v %v", claimed, err)
}
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
if claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt-1), time.Minute); err != nil || claimed != nil {
t.Fatalf("backoff ignored: %+v %v", claimed, err)
}
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
if err != nil || claimed == nil || claimed.Attempts != 1 {
t.Fatalf("settled due job not claimed: %+v %v", claimed, err)
}
now := time.UnixMilli(job.NextAt)
if err := s.FinishSessionExportJob(ctx, claimed, now, "empty"); err != nil {
t.Fatal(err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Attempts != 1 || job.Status != "pending" || job.NextAt != now.Add(20*time.Second).UnixMilli() {
t.Fatalf("first backoff wrong: %+v", job)
}
}
func TestSessionExportJobBackoffRestartAndExhaustion(t *testing.T) {
s, id, path := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
if err := s.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil {
t.Fatal(err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
now := time.UnixMilli(job.NextAt)
claimed, err := s.ClaimSessionExportJob(ctx, now, time.Minute)
if err != nil || claimed == nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := s.ReconcileSessionExportJobs(ctx, now); err != nil {
t.Fatal(err)
}
if other, err := s.ClaimSessionExportJob(ctx, now.Add(59*time.Second), time.Minute); err != nil || other != nil {
t.Fatalf("live lease stolen: %+v %v", other, err)
}
recovered, err := s.ClaimSessionExportJob(ctx, now.Add(time.Minute), time.Minute)
if err != nil || recovered == nil || recovered.Attempts != 2 || recovered.Token == claimed.Token {
t.Fatalf("restart did not recover lease: %+v %v", recovered, err)
}
if err := s.FinishSessionExportJob(ctx, claimed, now, "stale worker"); err == nil {
t.Fatal("stale lease overwrote recovery")
}
if err := s.FinishSessionExportJob(ctx, recovered, now, "upstream unavailable"); err != nil {
t.Fatal(err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "pending" || job.NextAt != now.Add(60*time.Second).UnixMilli() {
t.Fatalf("second backoff wrong: %+v", job)
}
last, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
if err != nil || last == nil || last.Attempts != 3 {
t.Fatalf("third attempt missing: %+v %v", last, err)
}
if err := s.FinishSessionExportJob(ctx, last, now.Add(time.Minute), "session gone"); err != nil {
t.Fatal(err)
}
if err := s.ReconcileSessionExportJobs(ctx, now.Add(time.Hour)); err != nil {
t.Fatal(err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 3 || job.Diagnostic != "session gone" {
t.Fatalf("budget reset: %+v", job)
}
if next, err := s.ClaimSessionExportJob(ctx, now.Add(24*time.Hour), time.Minute); err != nil || next != nil {
t.Fatalf("infinite retry: %+v %v", next, err)
}
}
func TestSessionExportJobGuardsCleanupAgentsAndExactOwnership(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec("ALTER TABLE pipeline_runs ADD COLUMN cleanup_unconfirmed INTEGER NOT NULL DEFAULT 0"); err != nil {
t.Fatal(err)
}
for _, queries := range [][2]string{
{"UPDATE pipeline_runs SET cleanup_unconfirmed=1", "UPDATE pipeline_runs SET cleanup_unconfirmed=0"},
{"UPDATE run_steps SET agent_active=1", "UPDATE run_steps SET agent_active=0"},
} {
if _, err := s.db.Exec(queries[0]); err != nil {
t.Fatal(err)
}
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
t.Fatalf("unsafe job claimed: %+v %v", job, err)
}
if _, err := s.db.Exec(queries[1]); err != nil {
t.Fatal(err)
}
}
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='v1'"); err != nil {
t.Fatal(err)
}
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
t.Fatalf("ownership drift exported: %+v %v", job, err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 0 || job.Backend != "v2" {
t.Fatalf("snapshot overwritten: %+v", job)
}
}
func TestSessionExportJobRecoversPersistedSuccessWithoutAnotherRead(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
t.Fatal(err)
}
job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute)
if err != nil || job == nil {
t.Fatal(err)
}
evidence := &protocol.SessionExport{Status: "complete", SessionID: "ses_wrong", Backend: job.Backend, Directory: job.Directory, HTML: "public/repaired.html", HTMLBytes: 100, Messages: 2}
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
t.Fatal(err)
}
if err := s.FinishSessionExportJob(ctx, job, time.Now(), "wrong identity"); err != nil {
t.Fatal(err)
}
state, _ := s.SessionExportJob(ctx, id, 1)
if state.Status == "complete" {
t.Fatal("wrong-session evidence marked clean")
}
evidence.SessionID = job.SessionID
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
t.Fatal(err)
}
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(24*time.Hour), time.Minute); err != nil || claimed != nil {
t.Fatalf("successful persisted repair repeated: %+v %v", claimed, err)
}
state, _ = s.SessionExportJob(ctx, id, 1)
if state.Status != "complete" || state.Attempts != 1 {
t.Fatalf("acknowledgement not recovered: %+v", state)
}
}
func TestSessionExportJobMigrationReconcilesMissingJobsWithoutReset(t *testing.T) {
s, id, path := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
original, err := s.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec("DROP TABLE session_export_jobs"); err != nil {
t.Fatal(err)
}
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := s.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
t.Fatal(err)
}
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil || job.SessionID != "ses_original" {
t.Fatalf("historical incomplete export not reconciled: %+v %v", job, err)
}
if err := s.ReconcileSessionExportJobs(ctx, time.Now().Add(time.Hour)); err != nil {
t.Fatal(err)
}
again, _ := s.SessionExportJob(ctx, id, 1)
if again.NextAt != job.NextAt || again.Attempts != job.Attempts {
t.Fatal("restart reset retry schedule")
}
current, err := s.GetRun(ctx, id)
if err != nil || current.Steps[0].SessionExport.Status != original.Steps[0].SessionExport.Status || current.Steps[0].SessionExport.Messages != 0 {
t.Fatal("migration changed original evidence")
}
}
func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) {
for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "wrong-session"} {
t.Run(status, func(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
evidence := &protocol.SessionExport{Status: status, SessionID: "ses_original", Backend: "v2", Messages: 2, HTML: "session.html", HTMLBytes: 100}
if status == "missing" {
evidence = nil
}
if status == "wrong-session" {
evidence.Status = "complete"
evidence.SessionID = "ses_wrong"
}
if status == "unsupported" {
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi'"); err != nil {
t.Fatal(err)
}
}
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", SessionExport: evidence, Output: map[string]any{}}); err != nil {
t.Fatal(err)
}
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
want := status != "complete" && status != "unsupported"
if (job != nil) != want {
t.Fatalf("%s: %+v", status, job)
}
})
}
}
func TestSessionExportJobFinalCrashedLeaseDoesNotResetBudget(t *testing.T) {
s, id, _ := exportJobFixture(t)
ctx := context.Background()
finishExportJobStep(t, s, id)
if err := s.FinishRun(ctx, id, "failed", "error", "", nil); err != nil {
t.Fatal(err)
}
if _, err := s.db.Exec("UPDATE session_export_jobs SET status='running',attempts=3,lease_until=0,token='lost'"); err != nil {
t.Fatal(err)
}
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
t.Fatalf("final crash retried: %+v %v", job, err)
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 3 || job.NextAt != 0 {
t.Fatalf("final lease not exhausted: %+v", job)
}
}
+28 -19
View File
@@ -125,6 +125,14 @@ type SessionExport struct {
Diagnostic string `json:"diagnostic,omitempty"`
}
// SessionExportRecovery describes the runtime's bounded durable repair job.
type SessionExportRecovery struct {
Status string `json:"status"`
Attempts int `json:"attempts"`
NextAt int64 `json:"next_attempt_at_ms"`
Diagnostic string `json:"diagnostic,omitempty"`
}
type RunSummary struct {
NotifyMode string `json:"notify_mode,omitempty"`
ID int64 `json:"id"`
@@ -168,25 +176,26 @@ func (r RunSummary) Label() string {
}
type RunStep struct {
Control *ControlEvidence `json:"control,omitempty"`
FileChanges *StageFileChanges `json:"file_changes,omitempty"`
Verdict *StageVerdict `json:"verdict,omitempty"`
SessionExport *SessionExport `json:"session_export,omitempty"`
SessionExportRepairs []*SessionExport `json:"session_export_repairs,omitempty"`
Usage *StageUsage `json:"usage,omitempty"`
Notifications []NotificationReceipt `json:"notifications,omitempty"`
Artifacts []StageArtifact `json:"artifacts,omitempty"`
Sequence int `json:"sequence"`
Stage string `json:"stage"`
Status string `json:"status"`
Outcome string `json:"outcome"`
Target string `json:"target"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Diagnostic string `json:"diagnostic"`
SessionID string `json:"session_id"`
AgentActive bool `json:"agent_active"`
Transcript string `json:"transcript"`
Control *ControlEvidence `json:"control,omitempty"`
FileChanges *StageFileChanges `json:"file_changes,omitempty"`
Verdict *StageVerdict `json:"verdict,omitempty"`
SessionExport *SessionExport `json:"session_export,omitempty"`
SessionExportRepairs []*SessionExport `json:"session_export_repairs,omitempty"`
SessionExportRecovery *SessionExportRecovery `json:"session_export_recovery,omitempty"`
Usage *StageUsage `json:"usage,omitempty"`
Notifications []NotificationReceipt `json:"notifications,omitempty"`
Artifacts []StageArtifact `json:"artifacts,omitempty"`
Sequence int `json:"sequence"`
Stage string `json:"stage"`
Status string `json:"status"`
Outcome string `json:"outcome"`
Target string `json:"target"`
Input map[string]any `json:"input"`
Output map[string]any `json:"output"`
Diagnostic string `json:"diagnostic"`
SessionID string `json:"session_id"`
AgentActive bool `json:"agent_active"`
Transcript string `json:"transcript"`
// Provider and Model are the effective provider/model this visit actually ran
// with, recorded for every model stage (including failures). They are empty
// for non-model stages and historical steps.
+3
View File
@@ -118,6 +118,9 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
}
}
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
if err := r.startSessionExportJobs(); err != nil {
return nil, err
}
r.schedulerWG.Add(1)
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
return r, nil
+116
View File
@@ -0,0 +1,116 @@
package server
import (
"context"
"log"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// startSessionExportJobs is the only runner startup integration. Current main's
// Store.Done stops the worker and in-flight readback even with Background callers.
// The interface avoids duplicating that newer store implementation/server lock.
func (r *runner) startSessionExportJobs() error {
if err := r.store.ReconcileSessionExportJobs(r.ctx, time.Now()); err != nil {
return err
}
var done <-chan struct{}
if lifecycle, ok := any(r.store).(interface{ Done() <-chan struct{} }); ok {
done = lifecycle.Done()
}
ctx, cancel := sessionExportWorkerContext(r.ctx, done)
r.schedulerWG.Add(1)
go func() {
defer r.schedulerWG.Done()
defer cancel()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
nextReconcile := time.Now().Add(30 * time.Second)
for {
select {
case <-ctx.Done():
return
case <-done:
return
case <-ticker.C:
}
if time.Now().After(nextReconcile) {
nextReconcile = time.Now().Add(30 * time.Second)
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export recovery reconciliation failed: %v", err)
continue
}
}
// Serial bounded batches; each readback has a 10-second budget.
for range 8 {
if ctx.Err() != nil {
return
}
worked, err := r.recoverSessionExportJob(ctx, time.Now())
if err != nil {
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
return
}
log.Printf("session export recovery failed: %v", err)
break
}
if !worked {
break
}
}
}
}()
return nil
}
func sessionExportWorkerContext(parent context.Context, done <-chan struct{}) (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(parent)
go func() {
select {
case <-ctx.Done():
case <-done:
cancel()
}
}()
return ctx, cancel
}
func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bool, error) {
job, err := r.store.ClaimSessionExportJob(ctx, now, 45*time.Second)
if err != nil || job == nil {
return false, err
}
r.mu.Lock()
_, executing := r.active[job.RunID]
r.mu.Unlock()
if executing {
return true, r.store.DeferSessionExportJob(ctx, job, now)
}
owner := &pipeline.SessionOwner{ID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Reference: job.Reference, RunID: job.RunID}
detail, repairErr := r.repairSessionExportOwned(ctx, job.RunID, job.Sequence, owner)
diagnostic := ""
if repairErr != nil {
diagnostic = stage.RedactEvidence(repairErr.Error())
} else if detail != nil {
for _, step := range detail.Steps {
if step.Sequence == job.Sequence && len(step.SessionExportRepairs) > 0 {
last := step.SessionExportRepairs[len(step.SessionExportRepairs)-1]
diagnostic = last.Diagnostic
if diagnostic == "" && last.Status != "complete" {
diagnostic = "Automatic native session export remains " + last.Status
}
}
}
}
// Save acknowledgement even on shutdown after an attempt. A process kill
// before this point leaves a recoverable, budgeted lease.
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
return true, r.store.FinishSessionExportJob(save, job, time.Now(), diagnostic)
}
+223
View File
@@ -0,0 +1,223 @@
package server
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) {
t.Helper()
h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) {
return stage.SessionExportResult{}, errors.New("initial readback unavailable")
})
// Synthetic executor has no agent process; model confirmed runtime cleanup.
if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil {
t.Fatal(err)
}
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("export recovery reran a stage")
return stage.Result{}, errors.New("forbidden rerun")
}
return h, s, id
}
func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
original, _ := s.GetRun(ctx, id)
before, _ := json.Marshal(original.Steps[0].SessionExport)
owner, _ := s.RepairSessionOwner(ctx, id, 1)
var calls atomic.Int32
h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) {
if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory {
t.Errorf("ownership changed: %+v", request)
}
if calls.Add(1) == 1 {
return stage.SessionExportResult{}, errors.New("temporary upstream read failure")
}
body := []byte("<!doctype html><p>recorded messages</p>")
if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil {
return stage.SessionExportResult{}, err
}
return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil
}
for i := 0; i < 2; i++ {
job, _ := s.SessionExportJob(ctx, id, 1)
worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt))
if err != nil || !worked {
t.Fatalf("attempt %d: %v %v", i, worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 {
t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load())
}
detail, _ := s.GetRun(ctx, id)
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 {
t.Fatalf("original evidence/execution changed: %+v", detail)
}
if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" {
t.Fatal("recovery state not visible")
}
}
func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
dir := t.TempDir()
calls := filepath.Join(dir, "calls")
bin := filepath.Join(dir, "opencode")
script := fmt.Sprintf("#!/bin/sh\necho read >> %q\necho '{\"info\":{\"id\":\"ses_wrong\"},\"messages\":[{\"info\":{\"role\":\"assistant\"},\"parts\":[{\"type\":\"text\",\"text\":\"wrong\"}]}]}'\n", calls)
if err := os.WriteFile(bin, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_OPENCODE", bin)
h.runner.exportSession = stage.ExportSession
for range 3 {
job, _ := s.SessionExportJob(ctx, id, 1)
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("job failed: %v %v", worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") {
t.Fatalf("wrong session looks clean: %+v", job)
}
if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked {
t.Fatalf("unbounded recovery: %v %v", worked, err)
}
body, _ := os.ReadFile(calls)
if strings.Count(string(body), "read") != 3 {
t.Fatalf("wrong-session read was retried internally: %s", body)
}
detail, _ := s.GetRun(ctx, id)
if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 {
t.Fatalf("lost failed attempts: %+v", detail.Steps)
}
for _, repair := range detail.Steps[0].SessionExportRepairs {
if repair.Status != "failed" || repair.HTML != "" {
t.Fatalf("wrong session marked clean: %+v", repair)
}
}
}
func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) {
h, s, id := recoveryRunner(t)
ctx := context.Background()
h.runner.mu.Lock()
h.runner.active[id] = activeRun{}
h.runner.mu.Unlock()
defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }()
job, _ := s.SessionExportJob(ctx, id, 1)
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("deferral failed: %v %v", worked, err)
}
job, _ = s.SessionExportJob(ctx, id, 1)
if job.Status != "pending" || job.Attempts != 0 {
t.Fatalf("cleanup spent attempt: %+v", job)
}
}
func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) {
for _, closeStore := range []bool{false, true} {
ctx, cancel := context.WithCancel(context.Background())
s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db"))
if err != nil {
t.Fatal(err)
}
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
if closeStore {
s.Close()
} else {
cancel()
}
done := make(chan struct{})
go func() { r.schedulerWG.Wait(); close(done) }()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("export worker did not stop")
}
cancel()
s.Close()
}
}
func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) {
h, s, id := recoveryRunner(t)
root := h.runner.artifactsRoot
path := filepath.Join(s.Directory(), "session.db")
original, _ := s.GetRun(context.Background(), id)
before, _ := json.Marshal(original.Steps[0].SessionExport)
if err := s.Close(); err != nil {
t.Fatal(err)
}
s, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessionExportCLI(t) // read-only native CLI fixture, never a live model/run
r, err := newRunner(ctx, s)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }()
r.artifactsRoot = root
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
t.Error("restart recovery reran a stage")
return stage.Result{}, errors.New("forbidden rerun")
}
deadline := time.Now().Add(9 * time.Second)
for {
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
if job != nil && job.Status == "complete" {
break
}
if time.Now().After(deadline) {
t.Fatalf("persisted job did not auto-recover: %+v", job)
}
time.Sleep(25 * time.Millisecond)
}
detail, err := s.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
after, _ := json.Marshal(detail.Steps[0].SessionExport)
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != original.Status {
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
}
}
func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) {
done := make(chan struct{})
ctx, cancel := sessionExportWorkerContext(context.Background(), done)
defer cancel()
exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout)
defer stop()
close(done)
select {
case <-exportCtx.Done():
case <-time.After(time.Second):
t.Fatal("Store.Done did not cancel in-flight readback context")
}
}
+8
View File
@@ -10,6 +10,7 @@ import (
"strconv"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
@@ -17,6 +18,10 @@ import (
// repairSessionExport is runtime-owned and read-only to OpenCode. It never
// resumes a session, runs a stage, changes a definition, or overwrites evidence.
func (r *runner) repairSessionExport(ctx context.Context, id int64, sequence int) (*protocol.RunDetail, error) {
return r.repairSessionExportOwned(ctx, id, sequence, nil)
}
func (r *runner) repairSessionExportOwned(ctx context.Context, id int64, sequence int, expected *pipeline.SessionOwner) (*protocol.RunDetail, error) {
r.exportRepairMu.Lock()
defer r.exportRepairMu.Unlock()
if id <= 0 || sequence <= 0 {
@@ -52,6 +57,9 @@ func (r *runner) repairSessionExport(ctx context.Context, id int64, sequence int
if err != nil {
return nil, fmt.Errorf("exact original session ownership metadata unavailable: %w", err)
}
if expected != nil && (owner.ID != expected.ID || owner.Backend != expected.Backend || owner.Directory != expected.Directory || owner.Reference != expected.Reference) {
return nil, fmt.Errorf("original session ownership differs from durable recovery snapshot; export refused")
}
active, err := r.store.ActiveSessionOwners(ctx)
if err != nil {
return nil, err
+5
View File
@@ -31,6 +31,11 @@ Audit additions (2026-10-20): **partial** — explicit run Report format selecto
empty/failed/partial explanations. Unit-tested; browser captures missing under
the audit's no-live-runs constraint. Existing captured states below are unchanged.
Automatic session export recovery additionally shows pending/running/complete/
exhausted job status and its bounded attempt count in the visit pane. Backend
recovery is fixture-tested; this new status line remains **partial**, without a
browser capture (no live runs were authorized).
## Transport and connection
| # | Feature (Android counterpart) | Protocol | Status |
| --- | --- | --- | --- |
+2
View File
@@ -1294,6 +1294,8 @@ function artifactAction(kind, label, disabled, onclick) {
function renderSessionExport(step) {
const file = sessionExportArtifact(step);
const wrap = h("div", { class: "session-export" });
const recovery = step?.session_export_recovery;
if (recovery) wrap.append(h("div", { class: "muted", text: `Automatic export recovery: ${recovery.status} · ${recovery.attempts}/3 attempts. ${recovery.diagnostic || ""}` }));
for (const repair of step?.session_export_repairs || []) {
const row = h("div", { class: "session-export-repair" });
row.append(h("span", { text: `Repair: ${repair.status || "partial"} · ${repair.messages || 0} messages. ${repair.diagnostic || ""}` }));